diff --git a/.editorconfig b/.editorconfig index ef8ed24c52a5..6c8560aa1f54 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,13 +7,25 @@ root = true [*] end_of_line = lf charset = utf-8 -trim_trailing_whitespace = true insert_final_newline = true - -[!src/llvm-project] +trim_trailing_whitespace = true indent_style = space indent_size = 4 +# some tests need trailing whitespace in output snapshots +[tests/**] +trim_trailing_whitespace = false +# for actual source code files of test, we still don't want trailing whitespace +[tests/**.{rs,js}] +trim_trailing_whitespace = true +# these specific source files need to have trailing whitespace. +[tests/ui/{frontmatter/frontmatter-whitespace-3.rs,parser/shebang/shebang-space.rs}] +trim_trailing_whitespace = false + +[src/llvm-project] +indent_style = unset +indent_size = unset + [*.rs] max_line_length = 100 diff --git a/.github/ISSUE_TEMPLATE/documentation.yaml b/.github/ISSUE_TEMPLATE/documentation.yaml index 712b32759ae0..4a79cd4ba97c 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yaml +++ b/.github/ISSUE_TEMPLATE/documentation.yaml @@ -1,5 +1,5 @@ name: Documentation problem -description: Create a report for a documentation problem. +description: Report an issue with documentation content. labels: ["A-docs"] body: - type: markdown @@ -19,20 +19,20 @@ body: - [The Rustonomicon](https://github.com/rust-lang/nomicon/issues) - [The Embedded Book](https://github.com/rust-embedded/book/issues) - All other documentation issues should be filed here. + Or, if you find an issue related to rustdoc (e.g. doctest, rustdoc UI), please use the rustdoc issue template instead. - Or, if you find an issue related to rustdoc (e.g. doctest, rustdoc UI), please use the bug report or blank issue template instead. + All other documentation issues should be filed here. - type: textarea id: location attributes: - label: Location + label: Location (URL) validations: - required: true + required: true - type: textarea id: summary attributes: label: Summary validations: - required: true \ No newline at end of file + required: true diff --git a/.github/ISSUE_TEMPLATE/library_tracking_issue.md b/.github/ISSUE_TEMPLATE/library_tracking_issue.md index d56da9d5d025..de823dc300d1 100644 --- a/.github/ISSUE_TEMPLATE/library_tracking_issue.md +++ b/.github/ISSUE_TEMPLATE/library_tracking_issue.md @@ -51,6 +51,7 @@ If the feature is changed later, please add those PRs here as well. (Remember to update the `S-tracking-*` label when checking boxes.) +- [ ] ACP: rust-lang/libs-team#... - [ ] Implementation: #... - [ ] Final comment period (FCP)[^1] - [ ] Stabilization PR diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e91ef4abb120..d440b296d275 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: run_type: ${{ steps.jobs.outputs.run_type }} steps: - name: Checkout the source code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test citool # Only test citool on the auto branch, to reduce latency of the calculate matrix job # on PR/try builds. @@ -113,16 +113,16 @@ jobs: run: git config --global core.autocrlf false - name: checkout the source code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 2 - # Free up disk space on Linux and Windows by removing preinstalled components that + # Free up disk space on Linux by removing preinstalled components that # we do not need. We do this to enable some of the less resource # intensive jobs to run on free runners, which however also have # less disk space. - name: free up disk space - run: src/ci/scripts/free-disk-space.sh + run: src/ci/scripts/free-disk-space-linux.sh if: matrix.free_disk # If we don't need to free up disk space then just report how much space we have @@ -159,9 +159,6 @@ jobs: - name: show the current environment run: src/ci/scripts/dump-environment.sh - - name: install rust - run: src/ci/scripts/install-rust.sh - - name: install awscli run: src/ci/scripts/install-awscli.sh @@ -249,6 +246,8 @@ jobs: run: src/ci/scripts/create-doc-artifacts.sh - name: print disk usage + # We also want to know the disk usage when the job fails. + if: always() run: | echo "disk usage:" df -h @@ -316,7 +315,7 @@ jobs: if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }} steps: - name: checkout the source code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 2 # Calculate the exit status of the whole CI workflow. diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index 9d4b6192d6ea..80ffd67e04e1 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: checkout the source code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: submodules: recursive - name: install the bootstrap toolchain @@ -101,7 +101,7 @@ jobs: pull-requests: write steps: - name: checkout the source code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: download Cargo.lock from update job uses: actions/download-artifact@v4 diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 6d050d98cb2e..a89867efe666 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -29,7 +29,7 @@ jobs: # Needed to write to the ghcr.io registry packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: persist-credentials: false diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index ca088ba31fdf..12ff4be4f1e8 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -15,7 +15,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: # Make sure that we have enough commits to find the parent merge commit. # Since all merges should be through merge commits, fetching two commits diff --git a/.gitmodules b/.gitmodules index 439fde6d7660..8617643a1202 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,7 +25,7 @@ [submodule "src/llvm-project"] path = src/llvm-project url = https://github.com/rust-lang/llvm-project.git - branch = rustc/20.1-2025-07-13 + branch = rustc/21.1-2025-08-01 shallow = true [submodule "src/doc/embedded-book"] path = src/doc/embedded-book diff --git a/.mailmap b/.mailmap index 90533e81b39f..6c168bbbcb55 100644 --- a/.mailmap +++ b/.mailmap @@ -255,6 +255,7 @@ Guillaume Gomez Guillaume Gomez ggomez Guillaume Gomez Guillaume Gomez Guillaume Gomez Guillaume Gomez +gnzlbg hamidreza kalbasi Hanna Kruppe Heather @@ -426,6 +427,7 @@ Marcell Pardavi Marco Ieni <11428655+MarcoIeni@users.noreply.github.com> Marcus Klaas de Vries Margaret Meyerhofer +Marijn Schouten Mark Mansi Mark Mansi Mark Rousskov @@ -597,6 +599,7 @@ Sam Radhakrishnan Samuel Tardieu Santiago Pastorino Santiago Pastorino +Sasha Pourcelot Sasha Scott McMurray Scott McMurray Scott Olson Scott Olson @@ -607,6 +610,7 @@ Shohei Wada Shotaro Yamada Shotaro Yamada Shyam Sundar B +Sidney Cammeresi Simon Barber-Dueck Simon BD Simon Sapin Simonas Kazlauskas Simonas Kazlauskas @@ -676,6 +680,7 @@ Valerii Lashmanov Vitali Haravy Vitali Haravy Vitaly Shukela Waffle Lapkin +Waffle Lapkin Waffle Lapkin Weihang Lo Weihang Lo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aadc7c48ea83..2b5699dcd098 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,8 +31,8 @@ bootstrapping, the compiler architecture, source code representation, and more. ## [Getting help](https://rustc-dev-guide.rust-lang.org/getting-started.html#asking-questions) -There are many ways you can get help when you're stuck. Rust has many platforms for this: -[internals], [rust-zulip], and [rust-discord]. It is recommended to ask for help on +There are many ways you can get help when you're stuck. Rust has two platforms for this: +[internals] and [rust-zulip]. It is recommended to ask for help on the [rust-zulip], but any of these platforms are great ways to seek help and even find a mentor! You can learn more about asking questions and getting help in the [Asking Questions](https://rustc-dev-guide.rust-lang.org/getting-started.html#asking-questions) chapter of the [rustc-dev-guide]. @@ -47,5 +47,4 @@ refer to [this section][contributing-bug-reports] and [open an issue][issue temp [contributing-bug-reports]: https://rustc-dev-guide.rust-lang.org/contributing.html#bug-reports [issue template]: https://github.com/rust-lang/rust/issues/new/choose [internals]: https://internals.rust-lang.org -[rust-discord]: http://discord.gg/rust-lang [rust-zulip]: https://rust-lang.zulipchat.com diff --git a/Cargo.lock b/Cargo.lock index c1076f05ef12..a838c0775b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,14 +75,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "710e8eae58854cdc1790fcb56cca04d712a17be849eeb81da2a724bf4bae2bc4" dependencies = [ "anstyle", - "unicode-width 0.2.1", + "unicode-width 0.2.2", +] + +[[package]] +name = "annotate-snippets" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47224528f74de27d1d06aad6a5dda4f865b6ebe2e56c538943d746a7270cb67e" +dependencies = [ + "anstyle", + "unicode-width 0.2.2", ] [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -95,9 +105,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-lossy" @@ -119,50 +129,50 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-svg" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a43964079ef399480603125d5afae2b219aceffb77478956e25f17b9bc3435c" +checksum = "26b9ec8c976eada1b0f9747a3d7cc4eae3bef10613e443746e7487f26c872fde" dependencies = [ "anstyle", "anstyle-lossy", "anstyle-parse", "html-escape", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "ar_archive_writer" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01667f6f40216b9a0b2945e05fed5f1ad0ab6470e69cb9378001e37b1c0668e4" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ - "object 0.36.7", + "object 0.37.3", ] [[package]] @@ -204,7 +214,7 @@ dependencies = [ "rustc-hash 2.1.1", "serde", "serde_derive", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -216,7 +226,7 @@ dependencies = [ "memchr", "serde", "serde_derive", - "winnow 0.7.12", + "winnow 0.7.13", ] [[package]] @@ -266,9 +276,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" [[package]] name = "blake3" @@ -315,7 +325,7 @@ dependencies = [ "serde_json", "sha2", "tar", - "toml 0.5.11", + "toml 0.7.8", "xz2", ] @@ -334,9 +344,11 @@ dependencies = [ "anyhow", "build_helper", "curl", + "hex", "indexmap", "serde", - "toml 0.5.11", + "sha2", + "toml 0.8.23", ] [[package]] @@ -353,9 +365,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "camino" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0da45bc31171d8d6960122e222a67740df867c1dd53b4d51caa297084c185cab" +checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" dependencies = [ "serde", ] @@ -421,7 +433,7 @@ dependencies = [ "serde", "serde-untagged", "serde-value", - "thiserror 2.0.12", + "thiserror 2.0.15", "toml 0.8.23", "unicode-xid", "url", @@ -453,7 +465,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.15", ] [[package]] @@ -493,7 +505,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -518,9 +530,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" dependencies = [ "clap_builder", "clap_derive", @@ -538,9 +550,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" dependencies = [ "anstream", "anstyle", @@ -550,14 +562,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -568,7 +580,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clippy" -version = "0.1.90" +version = "0.1.93" dependencies = [ "anstream", "askama", @@ -588,14 +600,14 @@ dependencies = [ "serde_json", "tempfile", "termize", - "toml 0.7.8", + "toml 0.9.7", "ui_test", "walkdir", ] [[package]] name = "clippy_config" -version = "0.1.90" +version = "0.1.93" dependencies = [ "clippy_utils", "itertools", @@ -618,7 +630,7 @@ dependencies = [ [[package]] name = "clippy_lints" -version = "0.1.90" +version = "0.1.93" dependencies = [ "arrayvec", "cargo_metadata 0.18.1", @@ -630,7 +642,7 @@ dependencies = [ "regex-syntax 0.8.5", "semver", "serde", - "toml 0.7.8", + "toml 0.9.7", "unicode-normalization", "unicode-script", "url", @@ -643,13 +655,14 @@ version = "0.0.1" dependencies = [ "clippy_config", "clippy_utils", + "itertools", "regex", "rustc-semver", ] [[package]] name = "clippy_utils" -version = "0.1.90" +version = "0.1.93" dependencies = [ "arrayvec", "itertools", @@ -674,7 +687,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -684,6 +697,7 @@ dependencies = [ "anyhow", "serde", "serde_json", + "similar", "spdx-rs", ] @@ -720,7 +734,7 @@ dependencies = [ "nom", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -804,7 +818,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.1", + "unicode-width 0.2.2", "windows-sys 0.59.0", ] @@ -907,9 +921,9 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.48" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2d5c8f48d9c0c23250e52b55e82a6ab4fdba6650c931f5a0a57a43abda812b" +checksum = "79fc3b6dd0b87ba36e565715bf9a2ced221311db47bd18011676f24a6066edbc" dependencies = [ "curl-sys", "libc", @@ -922,9 +936,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.82+curl-8.14.1" +version = "0.4.83+curl-8.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4d63638b5ec65f1a4ae945287b3fd035be4554bbaf211901159c9a2a74fb5be" +checksum = "5830daf304027db10c82632a464879d46a3f7c4ba17a31592657ad16c719b483" dependencies = [ "cc", "libc", @@ -937,23 +951,24 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.161" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3523cc02ad831111491dd64b27ad999f1ae189986728e477604e61b81f828df" +checksum = "2f81de88da10862f22b5b3a60f18f6f42bbe7cb8faa24845dd7b1e4e22190e77" dependencies = [ "cc", + "cxx-build", "cxxbridge-cmd", "cxxbridge-flags", "cxxbridge-macro", - "foldhash", + "foldhash 0.2.0", "link-cplusplus", ] [[package]] name = "cxx-build" -version = "1.0.161" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212b754247a6f07b10fa626628c157593f0abf640a3dd04cce2760eca970f909" +checksum = "5edd58bf75c3fdfc80d79806403af626570662f7b6cc782a7fabe156166bd6d6" dependencies = [ "cc", "codespan-reporting", @@ -961,40 +976,40 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "cxxbridge-cmd" -version = "1.0.161" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f426a20413ec2e742520ba6837c9324b55ffac24ead47491a6e29f933c5b135a" +checksum = "fd46bf2b541a4e0c2d5abba76607379ee05d68e714868e3cb406dc8d591ce2d2" dependencies = [ "clap", "codespan-reporting", "indexmap", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "cxxbridge-flags" -version = "1.0.161" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258b6069020b4e5da6415df94a50ee4f586a6c38b037a180e940a43d06a070d" +checksum = "2c79b68f6a3a8f809d39b38ae8af61305a6113819b19b262643b9c21353b92d9" [[package]] name = "cxxbridge-macro" -version = "1.0.161" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dec184b52be5008d6eaf7e62fc1802caf1ad1227d11b3b7df2c409c7ffc3f4" +checksum = "862b7fdb048ff9ef0779a0d0a03affd09746c4c875543746b640756be9cff2af" dependencies = [ "indexmap", "proc-macro2", "quote", "rustversion", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1018,7 +1033,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1029,7 +1044,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1051,17 +1066,17 @@ dependencies = [ [[package]] name = "declare_clippy_lint" -version = "0.1.90" +version = "0.1.93" [[package]] name = "derive-where" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510c292c8cf384b1a340b816a9a6cf2599eb8f566a44949024af88418000c50b" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1082,7 +1097,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1092,7 +1107,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1104,7 +1119,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1129,16 +1144,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" dependencies = [ - "dirs-sys 0.5.0", + "dirs-sys", ] [[package]] name = "dirs" -version = "5.0.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -1151,18 +1166,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.6", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys" version = "0.5.0" @@ -1171,8 +1174,8 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.0", - "windows-sys 0.60.2", + "redox_users 0.5.2", + "windows-sys 0.61.2", ] [[package]] @@ -1194,7 +1197,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1203,6 +1206,12 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" +[[package]] +name = "dyn-clone" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" + [[package]] name = "either" version = "1.15.0" @@ -1279,7 +1288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -1337,6 +1346,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" + [[package]] name = "flate2" version = "1.1.2" @@ -1379,7 +1394,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54f0d287c53ffd184d04d8677f590f4ac5379785529e5e08b1c8083acdd5c198" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.15", ] [[package]] @@ -1394,6 +1409,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1470,11 +1491,11 @@ dependencies = [ [[package]] name = "getopts" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -1539,9 +1560,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "globset" @@ -1567,13 +1588,13 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", "serde", ] @@ -1686,30 +1707,40 @@ checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", "potential_utf", - "yoke 0.8.0", + "yoke", "zerofrom", - "zerovec 0.11.2", + "zerovec", ] [[package]] name = "icu_list" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfeda1d7775b6548edd4e8b7562304a559a91ed56ab56e18961a053f367c365" +checksum = "e26f94ec776bb8b28cedc7dcf91033b822c5cb4c1783cf7a3f796fc168aa0c8b" dependencies = [ "displaydoc", - "icu_list_data", - "icu_locid_transform", - "icu_provider 1.5.0", - "regex-automata 0.2.0", - "writeable 0.5.5", + "icu_provider", + "regex-automata 0.4.9", + "serde", + "writeable", + "zerovec", ] [[package]] -name = "icu_list_data" -version = "1.5.1" +name = "icu_locale" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52b1a7fbdbf3958f1be8354cb59ac73f165b7b7082d447ff2090355c9a069120" +checksum = "6ae5921528335e91da1b6c695dbf1ec37df5ac13faa3f91e5640be93aa2fbefd" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] [[package]] name = "icu_locale_core" @@ -1718,44 +1749,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", - "litemap 0.8.0", - "tinystr 0.8.1", - "writeable 0.6.1", - "zerovec 0.11.2", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_data" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap 0.7.5", - "tinystr 0.7.6", - "writeable 0.5.5", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider 1.5.0", - "tinystr 0.7.6", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" +checksum = "4fdef0c124749d06a743c69e938350816554eb63ac979166590e2b4ee4252765" [[package]] name = "icu_normalizer" @@ -1767,9 +1771,9 @@ dependencies = [ "icu_collections", "icu_normalizer_data", "icu_properties", - "icu_provider 2.0.0", + "icu_provider", "smallvec", - "zerovec 0.11.2", + "zerovec", ] [[package]] @@ -1788,10 +1792,10 @@ dependencies = [ "icu_collections", "icu_locale_core", "icu_properties_data", - "icu_provider 2.0.0", + "icu_provider", "potential_utf", "zerotrie", - "zerovec 0.11.2", + "zerovec", ] [[package]] @@ -1800,23 +1804,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr 0.7.6", - "writeable 0.5.5", - "yoke 0.7.5", - "zerofrom", - "zerovec 0.10.4", -] - [[package]] name = "icu_provider" version = "2.0.0" @@ -1826,36 +1813,12 @@ dependencies = [ "displaydoc", "icu_locale_core", "stable_deref_trait", - "tinystr 0.8.1", - "writeable 0.6.1", - "yoke 0.8.0", + "tinystr", + "writeable", + "yoke", "zerofrom", "zerotrie", - "zerovec 0.11.2", -] - -[[package]] -name = "icu_provider_adapters" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6324dfd08348a8e0374a447ebd334044d766b1839bb8d5ccf2482a99a77c0bc" -dependencies = [ - "icu_locid", - "icu_locid_transform", - "icu_provider 1.5.0", - "tinystr 0.7.6", - "zerovec 0.10.4", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", + "zerovec", ] [[package]] @@ -1909,19 +1872,20 @@ dependencies = [ [[package]] name = "indenter" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" [[package]] name = "indexmap" -version = "2.10.0" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", "hashbrown", "serde", + "serde_core", ] [[package]] @@ -1933,7 +1897,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.1", + "unicode-width 0.2.2", "web-time", ] @@ -2044,7 +2008,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2094,15 +2058,15 @@ dependencies = [ [[package]] name = "jsonpath-rust" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d057f8fd19e20c3f14d3663983397155739b6bc1148dc5cd4c4a1a5b3130eb0" +checksum = "633a7320c4bb672863a3782e89b9094ad70285e097ff6832cddd0ec615beadfa" dependencies = [ "pest", "pest_derive", "regex", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.15", ] [[package]] @@ -2137,9 +2101,9 @@ checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libdbus-sys" @@ -2153,9 +2117,9 @@ dependencies = [ [[package]] name = "libffi" -version = "4.1.1" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7681c6fab541f799a829e44a445a0666cf8d8a6cfebf89419e6aed52c604e87" +checksum = "0444124f3ffd67e1b0b0c661a7f81a278a135eb54aaad4078e79fbc8be50c8a5" dependencies = [ "libc", "libffi-sys", @@ -2163,9 +2127,9 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "3.3.2" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0d828d367b4450ed08e7d510dc46636cd660055f50d67ac943bfe788767c29" +checksum = "3d722da8817ea580d0669da6babe2262d7b86a1af1103da24102b8bb9c101ce7" dependencies = [ "cc", ] @@ -2190,7 +2154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.2", + "windows-targets 0.52.6", ] [[package]] @@ -2201,9 +2165,9 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4488594b9328dee448adb906d8b126d9b7deb7cf5c22161ee591610bb1be83c0" +checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ "bitflags", "libc", @@ -2224,9 +2188,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6f6da007f968f9def0d65a05b187e2960183de70c160204ecfccf0ee330212" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" dependencies = [ "cc", ] @@ -2237,6 +2201,7 @@ version = "0.1.0" dependencies = [ "html5ever", "regex", + "urlencoding", ] [[package]] @@ -2251,15 +2216,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" - -[[package]] -name = "litemap" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -2337,7 +2296,7 @@ checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2422,11 +2381,11 @@ dependencies = [ [[package]] name = "miow" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "359f76430b20a79f9e20e115b3428614e654f04fab314482fc0fda0ebd3c6044" +checksum = "536bfad37a309d62069485248eeaba1e8d9853aaf951caaeaed0585a95346f08" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2643,9 +2602,9 @@ dependencies = [ [[package]] name = "object" -version = "0.37.1" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fd943161069e1768b4b3d050890ba48730e590f57e56d4aa04e7e090e61b4a" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "crc32fast", "flate2", @@ -2653,7 +2612,7 @@ dependencies = [ "indexmap", "memchr", "ruzstd 0.8.1", - "wasmparser 0.234.0", + "wasmparser 0.236.1", ] [[package]] @@ -2834,7 +2793,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.15", "ucd-trie", ] @@ -2858,7 +2817,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2971,7 +2930,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" dependencies = [ - "zerovec 0.11.2", + "serde", + "zerovec", ] [[package]] @@ -3007,9 +2967,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -3147,9 +3107,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -3157,9 +3117,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -3167,9 +3127,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7251471db004e509f4e75a62cca9435365b5ec7bcdff530d612ac7c87c44a792" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ "bitflags", ] @@ -3187,13 +3147,33 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.12", + "thiserror 2.0.15", +] + +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] @@ -3217,15 +3197,6 @@ dependencies = [ "regex-syntax 0.6.29", ] -[[package]] -name = "regex-automata" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9368763f5a9b804326f3af749e16f9abf378d227bcdee7634b13d8f17793782" -dependencies = [ - "memchr", -] - [[package]] name = "regex-automata" version = "0.4.9" @@ -3279,18 +3250,18 @@ dependencies = [ "build_helper", "gimli 0.32.0", "libc", - "object 0.37.1", + "object 0.37.3", "regex", "serde_json", "similar", - "wasmparser 0.219.2", + "wasmparser 0.236.1", ] [[package]] name = "rustc-build-sysroot" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb13874a0e55baf4ac3d49d38206aecb31a55b75d6c4d04fd850b53942c8cc8" +checksum = "3b881c015c729b43105bbd3702a9bdecee28fafaa21126d1d62e454ec011a4b7" dependencies = [ "anyhow", "rustc_version", @@ -3301,9 +3272,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustc-hash" @@ -3332,6 +3303,7 @@ dependencies = [ "rustc_driver_impl", "rustc_public", "rustc_public_bridge", + "rustc_windows_rc", "tikv-jemalloc-sys", ] @@ -3367,6 +3339,7 @@ dependencies = [ "rand 0.9.2", "rand_xoshiro", "rustc_data_structures", + "rustc_error_messages", "rustc_hashes", "rustc_index", "rustc_macros", @@ -3417,6 +3390,7 @@ dependencies = [ "rustc_data_structures", "rustc_macros", "rustc_serialize", + "rustc_span", ] [[package]] @@ -3426,7 +3400,6 @@ dependencies = [ "rustc_abi", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -3458,7 +3431,6 @@ dependencies = [ "rustc_feature", "rustc_fluent_macro", "rustc_macros", - "rustc_parse", "rustc_session", "rustc_span", "rustc_target", @@ -3476,20 +3448,6 @@ dependencies = [ "thin-vec", ] -[[package]] -name = "rustc_attr_data_structures" -version = "0.0.0" -dependencies = [ - "rustc_abi", - "rustc_ast", - "rustc_ast_pretty", - "rustc_data_structures", - "rustc_macros", - "rustc_serialize", - "rustc_span", - "thin-vec", -] - [[package]] name = "rustc_attr_parsing" version = "0.0.0" @@ -3497,15 +3455,16 @@ dependencies = [ "rustc_abi", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_errors", "rustc_feature", "rustc_fluent_macro", "rustc_hir", "rustc_lexer", "rustc_macros", + "rustc_parse", "rustc_session", "rustc_span", + "rustc_target", "thin-vec", ] @@ -3514,10 +3473,9 @@ name = "rustc_baked_icu_data" version = "0.0.0" dependencies = [ "icu_list", - "icu_locid", - "icu_locid_transform", - "icu_provider 1.5.0", - "zerovec 0.10.4", + "icu_locale", + "icu_provider", + "zerovec", ] [[package]] @@ -3553,7 +3511,6 @@ version = "0.0.0" dependencies = [ "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -3585,11 +3542,10 @@ dependencies = [ "itertools", "libc", "measureme", - "object 0.37.1", + "object 0.37.3", "rustc-demangle", "rustc_abi", "rustc_ast", - "rustc_attr_data_structures", "rustc_codegen_ssa", "rustc_data_structures", "rustc_errors", @@ -3621,16 +3577,15 @@ dependencies = [ "ar_archive_writer", "bitflags", "bstr", - "cc", + "find-msvc-tools", "itertools", "libc", - "object 0.37.1", + "object 0.37.3", "pathdiff", "regex", "rustc_abi", "rustc_arena", "rustc_ast", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -3640,6 +3595,7 @@ dependencies = [ "rustc_hir", "rustc_incremental", "rustc_index", + "rustc_lint_defs", "rustc_macros", "rustc_metadata", "rustc_middle", @@ -3668,7 +3624,6 @@ dependencies = [ "rustc_abi", "rustc_apfloat", "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -3724,6 +3679,7 @@ name = "rustc_driver" version = "0.0.0" dependencies = [ "rustc_driver_impl", + "rustc_windows_rc", ] [[package]] @@ -3793,9 +3749,10 @@ dependencies = [ "fluent-bundle", "fluent-syntax", "icu_list", - "icu_locid", - "icu_provider_adapters", + "icu_locale", "intl-memoizer", + "rustc_ast", + "rustc_ast_pretty", "rustc_baked_icu_data", "rustc_data_structures", "rustc_macros", @@ -3809,29 +3766,25 @@ dependencies = [ name = "rustc_errors" version = "0.0.0" dependencies = [ - "annotate-snippets 0.11.5", + "annotate-snippets 0.12.7", + "anstream", + "anstyle", "derive_setters", "rustc_abi", "rustc_ast", - "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_error_codes", "rustc_error_messages", "rustc_fluent_macro", "rustc_hashes", - "rustc_hir", "rustc_index", "rustc_lexer", "rustc_lint_defs", "rustc_macros", "rustc_serialize", "rustc_span", - "rustc_target", - "rustc_type_ir", "serde", "serde_json", - "termcolor", "termize", "tracing", "windows 0.61.3", @@ -3844,7 +3797,6 @@ dependencies = [ "rustc_ast", "rustc_ast_passes", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -3868,8 +3820,8 @@ dependencies = [ name = "rustc_feature" version = "0.0.0" dependencies = [ - "rustc_attr_data_structures", "rustc_data_structures", + "rustc_hir", "rustc_span", "serde", "serde_json", @@ -3884,7 +3836,7 @@ dependencies = [ "fluent-syntax", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "unic-langid", ] @@ -3910,13 +3862,16 @@ dependencies = [ name = "rustc_hir" version = "0.0.0" dependencies = [ + "bitflags", "odht", "rustc_abi", "rustc_arena", "rustc_ast", - "rustc_attr_data_structures", + "rustc_ast_pretty", "rustc_data_structures", + "rustc_error_messages", "rustc_hashes", + "rustc_hir_id", "rustc_index", "rustc_macros", "rustc_serialize", @@ -3935,7 +3890,6 @@ dependencies = [ "rustc_abi", "rustc_arena", "rustc_ast", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -3955,6 +3909,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "rustc_hir_id" +version = "0.0.0" +dependencies = [ + "rustc_data_structures", + "rustc_index", + "rustc_macros", + "rustc_serialize", + "rustc_span", +] + [[package]] name = "rustc_hir_pretty" version = "0.0.0" @@ -3962,7 +3927,6 @@ dependencies = [ "rustc_abi", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_hir", "rustc_span", ] @@ -3974,7 +3938,6 @@ dependencies = [ "itertools", "rustc_abi", "rustc_ast", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -4033,7 +3996,7 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4117,10 +4080,10 @@ dependencies = [ name = "rustc_lint" version = "0.0.0" dependencies = [ + "bitflags", "rustc_abi", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -4145,11 +4108,10 @@ dependencies = [ name = "rustc_lint_defs" version = "0.0.0" dependencies = [ - "rustc_abi", "rustc_ast", "rustc_data_structures", "rustc_error_messages", - "rustc_hir", + "rustc_hir_id", "rustc_macros", "rustc_serialize", "rustc_span", @@ -4180,7 +4142,7 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -4194,7 +4156,6 @@ dependencies = [ "odht", "rustc_abi", "rustc_ast", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -4230,7 +4191,6 @@ dependencies = [ "rustc_arena", "rustc_ast", "rustc_ast_ir", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_error_messages", "rustc_errors", @@ -4264,7 +4224,6 @@ dependencies = [ "rustc_apfloat", "rustc_arena", "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -4293,7 +4252,6 @@ dependencies = [ "rustc_errors", "rustc_fluent_macro", "rustc_graphviz", - "rustc_hir", "rustc_index", "rustc_macros", "rustc_middle", @@ -4307,11 +4265,11 @@ name = "rustc_mir_transform" version = "0.0.0" dependencies = [ "either", + "hashbrown", "itertools", "rustc_abi", "rustc_arena", "rustc_ast", - "rustc_attr_data_structures", "rustc_const_eval", "rustc_data_structures", "rustc_errors", @@ -4336,8 +4294,6 @@ name = "rustc_monomorphize" version = "0.0.0" dependencies = [ "rustc_abi", - "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -4346,7 +4302,6 @@ dependencies = [ "rustc_middle", "rustc_session", "rustc_span", - "rustc_symbol_mangling", "rustc_target", "serde", "serde_json", @@ -4374,7 +4329,6 @@ dependencies = [ "rustc-literal-escaper", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", "rustc_feature", @@ -4384,11 +4338,10 @@ dependencies = [ "rustc_macros", "rustc_session", "rustc_span", - "termcolor", "thin-vec", "tracing", "unicode-normalization", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -4408,7 +4361,6 @@ dependencies = [ "rustc_ast", "rustc_ast_lowering", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -4455,7 +4407,6 @@ name = "rustc_privacy" version = "0.0.0" dependencies = [ "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -4517,7 +4468,6 @@ dependencies = [ "rustc_middle", "rustc_query_system", "rustc_serialize", - "rustc_session", "rustc_span", "tracing", ] @@ -4530,7 +4480,6 @@ dependencies = [ "parking_lot", "rustc_abi", "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_feature", @@ -4552,12 +4501,12 @@ name = "rustc_resolve" version = "0.0.0" dependencies = [ "bitflags", + "indexmap", "itertools", "pulldown-cmark", "rustc_arena", "rustc_ast", "rustc_ast_pretty", - "rustc_attr_data_structures", "rustc_attr_parsing", "rustc_data_structures", "rustc_errors", @@ -4651,7 +4600,7 @@ dependencies = [ "sha1", "sha2", "tracing", - "unicode-width 0.2.1", + "unicode-width 0.2.2", ] [[package]] @@ -4676,13 +4625,15 @@ name = "rustc_target" version = "0.0.0" dependencies = [ "bitflags", - "object 0.37.1", + "object 0.37.3", "rustc_abi", "rustc_data_structures", + "rustc_error_messages", "rustc_fs_util", "rustc_macros", "rustc_serialize", "rustc_span", + "schemars", "serde", "serde_derive", "serde_json", @@ -4720,7 +4671,6 @@ dependencies = [ "itertools", "rustc_abi", "rustc_ast", - "rustc_attr_data_structures", "rustc_data_structures", "rustc_errors", "rustc_fluent_macro", @@ -4743,7 +4693,6 @@ name = "rustc_traits" version = "0.0.0" dependencies = [ "rustc_data_structures", - "rustc_hir", "rustc_infer", "rustc_middle", "rustc_span", @@ -4791,6 +4740,7 @@ dependencies = [ name = "rustc_type_ir" version = "0.0.0" dependencies = [ + "arrayvec", "bitflags", "derive-where", "ena", @@ -4798,6 +4748,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustc_ast_ir", "rustc_data_structures", + "rustc_error_messages", "rustc_index", "rustc_macros", "rustc_serialize", @@ -4814,7 +4765,7 @@ version = "0.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -4827,6 +4778,13 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_windows_rc" +version = "0.0.0" +dependencies = [ + "find-msvc-tools", +] + [[package]] name = "rustdoc" version = "0.0.0" @@ -4845,6 +4803,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", + "stringdex", "tempfile", "threadpool", "tracing", @@ -4858,7 +4817,6 @@ name = "rustdoc-gui-test" version = "0.1.0" dependencies = [ "build_helper", - "camino", "compiletest", "getopts", "walkdir", @@ -4888,9 +4846,9 @@ dependencies = [ [[package]] name = "rustfix" -version = "0.8.1" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81864b097046da5df3758fdc6e4822bbb70afa06317e8ca45ea1b51cb8c5e5a4" +checksum = "82fa69b198d894d84e23afde8e9ab2af4400b2cba20d6bf2b428a8b01c222c5a" dependencies = [ "serde", "serde_json", @@ -4905,7 +4863,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4939,22 +4897,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ruzstd" @@ -4998,6 +4956,31 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "schemars" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d020396d1d138dc19f1165df7545479dcd58d93810dc5d646a16e55abefa80" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.106", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -5012,9 +4995,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scratch" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" [[package]] name = "self_cell" @@ -5033,18 +5016,19 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde-untagged" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "299d9c19d7d466db4ab10addd5703e4c615dec2a5a16dbbafe191045e87ee66e" +checksum = "34836a629bcbc6f1afdf0907a744870039b1e14c0561cb26094fa683b158eff3" dependencies = [ "erased-serde", "serde", @@ -5062,21 +5046,41 @@ dependencies = [ ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" dependencies = [ "itoa", "memchr", @@ -5103,6 +5107,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5417783452c2be558477e104686f7de5dae53dba813c28435e0e70f82d9b04ee" +dependencies = [ + "serde_core", +] + [[package]] name = "sha1" version = "0.10.6" @@ -5160,12 +5173,12 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.5.10" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5258,6 +5271,15 @@ dependencies = [ "quote", ] +[[package]] +name = "stringdex" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b3bd4f10d15ef859c40291769f0d85209de6b0f1c30713ff9cdf45ac43ea36" +dependencies = [ + "stacker", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5296,9 +5318,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -5313,14 +5335,14 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "sysinfo" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d" +checksum = "07cec4dc2d2e357ca1e610cfb07de2fa7a10fc3e9fe89f72545f3d244ea87753" dependencies = [ "libc", "objc2-core-foundation", @@ -5431,11 +5453,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.15", ] [[package]] @@ -5446,18 +5468,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5523,16 +5545,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec 0.10.4", -] - [[package]] name = "tinystr" version = "0.8.1" @@ -5540,7 +5552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", - "zerovec 0.11.2", + "zerovec", ] [[package]] @@ -5558,15 +5570,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - [[package]] name = "toml" version = "0.7.8" @@ -5574,8 +5577,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit 0.19.15", ] @@ -5587,11 +5590,26 @@ checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "indexmap", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit 0.22.27", ] +[[package]] +name = "toml" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00e5e5d9bf2475ac9d4f0d9edab68cc573dc2fd644b0dba36b0c30a92dd9eaa0" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned 1.0.2", + "toml_datetime 0.7.2", + "toml_parser", + "toml_writer", + "winnow 0.7.13", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -5601,6 +5619,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.19.15" @@ -5609,8 +5636,8 @@ checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "winnow 0.5.40", ] @@ -5622,10 +5649,19 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", - "winnow 0.7.12", + "winnow 0.7.13", +] + +[[package]] +name = "toml_parser" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" +dependencies = [ + "winnow 0.7.13", ] [[package]] @@ -5635,12 +5671,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] -name = "tracing" -version = "0.1.37" +name = "toml_writer" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5654,14 +5695,14 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -5814,7 +5855,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" dependencies = [ - "tinystr 0.8.1", + "tinystr", ] [[package]] @@ -5824,7 +5865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5957eb82e346d7add14182a3315a7e298f04e1ba4baac36f7f0dbfedba5fc25" dependencies = [ "proc-macro-hack", - "tinystr 0.8.1", + "tinystr", "unic-langid-impl", "unic-langid-macros-impl", ] @@ -5837,7 +5878,7 @@ checksum = "a1249a628de3ad34b821ecb1001355bca3940bcb2f88558f1a8bd82e977f75b5" dependencies = [ "proc-macro-hack", "quote", - "syn 2.0.104", + "syn 2.0.106", "unic-langid-impl", ] @@ -5905,9 +5946,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode-xid" @@ -5943,6 +5984,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" @@ -5969,9 +6016,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom 0.3.3", "js-sys", @@ -6023,9 +6070,9 @@ dependencies = [ [[package]] name = "wasi-preview1-component-adapter-provider" -version = "34.0.2" +version = "37.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33696c5f1ff1e083de9f36c3da471abd736362bc173e093f8b0b1ed5a387e39b" +checksum = "8d0fcd636ad2b29a7c0490799a23ad61d1c8dedfafdb970447fddd0549502b60" [[package]] name = "wasm-bindgen" @@ -6049,7 +6096,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -6071,7 +6118,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6087,9 +6134,9 @@ dependencies = [ [[package]] name = "wasm-component-ld" -version = "0.5.15" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d95124e34fee1316222e03b9bbf41af186ecbae2c8b79f8debe6e21b3ff60c5" +checksum = "11f565dfcfd9aabb10d865b608a92ce1f93051aeb56f4c89550ed9cd97d8ce0e" dependencies = [ "anyhow", "clap", @@ -6097,9 +6144,9 @@ dependencies = [ "libc", "tempfile", "wasi-preview1-component-adapter-provider", - "wasmparser 0.234.0", + "wasmparser 0.240.0", "wat", - "windows-sys 0.59.0", + "windows-sys 0.61.2", "winsplit", "wit-component", "wit-parser", @@ -6124,34 +6171,24 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.234.0" +version = "0.240.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170a0157eef517a179f2d20ed7c68df9c3f7f6c1c047782d488bf5a464174684" +checksum = "06d642d8c5ecc083aafe9ceb32809276a304547a3a6eeecceb5d8152598bc71f" dependencies = [ "leb128fmt", - "wasmparser 0.234.0", -] - -[[package]] -name = "wasm-encoder" -version = "0.235.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3bc393c395cb621367ff02d854179882b9a351b4e0c93d1397e6090b53a5c2a" -dependencies = [ - "leb128fmt", - "wasmparser 0.235.0", + "wasmparser 0.240.0", ] [[package]] name = "wasm-metadata" -version = "0.234.0" +version = "0.240.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42fe3f5cbfb56fc65311ef827930d06189160038e81db62188f66b4bf468e3a" +checksum = "ee093e1e1ccffa005b9b778f7a10ccfd58e25a20eccad294a1a93168d076befb" dependencies = [ "anyhow", "indexmap", - "wasm-encoder 0.234.0", - "wasmparser 0.234.0", + "wasm-encoder 0.240.0", + "wasmparser 0.240.0", ] [[package]] @@ -6166,9 +6203,19 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.234.0" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be22e5a8f600afce671dd53c8d2dd26b4b7aa810fd18ae27dfc49737f3e02fc5" +checksum = "a9b1e81f3eb254cf7404a82cee6926a4a3ccc5aad80cc3d43608a070c67aa1d7" +dependencies = [ + "bitflags", + "indexmap", +] + +[[package]] +name = "wasmparser" +version = "0.240.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b722dcf61e0ea47440b53ff83ccb5df8efec57a69d150e4f24882e4eba7e24a4" dependencies = [ "bitflags", "hashbrown", @@ -6177,35 +6224,24 @@ dependencies = [ "serde", ] -[[package]] -name = "wasmparser" -version = "0.235.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161296c618fa2d63f6ed5fffd1112937e803cb9ec71b32b01a76321555660917" -dependencies = [ - "bitflags", - "indexmap", - "semver", -] - [[package]] name = "wast" -version = "235.0.0" +version = "240.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eda4293f626c99021bb3a6fbe4fbbe90c0e31a5ace89b5f620af8925de72e13" +checksum = "b0efe1c93db4ac562b9733e3dca19ed7fc878dba29aef22245acf84f13da4a19" dependencies = [ "bumpalo", "leb128fmt", "memchr", - "unicode-width 0.2.1", - "wasm-encoder 0.235.0", + "unicode-width 0.2.2", + "wasm-encoder 0.240.0", ] [[package]] name = "wat" -version = "1.235.0" +version = "1.240.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e777e0327115793cb96ab220b98f85327ec3d11f34ec9e8d723264522ef206aa" +checksum = "4ec9b6eab7ecd4d639d78515e9ea491c9bacf494aa5eda10823bd35992cf8c1e" dependencies = [ "wast", ] @@ -6270,7 +6306,7 @@ dependencies = [ "windows-collections", "windows-core 0.61.2", "windows-future", - "windows-link", + "windows-link 0.1.3", "windows-numerics", ] @@ -6315,7 +6351,7 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement 0.60.0", "windows-interface 0.59.1", - "windows-link", + "windows-link 0.1.3", "windows-result 0.3.4", "windows-strings 0.4.2", ] @@ -6327,7 +6363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", - "windows-link", + "windows-link 0.1.3", "windows-threading", ] @@ -6339,7 +6375,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6350,7 +6386,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6361,7 +6397,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6372,7 +6408,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6381,6 +6417,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-numerics" version = "0.2.0" @@ -6388,7 +6430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ "windows-core 0.61.2", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6406,7 +6448,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6425,16 +6467,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", + "windows-link 0.1.3", ] [[package]] @@ -6461,22 +6494,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.2", + "windows-targets 0.53.3", ] [[package]] -name = "windows-targets" -version = "0.48.5" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-link 0.2.1", ] [[package]] @@ -6497,10 +6524,11 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.2" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -6517,15 +6545,9 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -6538,12 +6560,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -6556,12 +6572,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -6586,12 +6596,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -6604,12 +6608,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -6622,12 +6620,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -6640,12 +6632,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -6669,9 +6655,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -6693,9 +6679,9 @@ dependencies = [ [[package]] name = "wit-component" -version = "0.234.0" +version = "0.240.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a8888169acf4c6c4db535beb405b570eedac13215d6821ca9bd03190f7f8b8c" +checksum = "7dc5474b078addc5fe8a72736de8da3acfb3ff324c2491133f8b59594afa1a20" dependencies = [ "anyhow", "bitflags", @@ -6704,17 +6690,17 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "wasm-encoder 0.234.0", + "wasm-encoder 0.240.0", "wasm-metadata", - "wasmparser 0.234.0", + "wasmparser 0.240.0", "wit-parser", ] [[package]] name = "wit-parser" -version = "0.234.0" +version = "0.240.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465492df47d8dcc015a3b7f241aed8ea03688fee7c5e04162285c5b1a3539c8b" +checksum = "9875ea3fa272f57cc1fc50f225a7b94021a7878c484b33792bccad0d93223439" dependencies = [ "anyhow", "id-arena", @@ -6725,15 +6711,9 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.234.0", + "wasmparser 0.240.0", ] -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - [[package]] name = "writeable" version = "0.6.1" @@ -6772,18 +6752,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive 0.7.5", - "zerofrom", -] - [[package]] name = "yoke" version = "0.8.0" @@ -6792,22 +6760,10 @@ checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", - "yoke-derive 0.8.0", + "yoke-derive", "zerofrom", ] -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", - "synstructure", -] - [[package]] name = "yoke-derive" version = "0.8.0" @@ -6816,7 +6772,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -6837,7 +6793,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6857,7 +6813,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure", ] @@ -6868,41 +6824,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" dependencies = [ "displaydoc", - "yoke 0.8.0", + "yoke", "zerofrom", ] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ - "yoke 0.7.5", + "yoke", "zerofrom", - "zerovec-derive 0.10.3", -] - -[[package]] -name = "zerovec" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" -dependencies = [ - "yoke 0.8.0", - "zerofrom", - "zerovec-derive 0.11.1", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", + "zerovec-derive", ] [[package]] @@ -6913,5 +6847,5 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] diff --git a/RELEASES.md b/RELEASES.md index 1ae221774dc9..74b0d4424c16 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,477 @@ +Version 1.91.0 (2025-10-30) +========================== + + + +Language +-------- + +- [Lower pattern bindings in the order they're written and base drop order on primary bindings' order](https://github.com/rust-lang/rust/pull/143764) +- [Stabilize declaration of C-style variadic functions for `sysv64`, `win64`, `efiapi`, and `aapcs` ABIs](https://github.com/rust-lang/rust/pull/144066). + This brings these ABIs in line with the C ABI: variadic functions can be declared in extern blocks but not defined. +- [Add `dangling_pointers_from_locals` lint to warn against dangling pointers from local variables](https://github.com/rust-lang/rust/pull/144322) +- [Upgrade `semicolon_in_expressions_from_macros` from warn to deny](https://github.com/rust-lang/rust/pull/144369) +- [Stabilize LoongArch32 inline assembly](https://github.com/rust-lang/rust/pull/144402) +- [Add warn-by-default `integer_to_ptr_transmutes` lint against integer-to-pointer transmutes](https://github.com/rust-lang/rust/pull/144531) +- [Stabilize `sse4a` and `tbm` target features](https://github.com/rust-lang/rust/pull/144542) +- [Add `target_env = "macabi"` and `target_env = "sim"` cfgs](https://github.com/rust-lang/rust/pull/139451) as replacements for the `target_abi` cfgs with the same values. + + + +Compiler +-------- + +- [Don't warn on never-to-any `as` casts as unreachable](https://github.com/rust-lang/rust/pull/144804) + + + +Platform Support +---------------- + +- [Promote `aarch64-pc-windows-gnullvm` and `x86_64-pc-windows-gnullvm` to Tier 2 with host tools.](https://github.com/rust-lang/rust/pull/143031) + Note: llvm-tools and MSI installers are missing but will be added in future releases. +- [Promote `aarch64-pc-windows-msvc` to Tier 1](https://github.com/rust-lang/rust/pull/145682) + +Refer to Rust's [platform support page][platform-support-doc] +for more information on Rust's tiered platform support. + +[platform-support-doc]: https://doc.rust-lang.org/rustc/platform-support.html + + + +Libraries +--------- + +- [Print thread ID in panic message](https://github.com/rust-lang/rust/pull/115746) +- [Fix overly restrictive lifetime in `core::panic::Location::file` return type](https://github.com/rust-lang/rust/pull/132087) +- [Guarantee parameter order for `_by()` variants of `min` / `max`/ `minmax` in `std::cmp`](https://github.com/rust-lang/rust/pull/139357) +- [Document assumptions about `Clone` and `Eq` traits](https://github.com/rust-lang/rust/pull/144330/) +- [`std::thread`: Return error if setting thread stack size fails](https://github.com/rust-lang/rust/pull/144210) + This used to panic within the standard library. + + + +Stabilized APIs +--------------- + +- [`Path::file_prefix`](https://doc.rust-lang.org/stable/std/path/struct.Path.html#method.file_prefix) +- [`AtomicPtr::fetch_ptr_add`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_ptr_add) +- [`AtomicPtr::fetch_ptr_sub`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_ptr_sub) +- [`AtomicPtr::fetch_byte_add`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_byte_add) +- [`AtomicPtr::fetch_byte_sub`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_byte_sub) +- [`AtomicPtr::fetch_or`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_or) +- [`AtomicPtr::fetch_and`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_and) +- [`AtomicPtr::fetch_xor`](https://doc.rust-lang.org/stable/std/sync/atomic/struct.AtomicPtr.html#method.fetch_xor) +- [`{integer}::strict_add`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_add) +- [`{integer}::strict_sub`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_sub) +- [`{integer}::strict_mul`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_mul) +- [`{integer}::strict_div`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_div) +- [`{integer}::strict_div_euclid`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_div_euclid) +- [`{integer}::strict_rem`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_rem) +- [`{integer}::strict_rem_euclid`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_rem_euclid) +- [`{integer}::strict_neg`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_neg) +- [`{integer}::strict_shl`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_shl) +- [`{integer}::strict_shr`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_shr) +- [`{integer}::strict_pow`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_pow) +- [`i{N}::strict_add_unsigned`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_add_unsigned) +- [`i{N}::strict_sub_unsigned`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_sub_unsigned) +- [`i{N}::strict_abs`](https://doc.rust-lang.org/stable/std/primitive.i32.html#method.strict_abs) +- [`u{N}::strict_add_signed`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_add_signed) +- [`u{N}::strict_sub_signed`](https://doc.rust-lang.org/stable/std/primitive.u32.html#method.strict_sub_signed) +- [`PanicHookInfo::payload_as_str`](https://doc.rust-lang.org/stable/std/panic/struct.PanicHookInfo.html#method.payload_as_str) +- [`core::iter::chain`](https://doc.rust-lang.org/stable/core/iter/fn.chain.html) +- [`u{N}::checked_signed_diff`](https://doc.rust-lang.org/stable/std/primitive.u16.html#method.checked_signed_diff) +- [`core::array::repeat`](https://doc.rust-lang.org/stable/core/array/fn.repeat.html) +- [`PathBuf::add_extension`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#method.add_extension) +- [`PathBuf::with_added_extension`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#method.with_added_extension) +- [`Duration::from_mins`](https://doc.rust-lang.org/stable/std/time/struct.Duration.html#method.from_mins) +- [`Duration::from_hours`](https://doc.rust-lang.org/stable/std/time/struct.Duration.html#method.from_hours) +- [`impl PartialEq for PathBuf`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#impl-PartialEq%3Cstr%3E-for-PathBuf) +- [`impl PartialEq for PathBuf`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#impl-PartialEq%3CString%3E-for-PathBuf) +- [`impl PartialEq for Path`](https://doc.rust-lang.org/stable/std/path/struct.Path.html#impl-PartialEq%3Cstr%3E-for-Path) +- [`impl PartialEq for Path`](https://doc.rust-lang.org/stable/std/path/struct.Path.html#impl-PartialEq%3CString%3E-for-Path) +- [`impl PartialEq for String`](https://doc.rust-lang.org/stable/std/string/struct.String.html#impl-PartialEq%3CPathBuf%3E-for-String) +- [`impl PartialEq for String`](https://doc.rust-lang.org/stable/std/string/struct.String.html#impl-PartialEq%3CPath%3E-for-String) +- [`impl PartialEq for str`](https://doc.rust-lang.org/stable/std/primitive.str.html#impl-PartialEq%3CPathBuf%3E-for-str) +- [`impl PartialEq for str`](https://doc.rust-lang.org/stable/std/primitive.str.html#impl-PartialEq%3CPath%3E-for-str) +- [`Ipv4Addr::from_octets`](https://doc.rust-lang.org/stable/std/net/struct.Ipv4Addr.html#method.from_octets) +- [`Ipv6Addr::from_octets`](https://doc.rust-lang.org/stable/std/net/struct.Ipv6Addr.html#method.from_octets) +- [`Ipv6Addr::from_segments`](https://doc.rust-lang.org/stable/std/net/struct.Ipv6Addr.html#method.from_segments) +- [`impl Default for Pin> where Box: Default, T: ?Sized`](https://doc.rust-lang.org/stable/std/default/trait.Default.html#impl-Default-for-Pin%3CBox%3CT%3E%3E) +- [`impl Default for Pin> where Rc: Default, T: ?Sized`](https://doc.rust-lang.org/stable/std/default/trait.Default.html#impl-Default-for-Pin%3CRc%3CT%3E%3E) +- [`impl Default for Pin> where Arc: Default, T: ?Sized`](https://doc.rust-lang.org/stable/std/default/trait.Default.html#impl-Default-for-Pin%3CArc%3CT%3E%3E) +- [`Cell::as_array_of_cells`](https://doc.rust-lang.org/stable/std/cell/struct.Cell.html#method.as_array_of_cells) +- [`u{N}::carrying_add`](https://doc.rust-lang.org/stable/std/primitive.u64.html#method.carrying_add) +- [`u{N}::borrowing_sub`](https://doc.rust-lang.org/stable/std/primitive.u64.html#method.borrowing_sub) +- [`u{N}::carrying_mul`](https://doc.rust-lang.org/stable/std/primitive.u64.html#method.carrying_mul) +- [`u{N}::carrying_mul_add`](https://doc.rust-lang.org/stable/std/primitive.u64.html#method.carrying_mul_add) +- [`BTreeMap::extract_if`](https://doc.rust-lang.org/stable/std/collections/struct.BTreeMap.html#method.extract_if) +- [`BTreeSet::extract_if`](https://doc.rust-lang.org/stable/std/collections/struct.BTreeSet.html#method.extract_if) +- [`impl Debug for windows::ffi::EncodeWide<'_>`](https://doc.rust-lang.org/stable/std/os/windows/ffi/struct.EncodeWide.html#impl-Debug-for-EncodeWide%3C'_%3E) +- [`str::ceil_char_boundary`](https://doc.rust-lang.org/stable/std/primitive.str.html#method.ceil_char_boundary) +- [`str::floor_char_boundary`](https://doc.rust-lang.org/stable/std/primitive.str.html#method.floor_char_boundary) +- [`impl Sum for Saturating`](https://doc.rust-lang.org/stable/std/num/struct.Saturating.html#impl-Sum-for-Saturating%3Cu32%3E) +- [`impl Sum<&Self> for Saturating`](https://doc.rust-lang.org/stable/std/num/struct.Saturating.html#impl-Sum%3C%26Saturating%3Cu32%3E%3E-for-Saturating%3Cu32%3E) +- [`impl Product for Saturating`](https://doc.rust-lang.org/stable/std/num/struct.Saturating.html#impl-Product-for-Saturating%3Cu32%3E) +- [`impl Product<&Self> for Saturating`](https://doc.rust-lang.org/stable/std/num/struct.Saturating.html#impl-Product%3C%26Saturating%3Cu32%3E%3E-for-Saturating%3Cu32%3E) + +These previously stable APIs are now stable in const contexts: + +- [`<[T; N]>::each_ref`](https://doc.rust-lang.org/stable/std/primitive.array.html#method.each_ref) +- [`<[T; N]>::each_mut`](https://doc.rust-lang.org/stable/std/primitive.array.html#method.each_mut) +- [`OsString::new`](https://doc.rust-lang.org/stable/std/ffi/struct.OsString.html#method.new) +- [`PathBuf::new`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#method.new) +- [`TypeId::of`](https://doc.rust-lang.org/stable/std/any/struct.TypeId.html#method.of) +- [`ptr::with_exposed_provenance`](https://doc.rust-lang.org/stable/std/ptr/fn.with_exposed_provenance.html) +- [`ptr::with_exposed_provenance_mut`](https://doc.rust-lang.org/stable/std/ptr/fn.with_exposed_provenance_mut.html) + + + +Cargo +----- + +- 🎉 Stabilize `build.build-dir`. + This config sets the directory where intermediate build artifacts are stored. + These artifacts are produced by Cargo and rustc during the build process. + End users usually won't need to interact with them, and the layout inside + `build-dir` is an implementation detail that may change without notice. + ([config doc](https://doc.rust-lang.org/stable/cargo/reference/config.html#buildbuild-dir)) + ([build cache doc](https://doc.rust-lang.org/stable/cargo/reference/build-cache.html)) + [#15833](https://github.com/rust-lang/cargo/pull/15833) + [#15840](https://github.com/rust-lang/cargo/pull/15840) +- The `--target` flag and the `build.target` configuration can now take literal + `"host-tuple"` string, which will internally be substituted by the host + machine's target triple. + [#15838](https://github.com/rust-lang/cargo/pull/15838) + [#16003](https://github.com/rust-lang/cargo/pull/16003) + [#16032](https://github.com/rust-lang/cargo/pull/16032) + + + +Rustdoc +----- +- [In search results, rank doc aliases lower than non-alias items with the same name](https://github.com/rust-lang/rust/pull/145100) +- [Raw pointers now work in type-based search like references](https://github.com/rust-lang/rust/pull/145731). This means you can now search for things like `*const u8 ->`, and additionally functions that take or return raw pointers will now display their signature properly in search results. + + + +Compatibility Notes +------------------- + +- [Always require coroutine captures to be drop-live](https://github.com/rust-lang/rust/pull/144156) +- [Apple: Always pass SDK root when linking with `cc`, and pass it via `SDKROOT` env var](https://github.com/rust-lang/rust/pull/131477). This should fix linking issues with `rustc` running inside Xcode. Libraries in `/usr/local/lib` may no longer be linked automatically, if you develop or use a crate that relies on this, you should explicitly set `cargo::rustc-link-search=/usr/local/lib` in a `build.rs` script. +- [Relaxed bounds in associated type bound position like in `TraitRef` are now correctly forbidden](https://github.com/rust-lang/rust/pull/135331) +- [Add unstable `#[sanitize(xyz = "on|off")]` built-in attribute that shadows procedural macros with the same name](https://github.com/rust-lang/rust/pull/142681) +- [Fix the drop checker being more permissive for bindings declared with let-else](https://github.com/rust-lang/rust/pull/143028) +- [Be more strict when parsing attributes, erroring on many invalid attributes](https://github.com/rust-lang/rust/pull/144689) + - [Error on invalid `#[should_panic]` attributes](https://github.com/rust-lang/rust/pull/143808) + - [Error on invalid `#[link]` attributes](https://github.com/rust-lang/rust/pull/143193) +- [Mark all deprecation lints in name resolution as deny-by-default and also report in dependencies](https://github.com/rust-lang/rust/pull/143929) +- The lint `semicolon_in_expressions_from_macros`, for `macro_rules!` macros in expression position that expand to end in a semicolon (`;`), is now deny-by-default. It was already warn-by-default, and a future compatibility warning (FCW) that warned even in dependencies. This lint will become a hard error in the future. +- [Trait impl modifiers (e.g., `unsafe`, `!`, `default`) in inherent impls are no longer syntactically valid](https://github.com/rust-lang/rust/pull/144386) +- [Start reporting future breakage for `ill_formed_attribute_input` in dependencies](https://github.com/rust-lang/rust/pull/144544) +- [Restrict the scope of temporaries created by the macros `pin!`, `format_args!`, `write!`, and `writeln!` in `if let` scrutinees in Rust Edition 2024.](https://github.com/rust-lang/rust/pull/145342) This applies [Rust Edition 2024's `if let` temporary scope rules](https://doc.rust-lang.org/edition-guide/rust-2024/temporary-if-let-scope.html) to these temporaries, which previously could live past the `if` expression regardless of Edition. +- [Invalid numeric literal suffixes in tuple indexing, tuple struct indexing, and struct field name positions are now correctly rejected](https://github.com/rust-lang/rust/pull/145463) +- [Closures marked with the keyword `static` are now syntactically invalid](https://github.com/rust-lang/rust/pull/145604) +- [Shebangs inside `--cfg` and `--check-cfg` arguments are no longer allowed](https://github.com/rust-lang/rust/pull/146211) +- [Add future incompatibility lint for temporary lifetime shortening in Rust 1.92](https://github.com/rust-lang/rust/pull/147056) + +Cargo compatibility notes: + +- `cargo publish` no longer keeps `.crate` tarballs as final build artifacts + when `build.build-dir` is set. These tarballs were previously included due to + an oversight and are now treated as intermediate artifacts. + To get `.crate` tarballs as final artifacts, use `cargo package`. + In a future version, this change will apply regardless of `build.build-dir`. + [#15910](https://github.com/rust-lang/cargo/pull/15910) +- Adjust Cargo messages to match rustc diagnostic style. + This changes some of the terminal colors used by Cargo messages. + [#15928](https://github.com/rust-lang/cargo/pull/15928) +- Tools and projects relying on the + [internal details of Cargo's `build-dir`](https://doc.rust-lang.org/cargo/reference/build-cache.html) + may not work for users changing their `build-dir` layout. + For those doing so, we'd recommend proactively testing these cases + particularly as we are considering changing the default location of the `build-dir` in the future + ([cargo#16147](https://github.com/rust-lang/cargo/issues/16147)). + If you can't migrate off of Cargo's internal details, + we'd like to learn more about your use case as we prepare to change the layout of the `build-dir` + ([cargo#15010](https://github.com/rust-lang/cargo/issues/15010)). + + + +Internal Changes +---------------- + +These changes do not affect any public interfaces of Rust, but they represent +significant improvements to the performance or internals of rustc and related +tools. + +- [Update to LLVM 21](https://github.com/rust-lang/rust/pull/143684) + + +Version 1.90.0 (2025-09-18) +=========================== + + + +Language +-------- +- [Split up the `unknown_or_malformed_diagnostic_attributes` lint](https://github.com/rust-lang/rust/pull/140717). This lint has been split up into four finer-grained lints, with `unknown_or_malformed_diagnostic_attributes` now being the lint group that contains these lints: + 1. `unknown_diagnostic_attributes`: unknown to the current compiler + 2. `misplaced_diagnostic_attributes`: placed on the wrong item + 3. `malformed_diagnostic_attributes`: malformed attribute syntax or options + 4. `malformed_diagnostic_format_literals`: malformed format string literal +- [Allow constants whose final value has references to mutable/external memory, but reject such constants as patterns](https://github.com/rust-lang/rust/pull/140942) +- [Allow volatile access to non-Rust memory, including address 0](https://github.com/rust-lang/rust/pull/141260) + + + + +Compiler +-------- +- [Use `lld` by default on `x86_64-unknown-linux-gnu`](https://github.com/rust-lang/rust/pull/140525). +- [Tier 3 `musl` targets now link dynamically by default](https://github.com/rust-lang/rust/pull/144410). Affected targets: + - `mips64-unknown-linux-muslabi64` + - `powerpc64-unknown-linux-musl` + - `powerpc-unknown-linux-musl` + - `powerpc-unknown-linux-muslspe` + - `riscv32gc-unknown-linux-musl` + - `s390x-unknown-linux-musl` + - `thumbv7neon-unknown-linux-musleabihf` + + + + +Platform Support +---------------- +- [Demote `x86_64-apple-darwin` to Tier 2 with host tools](https://github.com/rust-lang/rust/pull/145252) + + +Refer to Rust's [platform support page][platform-support-doc] +for more information on Rust's tiered platform support. + +[platform-support-doc]: https://doc.rust-lang.org/rustc/platform-support.html + + + +Libraries +--------- +- [Stabilize `u*::{checked,overflowing,saturating,wrapping}_sub_signed`](https://github.com/rust-lang/rust/issues/126043) +- [Allow comparisons between `CStr`, `CString`, and `Cow`](https://github.com/rust-lang/rust/pull/137268) +- [Remove some unsized tuple impls since unsized tuples can't be constructed](https://github.com/rust-lang/rust/pull/138340) +- [Set `MSG_NOSIGNAL` for `UnixStream`](https://github.com/rust-lang/rust/pull/140005) +- [`proc_macro::Ident::new` now supports `$crate`.](https://github.com/rust-lang/rust/pull/141996) +- [Guarantee the pointer returned from `Thread::into_raw` has at least 8 bytes of alignment](https://github.com/rust-lang/rust/pull/143859) + + + + +Stabilized APIs +--------------- + +- [`u{n}::checked_sub_signed`](https://doc.rust-lang.org/stable/std/primitive.usize.html#method.checked_sub_signed) +- [`u{n}::overflowing_sub_signed`](https://doc.rust-lang.org/stable/std/primitive.usize.html#method.overflowing_sub_signed) +- [`u{n}::saturating_sub_signed`](https://doc.rust-lang.org/stable/std/primitive.usize.html#method.saturating_sub_signed) +- [`u{n}::wrapping_sub_signed`](https://doc.rust-lang.org/stable/std/primitive.usize.html#method.wrapping_sub_signed) +- [`impl Copy for IntErrorKind`](https://doc.rust-lang.org/stable/std/num/enum.IntErrorKind.html#impl-Copy-for-IntErrorKind) +- [`impl Hash for IntErrorKind`](https://doc.rust-lang.org/stable/std/num/enum.IntErrorKind.html#impl-Hash-for-IntErrorKind) +- [`impl PartialEq<&CStr> for CStr`](https://doc.rust-lang.org/stable/std/ffi/struct.CStr.html#impl-PartialEq%3C%26CStr%3E-for-CStr) +- [`impl PartialEq for CStr`](https://doc.rust-lang.org/stable/std/ffi/struct.CStr.html#impl-PartialEq%3CCString%3E-for-CStr) +- [`impl PartialEq> for CStr`](https://doc.rust-lang.org/stable/std/ffi/struct.CStr.html#impl-PartialEq%3CCow%3C'_,+CStr%3E%3E-for-CStr) +- [`impl PartialEq<&CStr> for CString`](https://doc.rust-lang.org/stable/std/ffi/struct.CString.html#impl-PartialEq%3C%26CStr%3E-for-CString) +- [`impl PartialEq for CString`](https://doc.rust-lang.org/stable/std/ffi/struct.CString.html#impl-PartialEq%3CCStr%3E-for-CString) +- [`impl PartialEq> for CString`](https://doc.rust-lang.org/stable/std/ffi/struct.CString.html#impl-PartialEq%3CCow%3C'_,+CStr%3E%3E-for-CString) +- [`impl PartialEq<&CStr> for Cow`](https://doc.rust-lang.org/stable/std/borrow/enum.Cow.html#impl-PartialEq%3C%26CStr%3E-for-Cow%3C'_,+CStr%3E) +- [`impl PartialEq for Cow`](https://doc.rust-lang.org/stable/std/borrow/enum.Cow.html#impl-PartialEq%3CCStr%3E-for-Cow%3C'_,+CStr%3E) +- [`impl PartialEq for Cow`](https://doc.rust-lang.org/stable/std/borrow/enum.Cow.html#impl-PartialEq%3CCString%3E-for-Cow%3C'_,+CStr%3E) + + +These previously stable APIs are now stable in const contexts: + +- [`<[T]>::reverse`](https://doc.rust-lang.org/stable/std/primitive.slice.html#method.reverse) +- [`f32::floor`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.floor) +- [`f32::ceil`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.ceil) +- [`f32::trunc`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.trunc) +- [`f32::fract`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.fract) +- [`f32::round`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.round) +- [`f32::round_ties_even`](https://doc.rust-lang.org/stable/std/primitive.f32.html#method.round_ties_even) +- [`f64::floor`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.floor) +- [`f64::ceil`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.ceil) +- [`f64::trunc`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.trunc) +- [`f64::fract`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.fract) +- [`f64::round`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.round) +- [`f64::round_ties_even`](https://doc.rust-lang.org/stable/std/primitive.f64.html#method.round_ties_even) + + + + +Cargo +----- +- [Add `http.proxy-cainfo` config for proxy certs](https://github.com/rust-lang/cargo/pull/15374/) +- [Use `gix` for `cargo package`](https://github.com/rust-lang/cargo/pull/15534/) +- [feat(publish): Stabilize multi-package publishing](https://github.com/rust-lang/cargo/pull/15636/) + + + +Rustdoc +----- +- [Add ways to collapse all impl blocks](https://github.com/rust-lang/rust/pull/141663). Previously the "Summary" button and "-" keyboard shortcut would never collapse `impl` blocks, now they do when shift is held +- [Display unsafe attributes with `unsafe()` wrappers](https://github.com/rust-lang/rust/pull/143662) + + + + +Compatibility Notes +------------------- +- [Use `lld` by default on `x86_64-unknown-linux-gnu`](https://github.com/rust-lang/rust/pull/140525). + See also . +- [Make `core::iter::Fuse`'s `Default` impl construct `I::default()` internally as promised in the docs instead of always being empty](https://github.com/rust-lang/rust/pull/140985) +- [Set `MSG_NOSIGNAL` for `UnixStream`](https://github.com/rust-lang/rust/pull/140005) + This may change program behavior but results in the same behavior as other primitives (e.g., stdout, network sockets). + Programs relying on signals to terminate them should update handling of sockets to handle errors on write by exiting. +- [On Unix `std::env::home_dir` will use the fallback if the `HOME` environment variable is empty](https://github.com/rust-lang/rust/pull/141840) +- We now [reject unsupported `extern "{abi}"`s consistently in all positions](https://github.com/rust-lang/rust/pull/142134). This primarily affects the use of implementing traits on an `extern "{abi}"` function pointer, like `extern "stdcall" fn()`, on a platform that doesn't support that, like aarch64-unknown-linux-gnu. Direct usage of these unsupported ABI strings by declaring or defining functions was already rejected, so this is only a change for consistency. +- [const-eval: error when initializing a static writes to that static](https://github.com/rust-lang/rust/pull/143084) +- [Check that the `proc_macro_derive` macro has correct arguments when applied to the crate root](https://github.com/rust-lang/rust/pull/143607) + + +Version 1.89.0 (2025-08-07) +========================== + + + +Language +-------- +- [Stabilize explicitly inferred const arguments (`feature(generic_arg_infer)`)](https://github.com/rust-lang/rust/pull/141610) +- [Add a warn-by-default `mismatched_lifetime_syntaxes` lint.](https://github.com/rust-lang/rust/pull/138677) + This lint detects when the same lifetime is referred to by different syntax categories between function arguments and return values, which can be confusing to read, especially in unsafe code. + This lint supersedes the warn-by-default `elided_named_lifetimes` lint. +- [Expand `unpredictable_function_pointer_comparisons` to also lint on function pointer comparisons in external macros](https://github.com/rust-lang/rust/pull/134536) +- [Make the `dangerous_implicit_autorefs` lint deny-by-default](https://github.com/rust-lang/rust/pull/141661) +- [Stabilize the avx512 target features](https://github.com/rust-lang/rust/pull/138940) +- [Stabilize `kl` and `widekl` target features for x86](https://github.com/rust-lang/rust/pull/140766) +- [Stabilize `sha512`, `sm3` and `sm4` target features for x86](https://github.com/rust-lang/rust/pull/140767) +- [Stabilize LoongArch target features `f`, `d`, `frecipe`, `lasx`, `lbt`, `lsx`, and `lvz`](https://github.com/rust-lang/rust/pull/135015) +- [Remove `i128` and `u128` from `improper_ctypes_definitions`](https://github.com/rust-lang/rust/pull/137306) +- [Stabilize `repr128` (`#[repr(u128)]`, `#[repr(i128)]`)](https://github.com/rust-lang/rust/pull/138285) +- [Allow `#![doc(test(attr(..)))]` everywhere](https://github.com/rust-lang/rust/pull/140560) +- [Extend temporary lifetime extension to also go through tuple struct and tuple variant constructors](https://github.com/rust-lang/rust/pull/140593) +- [`extern "C"` functions on the `wasm32-unknown-unknown` target now have a standards compliant ABI](https://blog.rust-lang.org/2025/04/04/c-abi-changes-for-wasm32-unknown-unknown/) + + + +Compiler +-------- +- [Default to non-leaf frame pointers on aarch64-linux](https://github.com/rust-lang/rust/pull/140832) +- [Enable non-leaf frame pointers for Arm64EC Windows](https://github.com/rust-lang/rust/pull/140862) +- [Set Apple frame pointers by architecture](https://github.com/rust-lang/rust/pull/141797) + + + + +Platform Support +---------------- +- [Add new Tier-3 targets `loongarch32-unknown-none` and `loongarch32-unknown-none-softfloat`](https://github.com/rust-lang/rust/pull/142053) +- [`x86_64-apple-darwin` is in the process of being demoted to Tier 2 with host tools](https://github.com/rust-lang/rfcs/pull/3841) + +Refer to Rust's [platform support page][platform-support-doc] +for more information on Rust's tiered platform support. + +[platform-support-doc]: https://doc.rust-lang.org/rustc/platform-support.html + + + +Libraries +--------- +- [Specify the base path for `file!`](https://github.com/rust-lang/rust/pull/134442) +- [Allow storing `format_args!()` in a variable](https://github.com/rust-lang/rust/pull/140748) +- [Add `#[must_use]` to `[T; N]::map`](https://github.com/rust-lang/rust/pull/140957) +- [Implement `DerefMut` for `Lazy{Cell,Lock}`](https://github.com/rust-lang/rust/pull/129334) +- [Implement `Default` for `array::IntoIter`](https://github.com/rust-lang/rust/pull/141574) +- [Implement `Clone` for `slice::ChunkBy`](https://github.com/rust-lang/rust/pull/138016) +- [Implement `io::Seek` for `io::Take`](https://github.com/rust-lang/rust/pull/138023) + + + + +Stabilized APIs +--------------- + +- [`NonZero`](https://doc.rust-lang.org/stable/std/num/struct.NonZero.html) +- Many intrinsics for x86, not enumerated here + - [AVX512 intrinsics](https://github.com/rust-lang/rust/issues/111137) + - [`SHA512`, `SM3` and `SM4` intrinsics](https://github.com/rust-lang/rust/issues/126624) +- [`File::lock`](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.lock) +- [`File::lock_shared`](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.lock_shared) +- [`File::try_lock`](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.try_lock) +- [`File::try_lock_shared`](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.try_lock_shared) +- [`File::unlock`](https://doc.rust-lang.org/stable/std/fs/struct.File.html#method.unlock) +- [`NonNull::from_ref`](https://doc.rust-lang.org/stable/std/ptr/struct.NonNull.html#method.from_ref) +- [`NonNull::from_mut`](https://doc.rust-lang.org/stable/std/ptr/struct.NonNull.html#method.from_mut) +- [`NonNull::without_provenance`](https://doc.rust-lang.org/stable/std/ptr/struct.NonNull.html#method.without_provenance) +- [`NonNull::with_exposed_provenance`](https://doc.rust-lang.org/stable/std/ptr/struct.NonNull.html#method.with_exposed_provenance) +- [`NonNull::expose_provenance`](https://doc.rust-lang.org/stable/std/ptr/struct.NonNull.html#method.expose_provenance) +- [`OsString::leak`](https://doc.rust-lang.org/stable/std/ffi/struct.OsString.html#method.leak) +- [`PathBuf::leak`](https://doc.rust-lang.org/stable/std/path/struct.PathBuf.html#method.leak) +- [`Result::flatten`](https://doc.rust-lang.org/stable/std/result/enum.Result.html#method.flatten) +- [`std::os::linux::net::TcpStreamExt::quickack`](https://doc.rust-lang.org/stable/std/os/linux/net/trait.TcpStreamExt.html#tymethod.quickack) +- [`std::os::linux::net::TcpStreamExt::set_quickack`](https://doc.rust-lang.org/stable/std/os/linux/net/trait.TcpStreamExt.html#tymethod.set_quickack) + +These previously stable APIs are now stable in const contexts: + +- [`<[T; N]>::as_mut_slice`](https://doc.rust-lang.org/stable/std/primitive.array.html#method.as_mut_slice) +- [`<[u8]>::eq_ignore_ascii_case`](https://doc.rust-lang.org/stable/std/primitive.slice.html#impl-%5Bu8%5D/method.eq_ignore_ascii_case) +- [`str::eq_ignore_ascii_case`](https://doc.rust-lang.org/stable/std/primitive.str.html#impl-str/method.eq_ignore_ascii_case) + + + + +Cargo +----- +- [`cargo fix` and `cargo clippy --fix` now default to the same Cargo target selection as other build commands.](https://github.com/rust-lang/cargo/pull/15192/) Previously it would apply to all targets (like binaries, examples, tests, etc.). The `--edition` flag still applies to all targets. +- [Stabilize doctest-xcompile.](https://github.com/rust-lang/cargo/pull/15462/) Doctests are now tested when cross-compiling. Just like other tests, it will use the [`runner` setting](https://doc.rust-lang.org/cargo/reference/config.html#targettriplerunner) to run the tests. If you need to disable tests for a target, you can use the [ignore doctest attribute](https://doc.rust-lang.org/rustdoc/write-documentation/documentation-tests.html#ignoring-targets) to specify the targets to ignore. + + + + +Rustdoc +----- +- [On mobile, make the sidebar full width and linewrap](https://github.com/rust-lang/rust/pull/139831). This makes long section and item names much easier to deal with on mobile. + + + + +Compatibility Notes +------------------- +- [Make `missing_fragment_specifier` an unconditional error](https://github.com/rust-lang/rust/pull/128425) +- [Enabling the `neon` target feature on `aarch64-unknown-none-softfloat` causes a warning](https://github.com/rust-lang/rust/pull/135160) because mixing code with and without that target feature is not properly supported by LLVM +- [Sized Hierarchy: Part I](https://github.com/rust-lang/rust/pull/137944) + - Introduces a small breaking change affecting `?Sized` bounds on impls on recursive types which contain associated type projections. It is not expected to affect any existing published crates. Can be fixed by refactoring the involved types or opting into the `sized_hierarchy` unstable feature. See the [FCP report](https://github.com/rust-lang/rust/pull/137944#issuecomment-2912207485) for a code example. +- The warn-by-default `elided_named_lifetimes` lint is [superseded by the warn-by-default `mismatched_lifetime_syntaxes` lint.](https://github.com/rust-lang/rust/pull/138677) +- [Error on recursive opaque types earlier in the type checker](https://github.com/rust-lang/rust/pull/139419) +- [Type inference side effects from requiring element types of array repeat expressions are `Copy` are now only available at the end of type checking](https://github.com/rust-lang/rust/pull/139635) +- [The deprecated accidentally-stable `std::intrinsics::{copy,copy_nonoverlapping,write_bytes}` are now proper intrinsics](https://github.com/rust-lang/rust/pull/139916). There are no debug assertions guarding against UB, and they cannot be coerced to function pointers. +- [Remove long-deprecated `std::intrinsics::drop_in_place`](https://github.com/rust-lang/rust/pull/140151) +- [Make well-formedness predicates no longer coinductive](https://github.com/rust-lang/rust/pull/140208) +- [Remove hack when checking impl method compatibility](https://github.com/rust-lang/rust/pull/140557) +- [Remove unnecessary type inference due to built-in trait object impls](https://github.com/rust-lang/rust/pull/141352) +- [Lint against "stdcall", "fastcall", and "cdecl" on non-x86-32 targets](https://github.com/rust-lang/rust/pull/141435) +- [Future incompatibility warnings relating to the never type (`!`) are now reported in dependencies](https://github.com/rust-lang/rust/pull/141937) +- [Ensure `std::ptr::copy_*` intrinsics also perform the static self-init checks](https://github.com/rust-lang/rust/pull/142575) +- [`extern "C"` functions on the `wasm32-unknown-unknown` target now have a standards compliant ABI](https://blog.rust-lang.org/2025/04/04/c-abi-changes-for-wasm32-unknown-unknown/) + + + +Internal Changes +---------------- + +These changes do not affect any public interfaces of Rust, but they represent +significant improvements to the performance or internals of rustc and related +tools. + +- [Correctly un-remap compiler sources paths with the `rustc-dev` component](https://github.com/rust-lang/rust/pull/142377) + + Version 1.88.0 (2025-06-26) ========================== @@ -1641,7 +2115,7 @@ Language - [Undeprecate lint `unstable_features` and make use of it in the compiler.](https://github.com/rust-lang/rust/pull/118639/) - [Make inductive cycles in coherence ambiguous always.](https://github.com/rust-lang/rust/pull/118649/) - [Get rid of type-driven traversal in const-eval interning](https://github.com/rust-lang/rust/pull/119044/), - only as a [future compatiblity lint](https://github.com/rust-lang/rust/pull/122204) for now. + only as a [future compatibility lint](https://github.com/rust-lang/rust/pull/122204) for now. - [Deny braced macro invocations in let-else.](https://github.com/rust-lang/rust/pull/119062/) diff --git a/bootstrap.example.toml b/bootstrap.example.toml index 31966af33012..6f37e51a47de 100644 --- a/bootstrap.example.toml +++ b/bootstrap.example.toml @@ -9,7 +9,7 @@ # a custom configuration file can also be specified with `--config` to the build # system. # -# Note that the following are equivelent, for more details see . +# Note that the following are equivalent, for more details see . # # build.verbose = 1 # @@ -325,6 +325,9 @@ # Defaults to the Python interpreter used to execute x.py. #build.python = "python" +# The path to (or name of) the resource compiler executable to use on Windows. +#build.windows-rc = "rc.exe" + # The path to the REUSE executable to use. Note that REUSE is not required in # most cases, as our tooling relies on a cached (and shrunk) copy of the # REUSE output present in the git repository and in our source tarballs. @@ -345,9 +348,9 @@ # want to use vendoring. See https://forge.rust-lang.org/infra/other-installation-methods.html#source-code. #build.vendor = if "is a tarball source" && "vendor" dir exists && ".cargo/config.toml" file exists { true } else { false } -# Typically the build system will build the Rust compiler twice. The second -# compiler, however, will simply use its own libraries to link against. If you -# would rather to perform a full bootstrap, compiling the compiler three times, +# If you build the compiler more than twice (stage3+) or the standard library more than once +# (stage 2+), the third compiler and second library will get uplifted from stage2 and stage1, +# respectively. If you would like to disable this uplifting, and rather perform a full bootstrap, # then you can set this option to true. # # This is only useful for verifying that rustc generates reproducible builds. @@ -407,8 +410,11 @@ #build.profiler = false # Use the optimized LLVM C intrinsics for `compiler_builtins`, rather than Rust intrinsics. -# Requires the LLVM submodule to be managed by bootstrap (i.e. not external) so that `compiler-rt` -# sources are available. +# Choosing true requires the LLVM submodule to be managed by bootstrap (i.e. not external) +# so that `compiler-rt` sources are available. +# +# Setting this to a path removes the requirement for a C toolchain, but requires setting the +# path to an existing library containing the builtins library from LLVM's compiler-rt. # # Setting this to `false` generates slower code, but removes the requirement for a C toolchain in # order to run `x check`. @@ -470,9 +476,6 @@ # when the stage 0 compiler is actually built from in-tree sources. #build.compiletest-allow-stage0 = false -# Whether to use the precompiled stage0 libtest with compiletest. -#build.compiletest-use-stage0-libtest = true - # Default value for the `--extra-checks` flag of tidy. # # See `./x test tidy --help` for details. @@ -482,7 +485,7 @@ # Use `--extra-checks=''` to temporarily disable all extra checks. # # Automatically enabled in the "tools" profile. -# Set to the empty string to force disable (recommeded for hdd systems). +# Set to the empty string to force disable (recommended for hdd systems). #build.tidy-extra-checks = "" # Indicates whether ccache is used when building certain artifacts (e.g. LLVM). @@ -740,22 +743,25 @@ # result (broken, compiling, testing) into this JSON file. #rust.save-toolstates = (path) -# This is an array of the codegen backends that will be compiled for the rustc -# that's being compiled. The default is to only build the LLVM codegen backend, -# and currently the only standard options supported are `"llvm"`, `"cranelift"` -# and `"gcc"`. The first backend in this list will be used as default by rustc -# when no explicit backend is specified. +# This array serves three distinct purposes: +# - Backends in this list will be automatically compiled and included in the sysroot of each +# rustc compiled by bootstrap. +# - The first backend in this list will be configured as the **default codegen backend** by each +# rustc compiled by bootstrap. In other words, if the first backend is e.g. cranelift, then when +# we build a stage 1 rustc, it will by default compile Rust programs using the Cranelift backend. +# This also means that stage 2 rustc would get built by the Cranelift backend. +# - Running `x dist` (without additional arguments, or with `--include-default-paths`) will produce +# a dist component/tarball for the Cranelift backend if it is included in this array. +# +# Note that the LLVM codegen backend is special and will always be built and distributed. +# +# Currently, the only standard options supported here are `"llvm"`, `"cranelift"` and `"gcc"`. #rust.codegen-backends = ["llvm"] -# Indicates whether LLD will be compiled and made available in the sysroot for rustc to execute, and -# whether to set it as rustc's default linker on `x86_64-unknown-linux-gnu`. This will also only be -# when *not* building an external LLVM (so only when using `download-ci-llvm` or building LLVM from -# the in-tree source): setting `llvm-config` in the `[target.x86_64-unknown-linux-gnu]` section will -# make this default to false. -#rust.lld = false in all cases, except on `x86_64-unknown-linux-gnu` as described above, where it is true +# Indicates whether LLD will be compiled and made available in the sysroot for rustc to execute, +#rust.lld = false, except for targets that opt into LLD (see `target.default-linker-linux-override`) -# Indicates whether LLD will be used to link Rust crates during bootstrap on -# supported platforms. +# Indicates if we should override the linker used to link Rust crates during bootstrap to be LLD. # If set to `true` or `"external"`, a global `lld` binary that has to be in $PATH # will be used. # If set to `"self-contained"`, rust-lld from the snapshot compiler will be used. @@ -763,7 +769,7 @@ # On MSVC, LLD will not be used if we're cross linking. # # Explicitly setting the linker for a target will override this option when targeting MSVC. -#rust.use-lld = false +#rust.bootstrap-override-lld = false # Indicates whether some LLVM tools, like llvm-objdump, will be made available in the # sysroot. @@ -845,6 +851,17 @@ # as libstd features, this option can also be used to configure features such as optimize_for_size. #rust.std-features = ["panic_unwind"] +# Trigger a `DebugBreak` after an internal compiler error during bootstrap on Windows +#rust.break-on-ice = true + +# Set the number of threads for the compiler frontend used during compilation of Rust code (passed to `-Zthreads`). +# The valid options are: +# 0 - Set the number of threads according to the detected number of threads of the host system +# 1 - Use a single thread for compilation of Rust code (the default) +# N - Number of threads used for compilation of Rust code +# +#rust.parallel-frontend-threads = 1 + # ============================================================================= # Distribution options # @@ -925,7 +942,7 @@ # Linker to be used to bootstrap Rust code. Note that the # default value is platform specific, and if not specified it may also depend on # what platform is crossing to what platform. -# Setting this will override the `use-lld` option for Rust code when targeting MSVC. +# Setting this will override the `bootstrap-override-lld` option for Rust code when targeting MSVC. #linker = "cc" (path) # Should rustc and the standard library be built with split debuginfo? Default @@ -1033,14 +1050,30 @@ #runner = (string) # Use the optimized LLVM C intrinsics for `compiler_builtins`, rather than Rust intrinsics -# on this target. -# Requires the LLVM submodule to be managed by bootstrap (i.e. not external) so that `compiler-rt` -# sources are available. +# on this target. Choosing true requires the LLVM submodule to be managed by bootstrap +# (i.e. not external) so that `compiler-rt` sources are available. +# +# Setting this to a path removes the requirement for a C toolchain, but requires setting the +# path to an existing library containing the builtins library from LLVM's compiler-rt. # # Setting this to `false` generates slower code, but removes the requirement for a C toolchain in # order to run `x check`. -#optimized-compiler-builtins = build.optimized-compiler-builtins (bool) +#optimized-compiler-builtins = build.optimized-compiler-builtins (bool or path) # Link the compiler and LLVM against `jemalloc` instead of the default libc allocator. # This overrides the global `rust.jemalloc` option. See that option for more info. #jemalloc = rust.jemalloc (bool) + +# The linker configuration that will *override* the default linker used for Linux +# targets in the built compiler. +# +# The following values are supported: +# - `off` => do not apply any override and use the default linker. This can be used to opt out of +# linker overrides set by bootstrap for specific targets (see below). +# - `self-contained-lld-cc` => override the default linker to be self-contained LLD (`rust-lld`) +# that is invoked through `cc`. +# +# Currently, the following targets automatically opt into the self-contained LLD linker, unless you +# pass `off`: +# - x86_64-unknown-linux-gnu +#default-linker-linux-override = "off" (for most targets) diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml index 3ca752354466..9ef8fa75062a 100644 --- a/compiler/rustc/Cargo.toml +++ b/compiler/rustc/Cargo.toml @@ -30,6 +30,12 @@ features = ['unprefixed_malloc_on_supported_platforms'] check_only = ['rustc_driver_impl/check_only'] jemalloc = ['dep:tikv-jemalloc-sys'] llvm = ['rustc_driver_impl/llvm'] +llvm_enzyme = ['rustc_driver_impl/llvm_enzyme'] max_level_info = ['rustc_driver_impl/max_level_info'] rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts'] # tidy-alphabetical-end + +[build-dependencies] +# tidy-alphabetical-start +rustc_windows_rc = { path = "../rustc_windows_rc" } +# tidy-alphabetical-end diff --git a/compiler/rustc/build.rs b/compiler/rustc/build.rs index 8b7d28d2b8aa..9b5def53e3cb 100644 --- a/compiler/rustc/build.rs +++ b/compiler/rustc/build.rs @@ -1,4 +1,6 @@ -use std::env; +use std::{env, path}; + +use rustc_windows_rc::{VersionInfoFileType, compile_windows_resource_file}; fn main() { let target_os = env::var("CARGO_CFG_TARGET_OS"); @@ -13,6 +15,18 @@ fn main() { // Add a manifest file to rustc.exe. fn set_windows_exe_options() { + set_windows_resource(); + set_windows_manifest(); +} + +fn set_windows_resource() { + let stem = path::PathBuf::from("rustc_main_resource"); + let file_description = "rustc"; + let res_file = compile_windows_resource_file(&stem, file_description, VersionInfoFileType::App); + println!("cargo:rustc-link-arg={}", res_file.display()); +} + +fn set_windows_manifest() { static WINDOWS_MANIFEST_FILE: &str = "Windows Manifest.xml"; let mut manifest = env::current_dir().unwrap(); diff --git a/compiler/rustc_abi/Cargo.toml b/compiler/rustc_abi/Cargo.toml index 5f9afc46a1ac..83d96d8d04da 100644 --- a/compiler/rustc_abi/Cargo.toml +++ b/compiler/rustc_abi/Cargo.toml @@ -9,6 +9,7 @@ bitflags = "2.4.1" rand = { version = "0.9.0", default-features = false, optional = true } rand_xoshiro = { version = "0.7.0", optional = true } rustc_data_structures = { path = "../rustc_data_structures", optional = true } +rustc_error_messages = { path = "../rustc_error_messages", optional = true } rustc_hashes = { path = "../rustc_hashes" } rustc_index = { path = "../rustc_index", default-features = false } rustc_macros = { path = "../rustc_macros", optional = true } @@ -24,6 +25,7 @@ default = ["nightly", "randomize"] # without depending on rustc_data_structures, rustc_macros and rustc_serialize nightly = [ "dep:rustc_data_structures", + "dep:rustc_error_messages", "dep:rustc_macros", "dep:rustc_serialize", "dep:rustc_span", diff --git a/compiler/rustc_abi/src/callconv/reg.rs b/compiler/rustc_abi/src/callconv/reg.rs index 8cf140dbaad4..66c8056d0c2a 100644 --- a/compiler/rustc_abi/src/callconv/reg.rs +++ b/compiler/rustc_abi/src/callconv/reg.rs @@ -42,22 +42,22 @@ impl Reg { let dl = cx.data_layout(); match self.kind { RegKind::Integer => match self.size.bits() { - 1 => dl.i1_align.abi, - 2..=8 => dl.i8_align.abi, - 9..=16 => dl.i16_align.abi, - 17..=32 => dl.i32_align.abi, - 33..=64 => dl.i64_align.abi, - 65..=128 => dl.i128_align.abi, + 1 => dl.i1_align, + 2..=8 => dl.i8_align, + 9..=16 => dl.i16_align, + 17..=32 => dl.i32_align, + 33..=64 => dl.i64_align, + 65..=128 => dl.i128_align, _ => panic!("unsupported integer: {self:?}"), }, RegKind::Float => match self.size.bits() { - 16 => dl.f16_align.abi, - 32 => dl.f32_align.abi, - 64 => dl.f64_align.abi, - 128 => dl.f128_align.abi, + 16 => dl.f16_align, + 32 => dl.f32_align, + 64 => dl.f64_align, + 128 => dl.f128_align, _ => panic!("unsupported float: {self:?}"), }, - RegKind::Vector => dl.llvmlike_vector_align(self.size).abi, + RegKind::Vector => dl.llvmlike_vector_align(self.size), } } } diff --git a/compiler/rustc_abi/src/extern_abi.rs b/compiler/rustc_abi/src/extern_abi.rs index 29a3678abf3f..e3b2b1eff72d 100644 --- a/compiler/rustc_abi/src/extern_abi.rs +++ b/compiler/rustc_abi/src/extern_abi.rs @@ -6,6 +6,8 @@ use std::hash::{Hash, Hasher}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd}; #[cfg(feature = "nightly")] use rustc_macros::{Decodable, Encodable}; +#[cfg(feature = "nightly")] +use rustc_span::Symbol; use crate::AbiFromStrErr; @@ -223,6 +225,16 @@ impl StableOrd for ExternAbi { const THIS_IMPLEMENTATION_HAS_BEEN_TRIPLE_CHECKED: () = (); } +#[cfg(feature = "nightly")] +rustc_error_messages::into_diag_arg_using_display!(ExternAbi); + +#[cfg(feature = "nightly")] +pub enum CVariadicStatus { + NotSupported, + Stable, + Unstable { feature: Symbol }, +} + impl ExternAbi { /// An ABI "like Rust" /// @@ -235,23 +247,33 @@ impl ExternAbi { matches!(self, Rust | RustCall | RustCold) } - pub fn supports_varargs(self) -> bool { + /// Returns whether the ABI supports C variadics. This only controls whether we allow *imports* + /// of such functions via `extern` blocks; there's a separate check during AST construction + /// guarding *definitions* of variadic functions. + #[cfg(feature = "nightly")] + pub fn supports_c_variadic(self) -> CVariadicStatus { // * C and Cdecl obviously support varargs. // * C can be based on Aapcs, SysV64 or Win64, so they must support varargs. // * EfiApi is based on Win64 or C, so it also supports it. + // * System automatically falls back to C when used with variadics, therefore supports it. // // * Stdcall does not, because it would be impossible for the callee to clean // up the arguments. (callee doesn't know how many arguments are there) // * Same for Fastcall, Vectorcall and Thiscall. // * Other calling conventions are related to hardware or the compiler itself. + // + // All of the supported ones must have a test in `tests/codegen/cffi/c-variadic-ffi.rs`. match self { Self::C { .. } | Self::Cdecl { .. } | Self::Aapcs { .. } | Self::Win64 { .. } | Self::SysV64 { .. } - | Self::EfiApi => true, - _ => false, + | Self::EfiApi => CVariadicStatus::Stable, + Self::System { .. } => { + CVariadicStatus::Unstable { feature: rustc_span::sym::extern_system_varargs } + } + _ => CVariadicStatus::NotSupported, } } } diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index c2405553756b..14356813b7bb 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -174,11 +174,11 @@ impl LayoutCalculator { // Non-power-of-two vectors have padding up to the next power-of-two. // If we're a packed repr, remove the padding while keeping the alignment as close // to a vector as possible. - (BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) }) + (BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size)) } else { (BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size)) }; - let size = size.align_to(align.abi); + let size = size.align_to(align); Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, @@ -190,7 +190,7 @@ impl LayoutCalculator { largest_niche: elt.largest_niche, uninhabited: false, size, - align, + align: AbiAlign::new(align), max_repr_align: None, unadjusted_abi_align: elt.align.abi, randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)), @@ -388,7 +388,7 @@ impl LayoutCalculator { return Err(LayoutCalculatorError::UnexpectedUnsized(*field)); } - align = align.max(field.align); + align = align.max(field.align.abi); max_repr_align = max_repr_align.max(field.max_repr_align); size = cmp::max(size, field.size); @@ -423,13 +423,13 @@ impl LayoutCalculator { } if let Some(pack) = repr.pack { - align = align.min(AbiAlign::new(pack)); + align = align.min(pack); } // The unadjusted ABI alignment does not include repr(align), but does include repr(pack). // See documentation on `LayoutData::unadjusted_abi_align`. - let unadjusted_abi_align = align.abi; + let unadjusted_abi_align = align; if let Some(repr_align) = repr.align { - align = align.max(AbiAlign::new(repr_align)); + align = align.max(repr_align); } // `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate. let align = align; @@ -441,14 +441,12 @@ impl LayoutCalculator { Ok(Some((repr, _))) => match repr { // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _) - if repr.scalar_align(dl).unwrap() != align.abi => + if repr.scalar_align(dl).unwrap() != align => { BackendRepr::Memory { sized: true } } // Vectors require at least element alignment, else disable the opt - BackendRepr::SimdVector { element, count: _ } - if element.align(dl).abi > align.abi => - { + BackendRepr::SimdVector { element, count: _ } if element.align(dl).abi > align => { BackendRepr::Memory { sized: true } } // the alignment tests passed and we can use this @@ -474,8 +472,8 @@ impl LayoutCalculator { backend_repr, largest_niche: None, uninhabited: false, - align, - size: size.align_to(align.abi), + align: AbiAlign::new(align), + size: size.align_to(align), max_repr_align, unadjusted_abi_align, randomization_seed: combined_seed, @@ -594,23 +592,13 @@ impl LayoutCalculator { discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool), discriminants: impl Iterator, ) -> LayoutCalculatorResult { - // Until we've decided whether to use the tagged or - // niche filling LayoutData, we don't want to intern the - // variant layouts, so we can't store them in the - // overall LayoutData. Store the overall LayoutData - // and the variant LayoutDatas here until then. - struct TmpLayout { - layout: LayoutData, - variants: IndexVec>, - } - let dl = self.cx.data_layout(); // bail if the enum has an incoherent repr that cannot be computed if repr.packed() { return Err(LayoutCalculatorError::ReprConflict); } - let calculate_niche_filling_layout = || -> Option> { + let calculate_niche_filling_layout = || -> Option> { if repr.inhibit_enum_layout_opt() { return None; } @@ -621,7 +609,7 @@ impl LayoutCalculator { let mut align = dl.aggregate_align; let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align.abi; + let mut unadjusted_abi_align = align; let mut variant_layouts = variants .iter_enumerated() @@ -629,7 +617,7 @@ impl LayoutCalculator { let mut st = self.univariant(v, repr, StructKind::AlwaysSized).ok()?; st.variants = Variants::Single { index: j }; - align = align.max(st.align); + align = align.max(st.align.abi); max_repr_align = max_repr_align.max(st.max_repr_align); unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); @@ -656,7 +644,7 @@ impl LayoutCalculator { let (niche_start, niche_scalar) = niche.reserve(dl, count)?; let niche_offset = niche.offset; let niche_size = niche.value.size(dl); - let size = variant_layouts[largest_variant_index].size.align_to(align.abi); + let size = variant_layouts[largest_variant_index].size.align_to(align); let all_variants_fit = variant_layouts.iter_enumerated_mut().all(|(i, layout)| { if i == largest_variant_index { @@ -709,7 +697,7 @@ impl LayoutCalculator { .iter_enumerated() .all(|(i, layout)| i == largest_variant_index || layout.size == Size::ZERO); let same_size = size == variant_layouts[largest_variant_index].size; - let same_align = align == variant_layouts[largest_variant_index].align; + let same_align = align == variant_layouts[largest_variant_index].align.abi; let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited()); let abi = if same_size && same_align && others_zst { @@ -746,7 +734,7 @@ impl LayoutCalculator { niche_start, }, tag_field: FieldIdx::new(0), - variants: IndexVec::new(), + variants: variant_layouts, }, fields: FieldsShape::Arbitrary { offsets: [niche_offset].into(), @@ -756,13 +744,13 @@ impl LayoutCalculator { largest_niche, uninhabited, size, - align, + align: AbiAlign::new(align), max_repr_align, unadjusted_abi_align, randomization_seed: combined_seed, }; - Some(TmpLayout { layout, variants: variant_layouts }) + Some(layout) }; let niche_filling_layout = calculate_niche_filling_layout(); @@ -828,7 +816,7 @@ impl LayoutCalculator { let mut align = dl.aggregate_align; let mut max_repr_align = repr.align; - let mut unadjusted_abi_align = align.abi; + let mut unadjusted_abi_align = align; let mut size = Size::ZERO; @@ -870,7 +858,7 @@ impl LayoutCalculator { } } size = cmp::max(size, st.size); - align = align.max(st.align); + align = align.max(st.align.abi); max_repr_align = max_repr_align.max(st.max_repr_align); unadjusted_abi_align = unadjusted_abi_align.max(st.unadjusted_abi_align); Ok(st) @@ -878,7 +866,7 @@ impl LayoutCalculator { .collect::, _>>()?; // Align the maximum variant size to the largest alignment. - size = size.align_to(align.abi); + size = size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { @@ -1052,7 +1040,7 @@ impl LayoutCalculator { }; if pair_offsets[FieldIdx::new(0)] == Size::ZERO && pair_offsets[FieldIdx::new(1)] == *offset - && align == pair.align + && align == pair.align.abi && size == pair.size { // We can use `ScalarPair` only when it matches our @@ -1076,7 +1064,7 @@ impl LayoutCalculator { // Also need to bump up the size and alignment, so that the entire value fits // in here. variant.size = cmp::max(variant.size, size); - variant.align.abi = cmp::max(variant.align.abi, align.abi); + variant.align.abi = cmp::max(variant.align.abi, align); } } } @@ -1093,7 +1081,7 @@ impl LayoutCalculator { tag, tag_encoding: TagEncoding::Direct, tag_field: FieldIdx::new(0), - variants: IndexVec::new(), + variants: layout_variants, }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), @@ -1102,25 +1090,23 @@ impl LayoutCalculator { largest_niche, uninhabited, backend_repr: abi, - align, + align: AbiAlign::new(align), size, max_repr_align, unadjusted_abi_align, randomization_seed: combined_seed, }; - let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants }; - - let mut best_layout = match (tagged_layout, niche_filling_layout) { + let best_layout = match (tagged_layout, niche_filling_layout) { (tl, Some(nl)) => { // Pick the smaller layout; otherwise, // pick the layout with the larger niche; otherwise, // pick tagged as it has simpler codegen. use cmp::Ordering::*; - let niche_size = |tmp_l: &TmpLayout| { - tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl)) + let niche_size = |l: &LayoutData| { + l.largest_niche.map_or(0, |n| n.available(dl)) }; - match (tl.layout.size.cmp(&nl.layout.size), niche_size(&tl).cmp(&niche_size(&nl))) { + match (tl.size.cmp(&nl.size), niche_size(&tl).cmp(&niche_size(&nl))) { (Greater, _) => nl, (Equal, Less) => nl, _ => tl, @@ -1129,16 +1115,7 @@ impl LayoutCalculator { (tl, None) => tl, }; - // Now we can intern the variant layouts and store them in the enum layout. - best_layout.layout.variants = match best_layout.layout.variants { - Variants::Multiple { tag, tag_encoding, tag_field, .. } => { - Variants::Multiple { tag, tag_encoding, tag_field, variants: best_layout.variants } - } - Variants::Single { .. } | Variants::Empty => { - panic!("encountered a single-variant or empty enum during multi-variant layout") - } - }; - Ok(best_layout.layout) + Ok(best_layout) } fn univariant_biased< @@ -1190,7 +1167,7 @@ impl LayoutCalculator { // To allow unsizing `&Foo` -> `&Foo`, the layout of the struct must // not depend on the layout of the tail. let max_field_align = - fields_excluding_tail.iter().map(|f| f.align.abi.bytes()).max().unwrap_or(1); + fields_excluding_tail.iter().map(|f| f.align.bytes()).max().unwrap_or(1); let largest_niche_size = fields_excluding_tail .iter() .filter_map(|f| f.largest_niche) @@ -1210,7 +1187,7 @@ impl LayoutCalculator { } else { // Returns `log2(effective-align)`. The calculation assumes that size is an // integer multiple of align, except for ZSTs. - let align = layout.align.abi.bytes(); + let align = layout.align.bytes(); let size = layout.size.bytes(); let niche_size = layout.largest_niche.map(|n| n.available(dl)).unwrap_or(0); // Group [u8; 4] with align-4 or [u8; 6] with align-2 fields. @@ -1309,7 +1286,7 @@ impl LayoutCalculator { if let StructKind::Prefixed(prefix_size, prefix_align) = kind { let prefix_align = if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align }; - align = align.max(AbiAlign::new(prefix_align)); + align = align.max(prefix_align); offset = prefix_size.align_to(prefix_align); } for &i in &inverse_memory_index { @@ -1333,7 +1310,7 @@ impl LayoutCalculator { field.align }; offset = offset.align_to(field_align.abi); - align = align.max(field_align); + align = align.max(field_align.abi); max_repr_align = max_repr_align.max(field.max_repr_align); debug!("univariant offset: {:?} field: {:#?}", offset, field); @@ -1360,9 +1337,9 @@ impl LayoutCalculator { // The unadjusted ABI alignment does not include repr(align), but does include repr(pack). // See documentation on `LayoutData::unadjusted_abi_align`. - let unadjusted_abi_align = align.abi; + let unadjusted_abi_align = align; if let Some(repr_align) = repr.align { - align = align.max(AbiAlign::new(repr_align)); + align = align.max(repr_align); } // `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate. let align = align; @@ -1381,7 +1358,7 @@ impl LayoutCalculator { debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices())); inverse_memory_index.into_iter().map(|it| it.index() as u32).collect() }; - let size = min_size.align_to(align.abi); + let size = min_size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { return Err(LayoutCalculatorError::SizeOverflow); @@ -1404,8 +1381,7 @@ impl LayoutCalculator { layout_of_single_non_zst_field = Some(field); // Field fills the struct and it has a scalar or scalar pair ABI. - if offsets[i].bytes() == 0 && align.abi == field.align.abi && size == field.size - { + if offsets[i].bytes() == 0 && align == field.align.abi && size == field.size { match field.backend_repr { // For plain scalars, or vectors of them, we can't unpack // newtypes for `#[repr(C)]`, as that affects C ABIs. @@ -1449,7 +1425,7 @@ impl LayoutCalculator { }; if offsets[i] == pair_offsets[FieldIdx::new(0)] && offsets[j] == pair_offsets[FieldIdx::new(1)] - && align == pair.align + && align == pair.align.abi && size == pair.size { // We can use `ScalarPair` only when it matches our @@ -1471,7 +1447,7 @@ impl LayoutCalculator { Some(l) => l.unadjusted_abi_align, None => { // `repr(transparent)` with all ZST fields. - align.abi + align } } } else { @@ -1486,7 +1462,7 @@ impl LayoutCalculator { backend_repr: abi, largest_niche, uninhabited, - align, + align: AbiAlign::new(align), size, max_repr_align, unadjusted_abi_align, @@ -1509,7 +1485,7 @@ impl LayoutCalculator { for i in layout.fields.index_by_increasing_offset() { let offset = layout.fields.offset(i); let f = &fields[FieldIdx::new(i)]; - write!(s, "[o{}a{}s{}", offset.bytes(), f.align.abi.bytes(), f.size.bytes()).unwrap(); + write!(s, "[o{}a{}s{}", offset.bytes(), f.align.bytes(), f.size.bytes()).unwrap(); if let Some(n) = f.largest_niche { write!( s, diff --git a/compiler/rustc_abi/src/layout/simple.rs b/compiler/rustc_abi/src/layout/simple.rs index 0d0706defc2e..b3807c872739 100644 --- a/compiler/rustc_abi/src/layout/simple.rs +++ b/compiler/rustc_abi/src/layout/simple.rs @@ -4,7 +4,8 @@ use rustc_hashes::Hash64; use rustc_index::{Idx, IndexVec}; use crate::{ - BackendRepr, FieldsShape, HasDataLayout, LayoutData, Niche, Primitive, Scalar, Size, Variants, + AbiAlign, BackendRepr, FieldsShape, HasDataLayout, LayoutData, Niche, Primitive, Scalar, Size, + Variants, }; /// "Simple" layout constructors that cannot fail. @@ -20,10 +21,10 @@ impl LayoutData { backend_repr: BackendRepr::Memory { sized }, largest_niche: None, uninhabited: false, - align: dl.i8_align, + align: AbiAlign::new(dl.i8_align), size: Size::ZERO, max_repr_align: None, - unadjusted_abi_align: dl.i8_align.abi, + unadjusted_abi_align: dl.i8_align, randomization_seed: Hash64::new(0), } } @@ -37,10 +38,10 @@ impl LayoutData { backend_repr: BackendRepr::Memory { sized: true }, largest_niche: None, uninhabited: true, - align: dl.i8_align, + align: AbiAlign::new(dl.i8_align), size: Size::ZERO, max_repr_align: None, - unadjusted_abi_align: dl.i8_align.abi, + unadjusted_abi_align: dl.i8_align, randomization_seed: Hash64::ZERO, } } @@ -89,10 +90,10 @@ impl LayoutData { pub fn scalar_pair(cx: &C, a: Scalar, b: Scalar) -> Self { let dl = cx.data_layout(); - let b_align = b.align(dl); - let align = a.align(dl).max(b_align).max(dl.aggregate_align); - let b_offset = a.size(dl).align_to(b_align.abi); - let size = (b_offset + b.size(dl)).align_to(align.abi); + let b_align = b.align(dl).abi; + let align = a.align(dl).abi.max(b_align).max(dl.aggregate_align); + let b_offset = a.size(dl).align_to(b_align); + let size = (b_offset + b.size(dl)).align_to(align); // HACK(nox): We iter on `b` and then `a` because `max_by_key` // returns the last maximum. @@ -112,10 +113,10 @@ impl LayoutData { backend_repr: BackendRepr::ScalarPair(a, b), largest_niche, uninhabited: false, - align, + align: AbiAlign::new(align), size, max_repr_align: None, - unadjusted_abi_align: align.abi, + unadjusted_abi_align: align, randomization_seed: Hash64::new(combined_seed), } } @@ -138,10 +139,10 @@ impl LayoutData { backend_repr: BackendRepr::Memory { sized: true }, largest_niche: None, uninhabited: true, - align: dl.i8_align, + align: AbiAlign::new(dl.i8_align), size: Size::ZERO, max_repr_align: None, - unadjusted_abi_align: dl.i8_align.abi, + unadjusted_abi_align: dl.i8_align, randomization_seed: Hash64::ZERO, } } diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 14e256b8045d..de44c8755a07 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -63,6 +63,8 @@ mod tests; pub use callconv::{Heterogeneous, HomogeneousAggregate, Reg, RegKind}; pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call}; +#[cfg(feature = "nightly")] +pub use extern_abi::CVariadicStatus; pub use extern_abi::{ExternAbi, all_names}; #[cfg(feature = "nightly")] pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx}; @@ -227,7 +229,7 @@ pub struct PointerSpec { /// The size of the bitwise representation of the pointer. pointer_size: Size, /// The alignment of pointers for this address space - pointer_align: AbiAlign, + pointer_align: Align, /// The size of the value a pointer can be offset by in this address space. pointer_offset: Size, /// Pointers into this address space contain extra metadata @@ -240,20 +242,20 @@ pub struct PointerSpec { #[derive(Debug, PartialEq, Eq)] pub struct TargetDataLayout { pub endian: Endian, - pub i1_align: AbiAlign, - pub i8_align: AbiAlign, - pub i16_align: AbiAlign, - pub i32_align: AbiAlign, - pub i64_align: AbiAlign, - pub i128_align: AbiAlign, - pub f16_align: AbiAlign, - pub f32_align: AbiAlign, - pub f64_align: AbiAlign, - pub f128_align: AbiAlign, - pub aggregate_align: AbiAlign, + pub i1_align: Align, + pub i8_align: Align, + pub i16_align: Align, + pub i32_align: Align, + pub i64_align: Align, + pub i128_align: Align, + pub f16_align: Align, + pub f32_align: Align, + pub f64_align: Align, + pub f128_align: Align, + pub aggregate_align: Align, /// Alignments for vector types. - pub vector_align: Vec<(Size, AbiAlign)>, + pub vector_align: Vec<(Size, Align)>, pub default_address_space: AddressSpace, pub default_address_space_pointer_spec: PointerSpec, @@ -280,25 +282,25 @@ impl Default for TargetDataLayout { let align = |bits| Align::from_bits(bits).unwrap(); TargetDataLayout { endian: Endian::Big, - i1_align: AbiAlign::new(align(8)), - i8_align: AbiAlign::new(align(8)), - i16_align: AbiAlign::new(align(16)), - i32_align: AbiAlign::new(align(32)), - i64_align: AbiAlign::new(align(32)), - i128_align: AbiAlign::new(align(32)), - f16_align: AbiAlign::new(align(16)), - f32_align: AbiAlign::new(align(32)), - f64_align: AbiAlign::new(align(64)), - f128_align: AbiAlign::new(align(128)), - aggregate_align: AbiAlign { abi: align(8) }, + i1_align: align(8), + i8_align: align(8), + i16_align: align(16), + i32_align: align(32), + i64_align: align(32), + i128_align: align(32), + f16_align: align(16), + f32_align: align(32), + f64_align: align(64), + f128_align: align(128), + aggregate_align: align(8), vector_align: vec![ - (Size::from_bits(64), AbiAlign::new(align(64))), - (Size::from_bits(128), AbiAlign::new(align(128))), + (Size::from_bits(64), align(64)), + (Size::from_bits(128), align(128)), ], default_address_space: AddressSpace::ZERO, default_address_space_pointer_spec: PointerSpec { pointer_size: Size::from_bits(64), - pointer_align: AbiAlign::new(align(64)), + pointer_align: align(64), pointer_offset: Size::from_bits(64), _is_fat: false, }, @@ -315,7 +317,7 @@ pub enum TargetDataLayoutErrors<'a> { MissingAlignment { cause: &'a str }, InvalidAlignment { cause: &'a str, err: AlignFromBytesError }, InconsistentTargetArchitecture { dl: &'a str, target: &'a str }, - InconsistentTargetPointerWidth { pointer_size: u64, target: u32 }, + InconsistentTargetPointerWidth { pointer_size: u64, target: u16 }, InvalidBitsSize { err: String }, UnknownPointerSpecification { err: String }, } @@ -358,7 +360,7 @@ impl TargetDataLayout { .map_err(|err| TargetDataLayoutErrors::InvalidAlignment { cause, err }) }; let abi = parse_bits(s, "alignment", cause)?; - Ok(AbiAlign::new(align_from_bits(abi)?)) + Ok(align_from_bits(abi)?) }; // Parse an alignment sequence, possibly in the form `[:]`, @@ -594,7 +596,7 @@ impl TargetDataLayout { /// psABI-mandated alignment for a vector type, if any #[inline] - fn cabi_vector_align(&self, vec_size: Size) -> Option { + fn cabi_vector_align(&self, vec_size: Size) -> Option { self.vector_align .iter() .find(|(size, _align)| *size == vec_size) @@ -603,10 +605,9 @@ impl TargetDataLayout { /// an alignment resembling the one LLVM would pick for a vector #[inline] - pub fn llvmlike_vector_align(&self, vec_size: Size) -> AbiAlign { - self.cabi_vector_align(vec_size).unwrap_or(AbiAlign::new( - Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap(), - )) + pub fn llvmlike_vector_align(&self, vec_size: Size) -> Align { + self.cabi_vector_align(vec_size) + .unwrap_or(Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap()) } /// Get the pointer size in the default data address space. @@ -652,21 +653,19 @@ impl TargetDataLayout { /// Get the pointer alignment in the default data address space. #[inline] pub fn pointer_align(&self) -> AbiAlign { - self.default_address_space_pointer_spec.pointer_align + AbiAlign::new(self.default_address_space_pointer_spec.pointer_align) } /// Get the pointer alignment in a specific address space. #[inline] pub fn pointer_align_in(&self, c: AddressSpace) -> AbiAlign { - if c == self.default_address_space { - return self.default_address_space_pointer_spec.pointer_align; - } - - if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) { + AbiAlign::new(if c == self.default_address_space { + self.default_address_space_pointer_spec.pointer_align + } else if let Some(e) = self.address_space_info.iter().find(|(a, _)| a == &c) { e.1.pointer_align } else { panic!("Use of unknown address space {c:?}"); - } + }) } } @@ -1183,13 +1182,13 @@ impl Integer { use Integer::*; let dl = cx.data_layout(); - match self { + AbiAlign::new(match self { I8 => dl.i8_align, I16 => dl.i16_align, I32 => dl.i32_align, I64 => dl.i64_align, I128 => dl.i128_align, - } + }) } /// Returns the largest signed value that can be represented by this Integer. @@ -1309,12 +1308,12 @@ impl Float { use Float::*; let dl = cx.data_layout(); - match self { + AbiAlign::new(match self { F16 => dl.f16_align, F32 => dl.f32_align, F64 => dl.f64_align, F128 => dl.f128_align, - } + }) } } @@ -2157,7 +2156,7 @@ impl LayoutData { /// Returns `true` if the type is sized and a 1-ZST (meaning it has size 0 and alignment 1). pub fn is_1zst(&self) -> bool { - self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1 + self.is_sized() && self.size.bytes() == 0 && self.align.bytes() == 1 } /// Returns `true` if the type is a ZST and not unsized. diff --git a/compiler/rustc_arena/src/tests.rs b/compiler/rustc_arena/src/tests.rs index bfde8abd5893..eb9406d691b1 100644 --- a/compiler/rustc_arena/src/tests.rs +++ b/compiler/rustc_arena/src/tests.rs @@ -19,8 +19,8 @@ impl TypedArena { unsafe { // Clear the last chunk, which is partially filled. let mut chunks_borrow = self.chunks.borrow_mut(); - if let Some(mut last_chunk) = chunks_borrow.last_mut() { - self.clear_last_chunk(&mut last_chunk); + if let Some(last_chunk) = chunks_borrow.last_mut() { + self.clear_last_chunk(last_chunk); let len = chunks_borrow.len(); // If `T` is ZST, code below has no effect. for mut chunk in chunks_borrow.drain(..len - 1) { diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 984b280e81b5..8f4493182639 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -23,7 +23,7 @@ use std::{cmp, fmt}; pub use GenericArgs::*; pub use UnsafeSource::*; -pub use rustc_ast_ir::{Movability, Mutability, Pinnedness}; +pub use rustc_ast_ir::{FloatTy, IntTy, Movability, Mutability, Pinnedness, UintTy}; use rustc_data_structures::packed::Pu128; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::stack::ensure_sufficient_stack; @@ -35,7 +35,6 @@ use rustc_span::{ByteSymbol, DUMMY_SP, ErrorGuaranteed, Ident, Span, Symbol, kw, use thin_vec::{ThinVec, thin_vec}; pub use crate::format::*; -use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter}; use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream}; use crate::util::parser::{ExprPrecedence, Fixity}; @@ -115,8 +114,7 @@ impl PartialEq for Path { impl PartialEq<&[Symbol]> for Path { #[inline] fn eq(&self, names: &&[Symbol]) -> bool { - self.segments.len() == names.len() - && self.segments.iter().zip(names.iter()).all(|(s1, s2)| s1 == s2) + self.segments.iter().eq(*names) } } @@ -225,7 +223,7 @@ pub struct PathSegment { /// `Some` means that parameter list is supplied (`Path`) /// but it can be empty (`Path<>`). /// `P` is used as a size optimization for the common case with no parameters. - pub args: Option>, + pub args: Option>, } // Succeeds if the path segment is arg-free and matches the given symbol. @@ -286,7 +284,7 @@ pub enum GenericArg { /// `'a` in `Foo<'a>`. Lifetime(#[visitable(extra = LifetimeCtxt::GenericArg)] Lifetime), /// `Bar` in `Foo`. - Type(P), + Type(Box), /// `1` in `Foo<1>`. Const(AnonConst), } @@ -328,15 +326,15 @@ impl AngleBracketedArg { } } -impl From for P { +impl From for Box { fn from(val: AngleBracketedArgs) -> Self { - P(GenericArgs::AngleBracketed(val)) + Box::new(GenericArgs::AngleBracketed(val)) } } -impl From for P { +impl From for Box { fn from(val: ParenthesizedArgs) -> Self { - P(GenericArgs::Parenthesized(val)) + Box::new(GenericArgs::Parenthesized(val)) } } @@ -350,7 +348,7 @@ pub struct ParenthesizedArgs { pub span: Span, /// `(A, B)` - pub inputs: ThinVec>, + pub inputs: ThinVec>, /// ```text /// Foo(A, B) -> C @@ -435,10 +433,10 @@ pub enum GenericParamKind { /// A lifetime definition (e.g., `'a: 'b + 'c + 'd`). Lifetime, Type { - default: Option>, + default: Option>, }, Const { - ty: P, + ty: Box, /// Span of the whole parameter definition, including default. span: Span, /// Optional default value for the const generic param. @@ -526,7 +524,7 @@ pub struct WhereBoundPredicate { /// Any generics from a `for` binding. pub bound_generic_params: ThinVec, /// The type being bounded. - pub bounded_ty: P, + pub bounded_ty: Box, /// Trait and lifetime bounds (`Clone + Send + 'static`). #[visitable(extra = BoundKind::Bound)] pub bounds: GenericBounds, @@ -548,8 +546,8 @@ pub struct WhereRegionPredicate { /// E.g., `T = int`. #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub struct WhereEqPredicate { - pub lhs_ty: P, - pub rhs_ty: P, + pub lhs_ty: Box, + pub rhs_ty: Box, } #[derive(Clone, Encodable, Decodable, Debug, Walkable)] @@ -558,7 +556,7 @@ pub struct Crate { /// expansion placeholders or an unassigned value (`DUMMY_NODE_ID`) before that. pub id: NodeId, pub attrs: AttrVec, - pub items: ThinVec>, + pub items: ThinVec>, pub spans: ModSpans, pub is_placeholder: bool, } @@ -638,7 +636,7 @@ pub struct Pat { impl Pat { /// Attempt reparsing the pattern as a type. /// This is intended for use by diagnostics. - pub fn to_ty(&self) -> Option> { + pub fn to_ty(&self) -> Option> { let kind = match &self.kind { PatKind::Missing => unreachable!(), // In a type expression `_` is an inference variable. @@ -671,7 +669,7 @@ impl Pat { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) + Some(Box::new(Ty { kind, id: self.id, span: self.span, tokens: None })) } /// Walk top-down and call `it` in each place where a pattern occurs @@ -764,8 +762,8 @@ impl Pat { } } -impl From> for Pat { - fn from(value: P) -> Self { +impl From> for Pat { + fn from(value: Box) -> Self { *value } } @@ -780,7 +778,7 @@ pub struct PatField { /// The identifier for the field. pub ident: Ident, /// The pattern the field is destructured to. - pub pat: P, + pub pat: Box, pub is_shorthand: bool, pub attrs: AttrVec, pub id: NodeId, @@ -791,14 +789,14 @@ pub struct PatField { #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[derive(Encodable, Decodable, HashStable_Generic, Walkable)] pub enum ByRef { - Yes(Mutability), + Yes(Pinnedness, Mutability), No, } impl ByRef { #[must_use] pub fn cap_ref_mutability(mut self, mutbl: Mutability) -> Self { - if let ByRef::Yes(old_mutbl) = &mut self { + if let ByRef::Yes(_, old_mutbl) = &mut self { *old_mutbl = cmp::min(*old_mutbl, mutbl); } self @@ -816,20 +814,33 @@ pub struct BindingMode(pub ByRef, pub Mutability); impl BindingMode { pub const NONE: Self = Self(ByRef::No, Mutability::Not); - pub const REF: Self = Self(ByRef::Yes(Mutability::Not), Mutability::Not); + pub const REF: Self = Self(ByRef::Yes(Pinnedness::Not, Mutability::Not), Mutability::Not); + pub const REF_PIN: Self = + Self(ByRef::Yes(Pinnedness::Pinned, Mutability::Not), Mutability::Not); pub const MUT: Self = Self(ByRef::No, Mutability::Mut); - pub const REF_MUT: Self = Self(ByRef::Yes(Mutability::Mut), Mutability::Not); - pub const MUT_REF: Self = Self(ByRef::Yes(Mutability::Not), Mutability::Mut); - pub const MUT_REF_MUT: Self = Self(ByRef::Yes(Mutability::Mut), Mutability::Mut); + pub const REF_MUT: Self = Self(ByRef::Yes(Pinnedness::Not, Mutability::Mut), Mutability::Not); + pub const REF_PIN_MUT: Self = + Self(ByRef::Yes(Pinnedness::Pinned, Mutability::Mut), Mutability::Not); + pub const MUT_REF: Self = Self(ByRef::Yes(Pinnedness::Not, Mutability::Not), Mutability::Mut); + pub const MUT_REF_PIN: Self = + Self(ByRef::Yes(Pinnedness::Pinned, Mutability::Not), Mutability::Mut); + pub const MUT_REF_MUT: Self = + Self(ByRef::Yes(Pinnedness::Not, Mutability::Mut), Mutability::Mut); + pub const MUT_REF_PIN_MUT: Self = + Self(ByRef::Yes(Pinnedness::Pinned, Mutability::Mut), Mutability::Mut); pub fn prefix_str(self) -> &'static str { match self { Self::NONE => "", Self::REF => "ref ", + Self::REF_PIN => "ref pin const ", Self::MUT => "mut ", Self::REF_MUT => "ref mut ", + Self::REF_PIN_MUT => "ref pin mut ", Self::MUT_REF => "mut ref ", + Self::MUT_REF_PIN => "mut ref pin ", Self::MUT_REF_MUT => "mut ref mut ", + Self::MUT_REF_PIN_MUT => "mut ref pin mut ", } } } @@ -865,44 +876,44 @@ pub enum PatKind { /// or a unit struct/variant pattern, or a const pattern (in the last two cases the third /// field must be `None`). Disambiguation cannot be done with parser alone, so it happens /// during name resolution. - Ident(BindingMode, Ident, Option>), + Ident(BindingMode, Ident, Option>), /// A struct or struct variant pattern (e.g., `Variant {x, y, ..}`). - Struct(Option>, Path, ThinVec, PatFieldsRest), + Struct(Option>, Path, ThinVec, PatFieldsRest), /// A tuple struct/variant pattern (`Variant(x, y, .., z)`). - TupleStruct(Option>, Path, ThinVec>), + TupleStruct(Option>, Path, ThinVec), /// An or-pattern `A | B | C`. /// Invariant: `pats.len() >= 2`. - Or(ThinVec>), + Or(ThinVec), /// A possibly qualified path pattern. /// Unqualified path patterns `A::B::C` can legally refer to variants, structs, constants /// or associated constants. Qualified path patterns `::B::C`/`::B::C` can /// only legally refer to associated constants. - Path(Option>, Path), + Path(Option>, Path), /// A tuple pattern (`(a, b)`). - Tuple(ThinVec>), + Tuple(ThinVec), /// A `box` pattern. - Box(P), + Box(Box), /// A `deref` pattern (currently `deref!()` macro-based syntax). - Deref(P), + Deref(Box), /// A reference pattern (e.g., `&mut (a, b)`). - Ref(P, Mutability), + Ref(Box, Mutability), /// A literal, const block or path. - Expr(P), + Expr(Box), /// A range pattern (e.g., `1...2`, `1..2`, `1..`, `..2`, `1..=2`, `..=2`). - Range(Option>, Option>, Spanned), + Range(Option>, Option>, Spanned), /// A slice pattern `[a, b, c]`. - Slice(ThinVec>), + Slice(ThinVec), /// A rest pattern `..`. /// @@ -922,13 +933,13 @@ pub enum PatKind { Never, /// A guard pattern (e.g., `x if guard(x)`). - Guard(P, P), + Guard(Box, Box), /// Parentheses in patterns used for grouping (i.e., `(PAT)`). - Paren(P), + Paren(Box), /// A macro pattern; pre-expansion. - MacCall(P), + MacCall(Box), /// Placeholder for a pattern that wasn't syntactically well formed in some way. Err(ErrorGuaranteed), @@ -938,7 +949,7 @@ pub enum PatKind { #[derive(Clone, Copy, Encodable, Decodable, Debug, PartialEq, Walkable)] pub enum PatFieldsRest { /// `module::StructName { field, ..}` - Rest, + Rest(Span), /// `module::StructName { field, syntax error }` Recovered(ErrorGuaranteed), /// `module::StructName { field }` @@ -1223,22 +1234,22 @@ impl Stmt { #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub enum StmtKind { /// A local (let) binding. - Let(P), + Let(Box), /// An item definition. - Item(P), + Item(Box), /// Expr without trailing semi-colon. - Expr(P), + Expr(Box), /// Expr with a trailing semi-colon. - Semi(P), + Semi(Box), /// Just a trailing semi-colon. Empty, /// Macro. - MacCall(P), + MacCall(Box), } #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub struct MacCallStmt { - pub mac: P, + pub mac: Box, pub style: MacStmtStyle, pub attrs: AttrVec, pub tokens: Option, @@ -1262,8 +1273,8 @@ pub enum MacStmtStyle { pub struct Local { pub id: NodeId, pub super_: Option, - pub pat: P, - pub ty: Option>, + pub pat: Box, + pub ty: Option>, pub kind: LocalKind, pub span: Span, pub colon_sp: Option, @@ -1278,10 +1289,10 @@ pub enum LocalKind { Decl, /// Local declaration with an initializer. /// Example: `let x = y;` - Init(P), + Init(Box), /// Local declaration with an initializer and an `else` clause. /// Example: `let Some(x) = y else { return };` - InitElse(P, P), + InitElse(Box, Box), } impl LocalKind { @@ -1315,11 +1326,11 @@ impl LocalKind { pub struct Arm { pub attrs: AttrVec, /// Match arm pattern, e.g. `10` in `match foo { 10 => {}, _ => {} }`. - pub pat: P, + pub pat: Box, /// Match arm guard, e.g. `n > 10` in `match foo { n if n > 10 => {}, _ => {} }`. - pub guard: Option>, + pub guard: Option>, /// Match arm body. Omitted if the pattern is a never pattern. - pub body: Option>, + pub body: Option>, pub span: Span, pub id: NodeId, pub is_placeholder: bool, @@ -1332,7 +1343,7 @@ pub struct ExprField { pub id: NodeId, pub span: Span, pub ident: Ident, - pub expr: P, + pub expr: Box, pub is_shorthand: bool, pub is_placeholder: bool, } @@ -1357,7 +1368,7 @@ pub enum UnsafeSource { #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub struct AnonConst { pub id: NodeId, - pub value: P, + pub value: Box, } /// An expression. @@ -1469,7 +1480,7 @@ impl Expr { } /// Attempts to reparse as `Ty` (for diagnostic purposes). - pub fn to_ty(&self) -> Option> { + pub fn to_ty(&self) -> Option> { let kind = match &self.kind { // Trivial conversions. ExprKind::Path(qself, path) => TyKind::Path(qself.clone(), path.clone()), @@ -1511,7 +1522,7 @@ impl Expr { _ => return None, }; - Some(P(Ty { kind, id: self.id, span: self.span, tokens: None })) + Some(Box::new(Ty { kind, id: self.id, span: self.span, tokens: None })) } pub fn precedence(&self) -> ExprPrecedence { @@ -1632,8 +1643,8 @@ impl Expr { } } -impl From> for Expr { - fn from(value: P) -> Self { +impl From> for Expr { + fn from(value: Box) -> Self { *value } } @@ -1645,8 +1656,8 @@ pub struct Closure { pub constness: Const, pub coroutine_kind: Option, pub movability: Movability, - pub fn_decl: P, - pub body: P, + pub fn_decl: Box, + pub body: Box, /// The span of the declaration block: 'move |...| -> ...' pub fn_decl_span: Span, /// The span of the argument block `|...|` @@ -1677,9 +1688,9 @@ pub struct MethodCall { /// The method name and its generic arguments, e.g. `foo::`. pub seg: PathSegment, /// The receiver, e.g. `x`. - pub receiver: P, + pub receiver: Box, /// The arguments, e.g. `a, b, c`. - pub args: ThinVec>, + pub args: ThinVec>, /// The span of the function, without the dot and receiver e.g. `foo::(a, b, c)`. pub span: Span, @@ -1688,7 +1699,7 @@ pub struct MethodCall { #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub enum StructRest { /// `..x`. - Base(P), + Base(Box), /// `..`. Rest(Span), /// No trailing `..` or expression. @@ -1697,7 +1708,7 @@ pub enum StructRest { #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub struct StructExpr { - pub qself: Option>, + pub qself: Option>, pub path: Path, pub fields: ThinVec, pub rest: StructRest, @@ -1707,7 +1718,7 @@ pub struct StructExpr { #[derive(Clone, Encodable, Decodable, Debug)] pub enum ExprKind { /// An array (e.g, `[a, b, c, d]`). - Array(ThinVec>), + Array(ThinVec>), /// Allow anonymous constants from an inline `const` block. ConstBlock(AnonConst), /// A function call. @@ -1716,90 +1727,90 @@ pub enum ExprKind { /// and the second field is the list of arguments. /// This also represents calling the constructor of /// tuple-like ADTs such as tuple structs and enum variants. - Call(P, ThinVec>), + Call(Box, ThinVec>), /// A method call (e.g., `x.foo::(a, b, c)`). MethodCall(Box), /// A tuple (e.g., `(a, b, c, d)`). - Tup(ThinVec>), + Tup(ThinVec>), /// A binary operation (e.g., `a + b`, `a * b`). - Binary(BinOp, P, P), + Binary(BinOp, Box, Box), /// A unary operation (e.g., `!x`, `*x`). - Unary(UnOp, P), + Unary(UnOp, Box), /// A literal (e.g., `1`, `"foo"`). Lit(token::Lit), /// A cast (e.g., `foo as f64`). - Cast(P, P), + Cast(Box, Box), /// A type ascription (e.g., `builtin # type_ascribe(42, usize)`). /// /// Usually not written directly in user code but /// indirectly via the macro `type_ascribe!(...)`. - Type(P, P), + Type(Box, Box), /// A `let pat = expr` expression that is only semantically allowed in the condition /// of `if` / `while` expressions. (e.g., `if let 0 = x { .. }`). /// /// `Span` represents the whole `let pat = expr` statement. - Let(P, P, Span, Recovered), + Let(Box, Box, Span, Recovered), /// An `if` block, with an optional `else` block. /// /// `if expr { block } else { expr }` /// /// If present, the "else" expr is always `ExprKind::Block` (for `else`) or /// `ExprKind::If` (for `else if`). - If(P, P, Option>), + If(Box, Box, Option>), /// A while loop, with an optional label. /// /// `'label: while expr { block }` - While(P, P, Option Foo { .. }` or `impl Trait for Foo { .. }`. - Impl(Box), + Impl(Impl), /// A macro invocation. /// /// E.g., `foo!(..)`. - MacCall(P), + MacCall(Box), /// A macro definition. MacroDef(Ident, MacroDef), /// A single delegation item (`reuse`). @@ -3922,7 +3854,7 @@ impl ItemKind { | ItemKind::Struct(ident, ..) | ItemKind::Union(ident, ..) | ItemKind::Trait(box Trait { ident, .. }) - | ItemKind::TraitAlias(ident, ..) + | ItemKind::TraitAlias(box TraitAlias { ident, .. }) | ItemKind::MacroDef(ident, _) | ItemKind::Delegation(box Delegation { ident, .. }) => Some(ident), @@ -3979,8 +3911,8 @@ impl ItemKind { | Self::Struct(_, generics, _) | Self::Union(_, generics, _) | Self::Trait(box Trait { generics, .. }) - | Self::TraitAlias(_, generics, _) - | Self::Impl(box Impl { generics, .. }) => Some(generics), + | Self::TraitAlias(box TraitAlias { generics, .. }) + | Self::Impl(Impl { generics, .. }) => Some(generics), _ => None, } } @@ -4007,7 +3939,7 @@ pub enum AssocItemKind { /// An associated type. Type(Box), /// A macro expanding to associated items. - MacCall(P), + MacCall(Box), /// An associated delegation item. Delegation(Box), /// An associated list or glob delegation item. @@ -4077,7 +4009,7 @@ pub enum ForeignItemKind { /// A foreign type. TyAlias(Box), /// A macro expanding to foreign items. - MacCall(P), + MacCall(Box), } impl ForeignItemKind { @@ -4140,19 +4072,20 @@ mod size_asserts { static_assert_size!(GenericArg, 24); static_assert_size!(GenericBound, 88); static_assert_size!(Generics, 40); - static_assert_size!(Impl, 136); - static_assert_size!(Item, 144); - static_assert_size!(ItemKind, 80); + static_assert_size!(Impl, 64); + static_assert_size!(Item, 136); + static_assert_size!(ItemKind, 72); static_assert_size!(LitKind, 24); static_assert_size!(Local, 96); static_assert_size!(MetaItemLit, 40); static_assert_size!(Param, 40); - static_assert_size!(Pat, 72); - static_assert_size!(PatKind, 48); + static_assert_size!(Pat, 80); + static_assert_size!(PatKind, 56); static_assert_size!(Path, 24); static_assert_size!(PathSegment, 24); static_assert_size!(Stmt, 32); static_assert_size!(StmtKind, 16); + static_assert_size!(TraitImplHeader, 80); static_assert_size!(Ty, 64); static_assert_size!(TyKind, 40); // tidy-alphabetical-end diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index 9d91f41d6c79..3d2477e5f033 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -5,7 +5,6 @@ use std::fmt; use std::marker::PhantomData; -use crate::ptr::P; use crate::tokenstream::LazyAttrTokenStream; use crate::{ Arm, AssocItem, AttrItem, AttrKind, AttrVec, Attribute, Block, Crate, Expr, ExprField, @@ -53,7 +52,7 @@ impl_has_node_id!( WherePredicate, ); -impl HasNodeId for P { +impl HasNodeId for Box { fn node_id(&self) -> NodeId { (**self).node_id() } @@ -119,7 +118,7 @@ impl HasTokens for Option { } } -impl HasTokens for P { +impl HasTokens for Box { fn tokens(&self) -> Option<&LazyAttrTokenStream> { (**self).tokens() } @@ -245,7 +244,7 @@ impl_has_attrs!( ); impl_has_attrs_none!(Attribute, AttrItem, Block, Pat, Path, Ty, Visibility); -impl HasAttrs for P { +impl HasAttrs for Box { const SUPPORTS_CUSTOM_INNER_ATTRS: bool = T::SUPPORTS_CUSTOM_INNER_ATTRS; fn attrs(&self) -> &[Attribute] { (**self).attrs() @@ -322,8 +321,8 @@ impl AstNodeWrapper { } // FIXME: remove after `stmt_expr_attributes` is stabilized. -impl From, Tag>> for AstNodeWrapper { - fn from(value: AstNodeWrapper, Tag>) -> Self { +impl From, Tag>> for AstNodeWrapper { + fn from(value: AstNodeWrapper, Tag>) -> Self { AstNodeWrapper { wrapped: *value.wrapped, tag: value.tag } } } diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 4348a4bb120e..901b645b8c4e 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -13,7 +13,6 @@ use crate::ast::{ Expr, ExprKind, LitKind, MetaItem, MetaItemInner, MetaItemKind, MetaItemLit, NormalAttr, Path, PathSegment, Safety, }; -use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, InvisibleOrigin, MetaVarKind, Token}; use crate::tokenstream::{ DelimSpan, LazyAttrTokenStream, Spacing, TokenStream, TokenStreamIter, TokenTree, @@ -87,10 +86,10 @@ impl AttributeExt for Attribute { /// Returns `true` if it is a sugared doc comment (`///` or `//!` for example). /// So `#[doc = "doc"]` (which is a doc comment) and `#[doc(...)]` (which is not /// a doc comment) will return `false`. - fn is_doc_comment(&self) -> bool { + fn is_doc_comment(&self) -> Option { match self.kind { - AttrKind::Normal(..) => false, - AttrKind::DocComment(..) => true, + AttrKind::Normal(..) => None, + AttrKind::DocComment(..) => Some(self.span), } } @@ -660,7 +659,7 @@ pub fn mk_attr_from_item( span: Span, ) -> Attribute { Attribute { - kind: AttrKind::Normal(P(NormalAttr { item, tokens })), + kind: AttrKind::Normal(Box::new(NormalAttr { item, tokens })), id: g.mk_attr_id(), style, span, @@ -710,7 +709,7 @@ pub fn mk_attr_name_value_str( span: Span, ) -> Attribute { let lit = token::Lit::new(token::Str, escape_string_symbol(val), None); - let expr = P(Expr { + let expr = Box::new(Expr { id: DUMMY_NODE_ID, kind: ExprKind::Lit(lit), span, @@ -777,7 +776,7 @@ pub trait AttributeExt: Debug { /// Returns `true` if it is a sugared doc comment (`///` or `//!` for example). /// So `#[doc = "doc"]` (which is a doc comment) and `#[doc(...)]` (which is not /// a doc comment) will return `false`. - fn is_doc_comment(&self) -> bool; + fn is_doc_comment(&self) -> Option; #[inline] fn has_name(&self, name: Symbol) -> bool { @@ -864,8 +863,9 @@ impl Attribute { AttributeExt::path_matches(self, name) } + // on ast attributes we return a bool since that's what most code already expects pub fn is_doc_comment(&self) -> bool { - AttributeExt::is_doc_comment(self) + AttributeExt::is_doc_comment(self).is_some() } #[inline] diff --git a/compiler/rustc_ast/src/entry.rs b/compiler/rustc_ast/src/entry.rs index 12cbb3b2a15f..ae0cc7350a78 100644 --- a/compiler/rustc_ast/src/entry.rs +++ b/compiler/rustc_ast/src/entry.rs @@ -1,7 +1,5 @@ use rustc_span::{Symbol, sym}; -use crate::attr::{self, AttributeExt}; - #[derive(Debug)] pub enum EntryPointType { /// This function is not an entrypoint. @@ -30,11 +28,11 @@ pub enum EntryPointType { } pub fn entry_point_type( - attrs: &[impl AttributeExt], + has_rustc_main: bool, at_root: bool, name: Option, ) -> EntryPointType { - if attr::contains_name(attrs, sym::rustc_main) { + if has_rustc_main { EntryPointType::RustcMainAttr } else if let Some(name) = name && name == sym::main diff --git a/compiler/rustc_ast/src/expand/allocator.rs b/compiler/rustc_ast/src/expand/allocator.rs index 7dee2ed17b4b..332ad50d927f 100644 --- a/compiler/rustc_ast/src/expand/allocator.rs +++ b/compiler/rustc_ast/src/expand/allocator.rs @@ -3,7 +3,9 @@ use rustc_span::{Symbol, sym}; #[derive(Clone, Debug, Copy, Eq, PartialEq, HashStable_Generic)] pub enum AllocatorKind { + /// Use `#[global_allocator]` as global allocator. Global, + /// Use the default implementation in libstd as global allocator. Default, } @@ -15,25 +17,37 @@ pub fn default_fn_name(base: Symbol) -> String { format!("__rdl_{base}") } -pub fn alloc_error_handler_name(alloc_error_handler_kind: AllocatorKind) -> &'static str { - match alloc_error_handler_kind { - AllocatorKind::Global => "__rg_oom", - AllocatorKind::Default => "__rdl_oom", - } -} - +pub const ALLOC_ERROR_HANDLER: Symbol = sym::alloc_error_handler; pub const NO_ALLOC_SHIM_IS_UNSTABLE: &str = "__rust_no_alloc_shim_is_unstable_v2"; +/// Argument or return type for methods in the allocator shim +#[derive(Copy, Clone)] pub enum AllocatorTy { Layout, + Never, Ptr, ResultPtr, Unit, Usize, } +/// Some allocator methods are known to the compiler: they act more like +/// intrinsics/language primitives than library-defined functions. +/// FIXME: ideally this would be derived from attributes like `#[rustc_allocator]`, +/// so we don't have two sources of truth. +#[derive(Copy, Clone, Debug)] +pub enum SpecialAllocatorMethod { + Alloc, + AllocZeroed, + Dealloc, + Realloc, +} + +/// A method that will be codegened in the allocator shim. +#[derive(Copy, Clone)] pub struct AllocatorMethod { pub name: Symbol, + pub special: Option, pub inputs: &'static [AllocatorMethodInput], pub output: AllocatorTy, } @@ -46,11 +60,13 @@ pub struct AllocatorMethodInput { pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ AllocatorMethod { name: sym::alloc, + special: Some(SpecialAllocatorMethod::Alloc), inputs: &[AllocatorMethodInput { name: "layout", ty: AllocatorTy::Layout }], output: AllocatorTy::ResultPtr, }, AllocatorMethod { name: sym::dealloc, + special: Some(SpecialAllocatorMethod::Dealloc), inputs: &[ AllocatorMethodInput { name: "ptr", ty: AllocatorTy::Ptr }, AllocatorMethodInput { name: "layout", ty: AllocatorTy::Layout }, @@ -59,6 +75,7 @@ pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ }, AllocatorMethod { name: sym::realloc, + special: Some(SpecialAllocatorMethod::Realloc), inputs: &[ AllocatorMethodInput { name: "ptr", ty: AllocatorTy::Ptr }, AllocatorMethodInput { name: "layout", ty: AllocatorTy::Layout }, @@ -68,6 +85,7 @@ pub static ALLOCATOR_METHODS: &[AllocatorMethod] = &[ }, AllocatorMethod { name: sym::alloc_zeroed, + special: Some(SpecialAllocatorMethod::AllocZeroed), inputs: &[AllocatorMethodInput { name: "layout", ty: AllocatorTy::Layout }], output: AllocatorTy::ResultPtr, }, diff --git a/compiler/rustc_ast/src/expand/autodiff_attrs.rs b/compiler/rustc_ast/src/expand/autodiff_attrs.rs index 2f918faaf752..90f15753e99c 100644 --- a/compiler/rustc_ast/src/expand/autodiff_attrs.rs +++ b/compiler/rustc_ast/src/expand/autodiff_attrs.rs @@ -6,8 +6,8 @@ use std::fmt::{self, Display, Formatter}; use std::str::FromStr; +use crate::expand::typetree::TypeTree; use crate::expand::{Decodable, Encodable, HashStable_Generic}; -use crate::ptr::P; use crate::{Ty, TyKind}; /// Forward and Reverse Mode are well known names for automatic differentiation implementations. @@ -85,6 +85,8 @@ pub struct AutoDiffItem { /// The name of the function being generated pub target: String, pub attrs: AutoDiffAttrs, + pub inputs: Vec, + pub output: TypeTree, } #[derive(Clone, Eq, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] @@ -162,7 +164,7 @@ pub fn valid_ret_activity(mode: DiffMode, activity: DiffActivity) -> bool { /// since Duplicated expects a mutable ref/ptr and we would thus end up with a shadow value /// who is an indirect type, which doesn't match the primal scalar type. We can't prevent /// users here from marking scalars as Duplicated, due to type aliases. -pub fn valid_ty_for_activity(ty: &P, activity: DiffActivity) -> bool { +pub fn valid_ty_for_activity(ty: &Box, activity: DiffActivity) -> bool { use DiffActivity::*; // It's always allowed to mark something as Const, since we won't compute derivatives wrt. it. // Dual variants also support all types. @@ -276,14 +278,22 @@ impl AutoDiffAttrs { !matches!(self.mode, DiffMode::Error | DiffMode::Source) } - pub fn into_item(self, source: String, target: String) -> AutoDiffItem { - AutoDiffItem { source, target, attrs: self } + pub fn into_item( + self, + source: String, + target: String, + inputs: Vec, + output: TypeTree, + ) -> AutoDiffItem { + AutoDiffItem { source, target, inputs, output, attrs: self } } } impl fmt::Display for AutoDiffItem { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "Differentiating {} -> {}", self.source, self.target)?; - write!(f, " with attributes: {:?}", self.attrs) + write!(f, " with attributes: {:?}", self.attrs)?; + write!(f, " with inputs: {:?}", self.inputs)?; + write!(f, " with output: {:?}", self.output) } } diff --git a/compiler/rustc_ast/src/expand/typetree.rs b/compiler/rustc_ast/src/expand/typetree.rs index 9a2dd2e85e0d..e7b4f3aff413 100644 --- a/compiler/rustc_ast/src/expand/typetree.rs +++ b/compiler/rustc_ast/src/expand/typetree.rs @@ -31,6 +31,7 @@ pub enum Kind { Half, Float, Double, + F128, Unknown, } diff --git a/compiler/rustc_ast/src/format.rs b/compiler/rustc_ast/src/format.rs index c2a1de60a981..cadebb2254d9 100644 --- a/compiler/rustc_ast/src/format.rs +++ b/compiler/rustc_ast/src/format.rs @@ -3,7 +3,6 @@ use rustc_macros::{Decodable, Encodable, Walkable}; use rustc_span::{Ident, Span, Symbol}; use crate::Expr; -use crate::ptr::P; use crate::token::LitKind; // Definitions: @@ -147,7 +146,7 @@ impl FormatArguments { #[derive(Clone, Encodable, Decodable, Debug, Walkable)] pub struct FormatArgument { pub kind: FormatArgumentKind, - pub expr: P, + pub expr: Box, } #[derive(Clone, Encodable, Decodable, Debug, Walkable)] diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs index 896d1e1148a9..5fe218776e53 100644 --- a/compiler/rustc_ast/src/lib.rs +++ b/compiler/rustc_ast/src/lib.rs @@ -15,6 +15,7 @@ #![feature(associated_type_defaults)] #![feature(box_patterns)] #![feature(if_let_guard)] +#![feature(iter_order_by)] #![feature(macro_metavar_expr)] #![feature(rustdoc_internals)] #![recursion_limit = "256"] @@ -37,7 +38,6 @@ pub mod expand; pub mod format; pub mod mut_visit; pub mod node_id; -pub mod ptr; pub mod token; pub mod tokenstream; pub mod visit; diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 06708e2e703a..be8e1d22c9db 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -17,7 +17,6 @@ use smallvec::{SmallVec, smallvec}; use thin_vec::ThinVec; use crate::ast::*; -use crate::ptr::P; use crate::tokenstream::*; use crate::visit::{AssocCtxt, BoundKind, FnCtxt, LifetimeCtxt, VisitorResult, try_visit}; @@ -41,7 +40,7 @@ pub(crate) trait MutVisitable { fn visit_mut(&mut self, visitor: &mut V, extra: Self::Extra); } -impl MutVisitable for P +impl MutVisitable for Box where T: MutVisitable, { @@ -293,15 +292,15 @@ macro_rules! generate_flat_map_visitor_fns { } generate_flat_map_visitor_fns! { - visit_items, P, flat_map_item; - visit_foreign_items, P, flat_map_foreign_item; + visit_items, Box, flat_map_item; + visit_foreign_items, Box, flat_map_foreign_item; visit_generic_params, GenericParam, flat_map_generic_param; visit_stmts, Stmt, flat_map_stmt; - visit_exprs, P, filter_map_expr; + visit_exprs, Box, filter_map_expr; visit_expr_fields, ExprField, flat_map_expr_field; visit_pat_fields, PatField, flat_map_pat_field; visit_variants, Variant, flat_map_variant; - visit_assoc_items, P, flat_map_assoc_item, ctxt: AssocCtxt; + visit_assoc_items, Box, flat_map_assoc_item, ctxt: AssocCtxt; visit_where_predicates, WherePredicate, flat_map_where_predicate; visit_params, Param, flat_map_param; visit_field_defs, FieldDef, flat_map_field_def; @@ -333,12 +332,12 @@ generate_walk_flat_map_fns! { walk_flat_map_where_predicate(WherePredicate) => visit_where_predicate; walk_flat_map_field_def(FieldDef) => visit_field_def; walk_flat_map_expr_field(ExprField) => visit_expr_field; - walk_flat_map_item(P) => visit_item; - walk_flat_map_foreign_item(P) => visit_foreign_item; - walk_flat_map_assoc_item(P, ctxt: AssocCtxt) => visit_assoc_item; + walk_flat_map_item(Box) => visit_item; + walk_flat_map_foreign_item(Box) => visit_foreign_item; + walk_flat_map_assoc_item(Box, ctxt: AssocCtxt) => visit_assoc_item; } -pub fn walk_filter_map_expr(vis: &mut T, mut e: P) -> Option> { +pub fn walk_filter_map_expr(vis: &mut T, mut e: Box) -> Option> { vis.visit_expr(&mut e); Some(e) } diff --git a/compiler/rustc_ast/src/ptr.rs b/compiler/rustc_ast/src/ptr.rs deleted file mode 100644 index fffeab8bbca6..000000000000 --- a/compiler/rustc_ast/src/ptr.rs +++ /dev/null @@ -1,11 +0,0 @@ -/// A pointer type that uniquely owns a heap allocation of type T. -/// -/// This used to be its own type, but now it's just a typedef for `Box` and we are planning to -/// remove it soon. -pub type P = Box; - -/// Construct a `P` from a `T` value. -#[allow(non_snake_case)] -pub fn P(value: T) -> P { - Box::new(value) -} diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index fc816f2cb792..e1231312a2af 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -7,6 +7,7 @@ pub use NtPatKind::*; pub use TokenKind::*; use rustc_macros::{Decodable, Encodable, HashStable_Generic}; use rustc_span::edition::Edition; +use rustc_span::symbol::IdentPrintMode; use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym}; #[allow(clippy::useless_attribute)] // FIXME: following use of `hidden_glob_reexports` incorrectly triggers `useless_attribute` lint. #[allow(hidden_glob_reexports)] @@ -21,8 +22,7 @@ pub enum CommentKind { Block, } -// This type must not implement `Hash` due to the unusual `PartialEq` impl below. -#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)] +#[derive(Copy, Clone, PartialEq, Debug, Encodable, Decodable, HashStable_Generic)] pub enum InvisibleOrigin { // From the expansion of a metavariable in a declarative macro. MetaVar(MetaVarKind), @@ -44,20 +44,6 @@ impl InvisibleOrigin { } } -impl PartialEq for InvisibleOrigin { - #[inline] - fn eq(&self, _other: &InvisibleOrigin) -> bool { - // When we had AST-based nonterminals we couldn't compare them, and the - // old `Nonterminal` type had an `eq` that always returned false, - // resulting in this restriction: - // https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment - // This `eq` emulates that behaviour. We could consider lifting this - // restriction now but there are still cases involving invisible - // delimiters that make it harder than it first appears. - false - } -} - /// Annoyingly similar to `NonterminalKind`, but the slight differences are important. #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)] pub enum MetaVarKind { @@ -141,7 +127,8 @@ impl Delimiter { } } - // This exists because `InvisibleOrigin`s should be compared. It is only used for assertions. + // This exists because `InvisibleOrigin`s should not be compared. It is only used for + // assertions. pub fn eq_ignoring_invisible_origin(&self, other: &Delimiter) -> bool { match (self, other) { (Delimiter::Parenthesis, Delimiter::Parenthesis) => true, @@ -344,15 +331,24 @@ pub enum IdentIsRaw { Yes, } -impl From for IdentIsRaw { - fn from(b: bool) -> Self { - if b { Self::Yes } else { Self::No } +impl IdentIsRaw { + pub fn to_print_mode_ident(self) -> IdentPrintMode { + match self { + IdentIsRaw::No => IdentPrintMode::Normal, + IdentIsRaw::Yes => IdentPrintMode::RawIdent, + } + } + pub fn to_print_mode_lifetime(self) -> IdentPrintMode { + match self { + IdentIsRaw::No => IdentPrintMode::Normal, + IdentIsRaw::Yes => IdentPrintMode::RawLifetime, + } } } -impl From for bool { - fn from(is_raw: IdentIsRaw) -> bool { - matches!(is_raw, IdentIsRaw::Yes) +impl From for IdentIsRaw { + fn from(b: bool) -> Self { + if b { Self::Yes } else { Self::No } } } @@ -885,11 +881,11 @@ impl Token { } pub fn is_qpath_start(&self) -> bool { - self == &Lt || self == &Shl + matches!(self.kind, Lt | Shl) } pub fn is_path_start(&self) -> bool { - self == &PathSep + self.kind == PathSep || self.is_qpath_start() || matches!(self.is_metavar_seq(), Some(MetaVarKind::Path)) || self.is_path_segment_keyword() diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index e55399adfb85..4111182c3b7d 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -3,15 +3,6 @@ //! `TokenStream`s represent syntactic objects before they are converted into ASTs. //! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s, //! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens. -//! -//! ## Ownership -//! -//! `TokenStream`s are persistent data structures constructed as ropes with reference -//! counted-children. In general, this means that calling an operation on a `TokenStream` -//! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to -//! the original. This essentially coerces `TokenStream`s into "views" of their subparts, -//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking -//! ownership of the original. use std::borrow::Cow; use std::ops::Range; @@ -57,9 +48,7 @@ impl TokenTree { match (self, other) { (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind, (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => { - delim == delim2 - && tts.len() == tts2.len() - && tts.iter().zip(tts2.iter()).all(|(a, b)| a.eq_unspanned(b)) + delim == delim2 && tts.iter().eq_by(tts2.iter(), |a, b| a.eq_unspanned(b)) } _ => false, } @@ -99,17 +88,6 @@ impl TokenTree { } } -impl HashStable for TokenStream -where - CTX: crate::HashStableContext, -{ - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - for sub_tt in self.iter() { - sub_tt.hash_stable(hcx, hasher); - } - } -} - /// A lazy version of [`AttrTokenStream`], which defers creation of an actual /// `AttrTokenStream` until it is needed. #[derive(Clone)] @@ -556,10 +534,6 @@ pub struct AttrsTarget { pub tokens: LazyAttrTokenStream, } -/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. -#[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct TokenStream(pub(crate) Arc>); - /// Indicates whether a token can join with the following token to form a /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to /// guide pretty-printing, which is where the `JointHidden` value (which isn't @@ -620,58 +594,9 @@ pub enum Spacing { JointHidden, } -impl TokenStream { - /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` - /// separating the two arguments with a comma for diagnostic suggestions. - pub fn add_comma(&self) -> Option<(TokenStream, Span)> { - // Used to suggest if a user writes `foo!(a b);` - let mut suggestion = None; - let mut iter = self.0.iter().enumerate().peekable(); - while let Some((pos, ts)) = iter.next() { - if let Some((_, next)) = iter.peek() { - let sp = match (&ts, &next) { - (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, - ( - TokenTree::Token(token_left, Spacing::Alone), - TokenTree::Token(token_right, _), - ) if (token_left.is_non_reserved_ident() || token_left.is_lit()) - && (token_right.is_non_reserved_ident() || token_right.is_lit()) => - { - token_left.span - } - (TokenTree::Delimited(sp, ..), _) => sp.entire(), - _ => continue, - }; - let sp = sp.shrink_to_hi(); - let comma = TokenTree::token_alone(token::Comma, sp); - suggestion = Some((pos, comma, sp)); - } - } - if let Some((pos, comma, sp)) = suggestion { - let mut new_stream = Vec::with_capacity(self.0.len() + 1); - let parts = self.0.split_at(pos + 1); - new_stream.extend_from_slice(parts.0); - new_stream.push(comma); - new_stream.extend_from_slice(parts.1); - return Some((TokenStream::new(new_stream), sp)); - } - None - } -} - -impl FromIterator for TokenStream { - fn from_iter>(iter: I) -> Self { - TokenStream::new(iter.into_iter().collect::>()) - } -} - -impl Eq for TokenStream {} - -impl PartialEq for TokenStream { - fn eq(&self, other: &TokenStream) -> bool { - self.iter().eq(other.iter()) - } -} +/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. +#[derive(Clone, Debug, Default, Encodable, Decodable)] +pub struct TokenStream(pub(crate) Arc>); impl TokenStream { pub fn new(tts: Vec) -> TokenStream { @@ -847,6 +772,68 @@ impl TokenStream { } } } + + /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` + /// separating the two arguments with a comma for diagnostic suggestions. + pub fn add_comma(&self) -> Option<(TokenStream, Span)> { + // Used to suggest if a user writes `foo!(a b);` + let mut suggestion = None; + let mut iter = self.0.iter().enumerate().peekable(); + while let Some((pos, ts)) = iter.next() { + if let Some((_, next)) = iter.peek() { + let sp = match (&ts, &next) { + (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue, + ( + TokenTree::Token(token_left, Spacing::Alone), + TokenTree::Token(token_right, _), + ) if (token_left.is_non_reserved_ident() || token_left.is_lit()) + && (token_right.is_non_reserved_ident() || token_right.is_lit()) => + { + token_left.span + } + (TokenTree::Delimited(sp, ..), _) => sp.entire(), + _ => continue, + }; + let sp = sp.shrink_to_hi(); + let comma = TokenTree::token_alone(token::Comma, sp); + suggestion = Some((pos, comma, sp)); + } + } + if let Some((pos, comma, sp)) = suggestion { + let mut new_stream = Vec::with_capacity(self.0.len() + 1); + let parts = self.0.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::new(new_stream), sp)); + } + None + } +} + +impl PartialEq for TokenStream { + fn eq(&self, other: &TokenStream) -> bool { + self.iter().eq(other.iter()) + } +} + +impl Eq for TokenStream {} + +impl FromIterator for TokenStream { + fn from_iter>(iter: I) -> Self { + TokenStream::new(iter.into_iter().collect::>()) + } +} + +impl HashStable for TokenStream +where + CTX: crate::HashStableContext, +{ + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + for sub_tt in self.iter() { + sub_tt.hash_stable(hcx, hasher); + } + } } #[derive(Clone)] @@ -907,6 +894,12 @@ impl TokenTreeCursor { pub fn bump(&mut self) { self.index += 1; } + + // For skipping ahead in rare circumstances. + #[inline] + pub fn bump_to_end(&mut self) { + self.index = self.stream.len(); + } } /// A `TokenStream` cursor that produces `Token`s. It's a bit odd that diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs index 2dfd695d8802..e9cf3cf07372 100644 --- a/compiler/rustc_ast/src/util/literal.rs +++ b/compiler/rustc_ast/src/util/literal.rs @@ -190,15 +190,15 @@ impl fmt::Display for LitKind { LitKind::Int(n, ty) => { write!(f, "{n}")?; match ty { - ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?, - ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?, + ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name_str())?, + ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name_str())?, ast::LitIntType::Unsuffixed => {} } } LitKind::Float(symbol, ty) => { write!(f, "{symbol}")?; match ty { - ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?, + ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name_str())?, ast::LitFloatType::Unsuffixed => {} } } diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs index ab15cb28fa12..99490be64175 100644 --- a/compiler/rustc_ast/src/visit.rs +++ b/compiler/rustc_ast/src/visit.rs @@ -20,7 +20,6 @@ use rustc_span::{Ident, Span, Symbol}; use thin_vec::ThinVec; use crate::ast::*; -use crate::ptr::P; use crate::tokenstream::DelimSpan; #[derive(Copy, Clone, Debug, PartialEq)] @@ -82,7 +81,7 @@ pub(crate) trait Visitable<'a, V: Visitor<'a>> { fn visit(&'a self, visitor: &mut V, extra: Self::Extra) -> V::Result; } -impl<'a, V: Visitor<'a>, T: ?Sized> Visitable<'a, V> for P +impl<'a, V: Visitor<'a>, T: ?Sized> Visitable<'a, V> for Box where T: Visitable<'a, V>, { @@ -322,7 +321,7 @@ macro_rules! common_visitor_and_walkers { Fn(FnCtxt, &'a $($mut)? Visibility, &'a $($mut)? Fn), /// E.g., `|x, y| body`. - Closure(&'a $($mut)? ClosureBinder, &'a $($mut)? Option, &'a $($mut)? P, &'a $($mut)? P), + Closure(&'a $($mut)? ClosureBinder, &'a $($mut)? Option, &'a $($mut)? Box, &'a $($mut)? Box), } impl<'a> FnKind<'a> { @@ -369,6 +368,7 @@ macro_rules! common_visitor_and_walkers { crate::tokenstream::TokenStream, Movability, Mutability, + Pinnedness, Result<(), rustc_span::ErrorGuaranteed>, rustc_data_structures::fx::FxHashMap, rustc_span::ErrorGuaranteed, @@ -390,9 +390,9 @@ macro_rules! common_visitor_and_walkers { ThinVec<(NodeId, Path)>, ThinVec, ThinVec, - ThinVec>, - ThinVec>, - ThinVec>, + ThinVec, + ThinVec>, + ThinVec, ); // This macro generates `impl Visitable` and `impl MutVisitable` that forward to `Walkable` @@ -472,8 +472,6 @@ macro_rules! common_visitor_and_walkers { TraitBoundModifiers, TraitObjectSyntax, TyAlias, - TyAliasWhereClause, - TyAliasWhereClauses, TyKind, TyPatKind, UnOp, @@ -676,11 +674,11 @@ macro_rules! common_visitor_and_walkers { // Do nothing. } - fn flat_map_foreign_item(&mut self, ni: P) -> SmallVec<[P; 1]> { + fn flat_map_foreign_item(&mut self, ni: Box) -> SmallVec<[Box; 1]> { walk_flat_map_foreign_item(self, ni) } - fn flat_map_item(&mut self, i: P) -> SmallVec<[P; 1]> { + fn flat_map_item(&mut self, i: Box) -> SmallVec<[Box; 1]> { walk_flat_map_item(self, i) } @@ -690,9 +688,9 @@ macro_rules! common_visitor_and_walkers { fn flat_map_assoc_item( &mut self, - i: P, + i: Box, ctxt: AssocCtxt, - ) -> SmallVec<[P; 1]> { + ) -> SmallVec<[Box; 1]> { walk_flat_map_assoc_item(self, i, ctxt) } @@ -704,7 +702,7 @@ macro_rules! common_visitor_and_walkers { walk_flat_map_arm(self, arm) } - fn filter_map_expr(&mut self, e: P) -> Option> { + fn filter_map_expr(&mut self, e: Box) -> Option> { walk_filter_map_expr(self, e) } @@ -836,8 +834,8 @@ macro_rules! common_visitor_and_walkers { visit_visitable!($($mut)? vis, impl_), ItemKind::Trait(trait_) => visit_visitable!($($mut)? vis, trait_), - ItemKind::TraitAlias(ident, generics, bounds) => { - visit_visitable!($($mut)? vis, ident, generics); + ItemKind::TraitAlias(box TraitAlias { constness, ident, generics, bounds}) => { + visit_visitable!($($mut)? vis, constness, ident, generics); visit_visitable_with!($($mut)? vis, bounds, BoundKind::Bound) } ItemKind::MacCall(m) => @@ -930,8 +928,13 @@ macro_rules! common_visitor_and_walkers { } impl_walkable!(|&$($mut)? $($lt)? self: Impl, vis: &mut V| { - let Impl { defaultness, safety, generics, constness, polarity, of_trait, self_ty, items } = self; - visit_visitable!($($mut)? vis, defaultness, safety, generics, constness, polarity, of_trait, self_ty); + let Impl { generics, of_trait, self_ty, items } = self; + try_visit!(vis.visit_generics(generics)); + if let Some(box of_trait) = of_trait { + let TraitImplHeader { defaultness, safety, constness, polarity, trait_ref } = of_trait; + visit_visitable!($($mut)? vis, defaultness, safety, constness, polarity, trait_ref); + } + try_visit!(vis.visit_ty(self_ty)); visit_visitable_with!($($mut)? vis, items, AssocCtxt::Impl { of_trait: of_trait.is_some() }); V::Result::output() }); @@ -1144,15 +1147,15 @@ macro_rules! generate_list_visit_fns { } generate_list_visit_fns! { - visit_items, P, visit_item; - visit_foreign_items, P, visit_foreign_item; + visit_items, Box, visit_item; + visit_foreign_items, Box, visit_foreign_item; visit_generic_params, GenericParam, visit_generic_param; visit_stmts, Stmt, visit_stmt; - visit_exprs, P, visit_expr; + visit_exprs, Box, visit_expr; visit_expr_fields, ExprField, visit_expr_field; visit_pat_fields, PatField, visit_pat_field; visit_variants, Variant, visit_variant; - visit_assoc_items, P, visit_assoc_item, ctxt: AssocCtxt; + visit_assoc_items, Box, visit_assoc_item, ctxt: AssocCtxt; visit_where_predicates, WherePredicate, visit_where_predicate; visit_params, Param, visit_param; visit_field_defs, FieldDef, visit_field_def; diff --git a/compiler/rustc_ast_ir/Cargo.toml b/compiler/rustc_ast_ir/Cargo.toml index 76bdd9f7eb6d..1d21a07dc440 100644 --- a/compiler/rustc_ast_ir/Cargo.toml +++ b/compiler/rustc_ast_ir/Cargo.toml @@ -8,12 +8,16 @@ edition = "2024" rustc_data_structures = { path = "../rustc_data_structures", optional = true } rustc_macros = { path = "../rustc_macros", optional = true } rustc_serialize = { path = "../rustc_serialize", optional = true } +rustc_span = { path = "../rustc_span", optional = true } # tidy-alphabetical-end [features] +# tidy-alphabetical-start default = ["nightly"] nightly = [ - "dep:rustc_serialize", "dep:rustc_data_structures", "dep:rustc_macros", + "dep:rustc_serialize", + "dep:rustc_span", ] +# tidy-alphabetical-end diff --git a/compiler/rustc_ast_ir/src/lib.rs b/compiler/rustc_ast_ir/src/lib.rs index 0898433a74c5..0f63fad15743 100644 --- a/compiler/rustc_ast_ir/src/lib.rs +++ b/compiler/rustc_ast_ir/src/lib.rs @@ -11,11 +11,221 @@ #![cfg_attr(feature = "nightly", feature(rustc_attrs))] // tidy-alphabetical-end +use std::fmt; + #[cfg(feature = "nightly")] use rustc_macros::{Decodable_NoContext, Encodable_NoContext, HashStable_NoContext}; +#[cfg(feature = "nightly")] +use rustc_span::{Symbol, sym}; pub mod visit; +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + feature = "nightly", + derive(Encodable_NoContext, Decodable_NoContext, HashStable_NoContext) +)] +pub enum IntTy { + Isize, + I8, + I16, + I32, + I64, + I128, +} + +impl IntTy { + pub fn name_str(&self) -> &'static str { + match *self { + IntTy::Isize => "isize", + IntTy::I8 => "i8", + IntTy::I16 => "i16", + IntTy::I32 => "i32", + IntTy::I64 => "i64", + IntTy::I128 => "i128", + } + } + + #[cfg(feature = "nightly")] + pub fn name(self) -> Symbol { + match self { + IntTy::Isize => sym::isize, + IntTy::I8 => sym::i8, + IntTy::I16 => sym::i16, + IntTy::I32 => sym::i32, + IntTy::I64 => sym::i64, + IntTy::I128 => sym::i128, + } + } + + pub fn bit_width(&self) -> Option { + Some(match *self { + IntTy::Isize => return None, + IntTy::I8 => 8, + IntTy::I16 => 16, + IntTy::I32 => 32, + IntTy::I64 => 64, + IntTy::I128 => 128, + }) + } + + pub fn normalize(&self, target_width: u16) -> Self { + match self { + IntTy::Isize => match target_width { + 16 => IntTy::I16, + 32 => IntTy::I32, + 64 => IntTy::I64, + _ => unreachable!(), + }, + _ => *self, + } + } + + pub fn to_unsigned(self) -> UintTy { + match self { + IntTy::Isize => UintTy::Usize, + IntTy::I8 => UintTy::U8, + IntTy::I16 => UintTy::U16, + IntTy::I32 => UintTy::U32, + IntTy::I64 => UintTy::U64, + IntTy::I128 => UintTy::U128, + } + } +} + +impl fmt::Debug for IntTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} + +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy)] +#[cfg_attr( + feature = "nightly", + derive(Encodable_NoContext, Decodable_NoContext, HashStable_NoContext) +)] +pub enum UintTy { + Usize, + U8, + U16, + U32, + U64, + U128, +} + +impl UintTy { + pub fn name_str(&self) -> &'static str { + match *self { + UintTy::Usize => "usize", + UintTy::U8 => "u8", + UintTy::U16 => "u16", + UintTy::U32 => "u32", + UintTy::U64 => "u64", + UintTy::U128 => "u128", + } + } + + #[cfg(feature = "nightly")] + pub fn name(self) -> Symbol { + match self { + UintTy::Usize => sym::usize, + UintTy::U8 => sym::u8, + UintTy::U16 => sym::u16, + UintTy::U32 => sym::u32, + UintTy::U64 => sym::u64, + UintTy::U128 => sym::u128, + } + } + + pub fn bit_width(&self) -> Option { + Some(match *self { + UintTy::Usize => return None, + UintTy::U8 => 8, + UintTy::U16 => 16, + UintTy::U32 => 32, + UintTy::U64 => 64, + UintTy::U128 => 128, + }) + } + + pub fn normalize(&self, target_width: u16) -> Self { + match self { + UintTy::Usize => match target_width { + 16 => UintTy::U16, + 32 => UintTy::U32, + 64 => UintTy::U64, + _ => unreachable!(), + }, + _ => *self, + } + } + + pub fn to_signed(self) -> IntTy { + match self { + UintTy::Usize => IntTy::Isize, + UintTy::U8 => IntTy::I8, + UintTy::U16 => IntTy::I16, + UintTy::U32 => IntTy::I32, + UintTy::U64 => IntTy::I64, + UintTy::U128 => IntTy::I128, + } + } +} + +impl fmt::Debug for UintTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + feature = "nightly", + derive(Encodable_NoContext, Decodable_NoContext, HashStable_NoContext) +)] +pub enum FloatTy { + F16, + F32, + F64, + F128, +} + +impl FloatTy { + pub fn name_str(self) -> &'static str { + match self { + FloatTy::F16 => "f16", + FloatTy::F32 => "f32", + FloatTy::F64 => "f64", + FloatTy::F128 => "f128", + } + } + + #[cfg(feature = "nightly")] + pub fn name(self) -> Symbol { + match self { + FloatTy::F16 => sym::f16, + FloatTy::F32 => sym::f32, + FloatTy::F64 => sym::f64, + FloatTy::F128 => sym::f128, + } + } + + pub fn bit_width(self) -> u64 { + match self { + FloatTy::F16 => 16, + FloatTy::F32 => 32, + FloatTy::F64 => 64, + FloatTy::F128 => 128, + } + } +} + +impl fmt::Debug for FloatTy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name_str()) + } +} + /// The movability of a coroutine / closure literal: /// whether a coroutine contains self-references, causing it to be `!Unpin`. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy)] @@ -101,3 +311,10 @@ pub enum Pinnedness { Not, Pinned, } + +impl Pinnedness { + /// Return `true` if self is pinned + pub fn is_pinned(self) -> bool { + matches!(self, Self::Pinned) + } +} diff --git a/compiler/rustc_ast_lowering/Cargo.toml b/compiler/rustc_ast_lowering/Cargo.toml index dc571f5c3671..6ac258155fe9 100644 --- a/compiler/rustc_ast_lowering/Cargo.toml +++ b/compiler/rustc_ast_lowering/Cargo.toml @@ -11,7 +11,6 @@ doctest = false rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_ast_pretty = { path = "../rustc_ast_pretty" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_attr_parsing = { path = "../rustc_attr_parsing" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs index af279e07acc6..d44faad017ee 100644 --- a/compiler/rustc_ast_lowering/src/asm.rs +++ b/compiler/rustc_ast_lowering/src/asm.rs @@ -48,6 +48,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { | asm::InlineAsmArch::Arm64EC | asm::InlineAsmArch::RiscV32 | asm::InlineAsmArch::RiscV64 + | asm::InlineAsmArch::LoongArch32 | asm::InlineAsmArch::LoongArch64 | asm::InlineAsmArch::S390x ); diff --git a/compiler/rustc_ast_lowering/src/block.rs b/compiler/rustc_ast_lowering/src/block.rs index 2cc07694afbc..5bfe63008516 100644 --- a/compiler/rustc_ast_lowering/src/block.rs +++ b/compiler/rustc_ast_lowering/src/block.rs @@ -1,5 +1,6 @@ use rustc_ast::{Block, BlockCheckMode, Local, LocalKind, Stmt, StmtKind}; use rustc_hir as hir; +use rustc_hir::Target; use rustc_span::sym; use smallvec::SmallVec; @@ -26,7 +27,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { hir::Block { hir_id, stmts, expr, rules, span: self.lower_span(b.span), targeted_by_break } } - fn lower_stmts( + pub(super) fn lower_stmts( &mut self, mut ast_stmts: &[Stmt], ) -> (&'hir [hir::Stmt<'hir>], Option<&'hir hir::Expr<'hir>>) { @@ -109,7 +110,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { }; let span = self.lower_span(l.span); let source = hir::LocalSource::Normal; - self.lower_attrs(hir_id, &l.attrs, l.span); + self.lower_attrs(hir_id, &l.attrs, l.span, Target::Statement); self.arena.alloc(hir::LetStmt { hir_id, super_, ty, pat, init, els, span, source }) } diff --git a/compiler/rustc_ast_lowering/src/contract.rs b/compiler/rustc_ast_lowering/src/contract.rs new file mode 100644 index 000000000000..56413160e04d --- /dev/null +++ b/compiler/rustc_ast_lowering/src/contract.rs @@ -0,0 +1,345 @@ +use std::sync::Arc; + +use thin_vec::thin_vec; + +use crate::LoweringContext; + +impl<'a, 'hir> LoweringContext<'a, 'hir> { + /// Lowered contracts are guarded with the `contract_checks` compiler flag, + /// i.e. the flag turns into a boolean guard in the lowered HIR. The reason + /// for not eliminating the contract code entirely when the `contract_checks` + /// flag is disabled is so that contracts can be type checked, even when + /// they are disabled, which avoids them becoming stale (i.e. out of sync + /// with the codebase) over time. + /// + /// The optimiser should be able to eliminate all contract code guarded + /// by `if false`, leaving the original body intact when runtime contract + /// checks are disabled. + pub(super) fn lower_contract( + &mut self, + body: impl FnOnce(&mut Self) -> rustc_hir::Expr<'hir>, + contract: &rustc_ast::FnContract, + ) -> rustc_hir::Expr<'hir> { + // The order in which things are lowered is important! I.e to + // refer to variables in contract_decls from postcond/precond, + // we must lower it first! + let contract_decls = self.lower_stmts(&contract.declarations).0; + + match (&contract.requires, &contract.ensures) { + (Some(req), Some(ens)) => { + // Lower the fn contract, which turns: + // + // { body } + // + // into: + // + // let __postcond = if contract_checks { + // CONTRACT_DECLARATIONS; + // contract_check_requires(PRECOND); + // Some(|ret_val| POSTCOND) + // } else { + // None + // }; + // { + // let ret = { body }; + // + // if contract_checks { + // contract_check_ensures(__postcond, ret) + // } else { + // ret + // } + // } + + let precond = self.lower_precond(req); + let postcond_checker = self.lower_postcond_checker(ens); + + let contract_check = self.lower_contract_check_with_postcond( + contract_decls, + Some(precond), + postcond_checker, + ); + + let wrapped_body = + self.wrap_body_with_contract_check(body, contract_check, postcond_checker.span); + self.expr_block(wrapped_body) + } + (None, Some(ens)) => { + // Lower the fn contract, which turns: + // + // { body } + // + // into: + // + // let __postcond = if contract_checks { + // Some(|ret_val| POSTCOND) + // } else { + // None + // }; + // { + // let ret = { body }; + // + // if contract_checks { + // CONTRACT_DECLARATIONS; + // contract_check_ensures(__postcond, ret) + // } else { + // ret + // } + // } + let postcond_checker = self.lower_postcond_checker(ens); + let contract_check = + self.lower_contract_check_with_postcond(contract_decls, None, postcond_checker); + + let wrapped_body = + self.wrap_body_with_contract_check(body, contract_check, postcond_checker.span); + self.expr_block(wrapped_body) + } + (Some(req), None) => { + // Lower the fn contract, which turns: + // + // { body } + // + // into: + // + // { + // if contracts_checks { + // CONTRACT_DECLARATIONS; + // contract_requires(PRECOND); + // } + // body + // } + let precond = self.lower_precond(req); + let precond_check = self.lower_contract_check_just_precond(contract_decls, precond); + + let body = self.arena.alloc(body(self)); + + // Flatten the body into precond check, then body. + let wrapped_body = self.block_all( + body.span, + self.arena.alloc_from_iter([precond_check].into_iter()), + Some(body), + ); + self.expr_block(wrapped_body) + } + (None, None) => body(self), + } + } + + /// Lower the precondition check intrinsic. + fn lower_precond(&mut self, req: &Box) -> rustc_hir::Stmt<'hir> { + let lowered_req = self.lower_expr_mut(&req); + let req_span = self.mark_span_with_reason( + rustc_span::DesugaringKind::Contract, + lowered_req.span, + Some(Arc::clone(&self.allow_contracts)), + ); + let precond = self.expr_call_lang_item_fn_mut( + req_span, + rustc_hir::LangItem::ContractCheckRequires, + &*arena_vec![self; lowered_req], + ); + self.stmt_expr(req.span, precond) + } + + fn lower_postcond_checker( + &mut self, + ens: &Box, + ) -> &'hir rustc_hir::Expr<'hir> { + let ens_span = self.lower_span(ens.span); + let ens_span = self.mark_span_with_reason( + rustc_span::DesugaringKind::Contract, + ens_span, + Some(Arc::clone(&self.allow_contracts)), + ); + let lowered_ens = self.lower_expr_mut(&ens); + self.expr_call_lang_item_fn( + ens_span, + rustc_hir::LangItem::ContractBuildCheckEnsures, + &*arena_vec![self; lowered_ens], + ) + } + + fn lower_contract_check_just_precond( + &mut self, + contract_decls: &'hir [rustc_hir::Stmt<'hir>], + precond: rustc_hir::Stmt<'hir>, + ) -> rustc_hir::Stmt<'hir> { + let stmts = self + .arena + .alloc_from_iter(contract_decls.into_iter().map(|d| *d).chain([precond].into_iter())); + + let then_block_stmts = self.block_all(precond.span, stmts, None); + let then_block = self.arena.alloc(self.expr_block(&then_block_stmts)); + + let precond_check = rustc_hir::ExprKind::If( + self.arena.alloc(self.expr_bool_literal(precond.span, self.tcx.sess.contract_checks())), + then_block, + None, + ); + + let precond_check = self.expr(precond.span, precond_check); + self.stmt_expr(precond.span, precond_check) + } + + fn lower_contract_check_with_postcond( + &mut self, + contract_decls: &'hir [rustc_hir::Stmt<'hir>], + precond: Option>, + postcond_checker: &'hir rustc_hir::Expr<'hir>, + ) -> &'hir rustc_hir::Expr<'hir> { + let stmts = self + .arena + .alloc_from_iter(contract_decls.into_iter().map(|d| *d).chain(precond.into_iter())); + let span = match precond { + Some(precond) => precond.span, + None => postcond_checker.span, + }; + + let postcond_checker = self.arena.alloc(self.expr_enum_variant_lang_item( + postcond_checker.span, + rustc_hir::lang_items::LangItem::OptionSome, + &*arena_vec![self; *postcond_checker], + )); + let then_block_stmts = self.block_all(span, stmts, Some(postcond_checker)); + let then_block = self.arena.alloc(self.expr_block(&then_block_stmts)); + + let none_expr = self.arena.alloc(self.expr_enum_variant_lang_item( + postcond_checker.span, + rustc_hir::lang_items::LangItem::OptionNone, + Default::default(), + )); + let else_block = self.block_expr(none_expr); + let else_block = self.arena.alloc(self.expr_block(else_block)); + + let contract_check = rustc_hir::ExprKind::If( + self.arena.alloc(self.expr_bool_literal(span, self.tcx.sess.contract_checks())), + then_block, + Some(else_block), + ); + self.arena.alloc(self.expr(span, contract_check)) + } + + fn wrap_body_with_contract_check( + &mut self, + body: impl FnOnce(&mut Self) -> rustc_hir::Expr<'hir>, + contract_check: &'hir rustc_hir::Expr<'hir>, + postcond_span: rustc_span::Span, + ) -> &'hir rustc_hir::Block<'hir> { + let check_ident: rustc_span::Ident = + rustc_span::Ident::from_str_and_span("__ensures_checker", postcond_span); + let (check_hir_id, postcond_decl) = { + // Set up the postcondition `let` statement. + let (checker_pat, check_hir_id) = self.pat_ident_binding_mode_mut( + postcond_span, + check_ident, + rustc_hir::BindingMode::NONE, + ); + ( + check_hir_id, + self.stmt_let_pat( + None, + postcond_span, + Some(contract_check), + self.arena.alloc(checker_pat), + rustc_hir::LocalSource::Contract, + ), + ) + }; + + // Install contract_ensures so we will intercept `return` statements, + // then lower the body. + self.contract_ensures = Some((postcond_span, check_ident, check_hir_id)); + let body = self.arena.alloc(body(self)); + + // Finally, inject an ensures check on the implicit return of the body. + let body = self.inject_ensures_check(body, postcond_span, check_ident, check_hir_id); + + // Flatten the body into precond, then postcond, then wrapped body. + let wrapped_body = self.block_all( + body.span, + self.arena.alloc_from_iter([postcond_decl].into_iter()), + Some(body), + ); + wrapped_body + } + + /// Create an `ExprKind::Ret` that is optionally wrapped by a call to check + /// a contract ensures clause, if it exists. + pub(super) fn checked_return( + &mut self, + opt_expr: Option<&'hir rustc_hir::Expr<'hir>>, + ) -> rustc_hir::ExprKind<'hir> { + let checked_ret = + if let Some((check_span, check_ident, check_hir_id)) = self.contract_ensures { + let expr = opt_expr.unwrap_or_else(|| self.expr_unit(check_span)); + Some(self.inject_ensures_check(expr, check_span, check_ident, check_hir_id)) + } else { + opt_expr + }; + rustc_hir::ExprKind::Ret(checked_ret) + } + + /// Wraps an expression with a call to the ensures check before it gets returned. + pub(super) fn inject_ensures_check( + &mut self, + expr: &'hir rustc_hir::Expr<'hir>, + span: rustc_span::Span, + cond_ident: rustc_span::Ident, + cond_hir_id: rustc_hir::HirId, + ) -> &'hir rustc_hir::Expr<'hir> { + // { + // let ret = { body }; + // + // if contract_checks { + // contract_check_ensures(__postcond, ret) + // } else { + // ret + // } + // } + let ret_ident: rustc_span::Ident = rustc_span::Ident::from_str_and_span("__ret", span); + + // Set up the return `let` statement. + let (ret_pat, ret_hir_id) = + self.pat_ident_binding_mode_mut(span, ret_ident, rustc_hir::BindingMode::NONE); + + let ret_stmt = self.stmt_let_pat( + None, + span, + Some(expr), + self.arena.alloc(ret_pat), + rustc_hir::LocalSource::Contract, + ); + + let ret = self.expr_ident(span, ret_ident, ret_hir_id); + + let cond_fn = self.expr_ident(span, cond_ident, cond_hir_id); + let contract_check = self.expr_call_lang_item_fn_mut( + span, + rustc_hir::LangItem::ContractCheckEnsures, + arena_vec![self; *cond_fn, *ret], + ); + let contract_check = self.arena.alloc(contract_check); + let call_expr = self.block_expr_block(contract_check); + + // same ident can't be used in 2 places, so we create a new one for the + // else branch + let ret = self.expr_ident(span, ret_ident, ret_hir_id); + let ret_block = self.block_expr_block(ret); + + let contracts_enabled: rustc_hir::Expr<'_> = + self.expr_bool_literal(span, self.tcx.sess.contract_checks()); + let contract_check = self.arena.alloc(self.expr( + span, + rustc_hir::ExprKind::If( + self.arena.alloc(contracts_enabled), + call_expr, + Some(ret_block), + ), + )); + + let attrs: rustc_ast::AttrVec = thin_vec![self.unreachable_code_attr(span)]; + self.lower_attrs(contract_check.hir_id, &attrs, span, rustc_hir::Target::Expression); + + let ret_block = self.block_all(span, arena_vec![self; ret_stmt], Some(contract_check)); + self.arena.alloc(self.expr_block(self.arena.alloc(ret_block))) + } +} diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index 657792c93971..2be2fca87c3c 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -1,14 +1,14 @@ use std::ops::ControlFlow; use std::sync::Arc; -use rustc_ast::ptr::P as AstP; use rustc_ast::*; use rustc_ast_pretty::pprust::expr_to_string; -use rustc_attr_data_structures::{AttributeKind, find_attr}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir as hir; -use rustc_hir::HirId; +use rustc_hir::attrs::AttributeKind; use rustc_hir::def::{DefKind, Res}; +use rustc_hir::definitions::DefPathData; +use rustc_hir::{HirId, Target, find_attr}; use rustc_middle::span_bug; use rustc_middle::ty::TyCtxt; use rustc_session::errors::report_lit_error; @@ -53,7 +53,7 @@ impl<'v> rustc_ast::visit::Visitor<'v> for WillCreateDefIdsVisitor { } impl<'hir> LoweringContext<'_, 'hir> { - fn lower_exprs(&mut self, exprs: &[AstP]) -> &'hir [hir::Expr<'hir>] { + fn lower_exprs(&mut self, exprs: &[Box]) -> &'hir [hir::Expr<'hir>] { self.arena.alloc_from_iter(exprs.iter().map(|x| self.lower_expr_mut(x))) } @@ -63,19 +63,20 @@ impl<'hir> LoweringContext<'_, 'hir> { pub(super) fn lower_expr_mut(&mut self, e: &Expr) -> hir::Expr<'hir> { ensure_sufficient_stack(|| { + let mut span = self.lower_span(e.span); match &e.kind { // Parenthesis expression does not have a HirId and is handled specially. ExprKind::Paren(ex) => { let mut ex = self.lower_expr_mut(ex); // Include parens in span, but only if it is a super-span. if e.span.contains(ex.span) { - ex.span = self.lower_span(e.span); + ex.span = self.lower_span(e.span.with_ctxt(ex.span.ctxt())); } // Merge attributes into the inner expression. if !e.attrs.is_empty() { let old_attrs = self.attrs.get(&ex.hir_id.local_id).copied().unwrap_or(&[]); let new_attrs = self - .lower_attrs_vec(&e.attrs, e.span, ex.hir_id) + .lower_attrs_vec(&e.attrs, e.span, ex.hir_id, Target::from_expr(e)) .into_iter() .chain(old_attrs.iter().cloned()); let new_attrs = &*self.arena.alloc_from_iter(new_attrs); @@ -98,7 +99,7 @@ impl<'hir> LoweringContext<'_, 'hir> { } let expr_hir_id = self.lower_node_id(e.id); - self.lower_attrs(expr_hir_id, &e.attrs, e.span); + let attrs = self.lower_attrs(expr_hir_id, &e.attrs, e.span, Target::from_expr(e)); let kind = match &e.kind { ExprKind::Array(exprs) => hir::ExprKind::Array(self.lower_exprs(exprs)), @@ -232,10 +233,10 @@ impl<'hir> LoweringContext<'_, 'hir> { *fn_arg_span, ), None => self.lower_expr_closure( + attrs, binder, *capture_clause, e.id, - expr_hir_id, *constness, *movability, fn_decl, @@ -288,7 +289,8 @@ impl<'hir> LoweringContext<'_, 'hir> { self.lower_span(*brackets_span), ), ExprKind::Range(e1, e2, lims) => { - self.lower_expr_range(e.span, e1.as_deref(), e2.as_deref(), *lims) + span = self.mark_span_with_reason(DesugaringKind::RangeExpr, span, None); + self.lower_expr_range(span, e1.as_deref(), e2.as_deref(), *lims) } ExprKind::Underscore => { let guar = self.dcx().emit_err(UnderscoreExprLhsAssign { span: e.span }); @@ -380,40 +382,10 @@ impl<'hir> LoweringContext<'_, 'hir> { ExprKind::MacCall(_) => panic!("{:?} shouldn't exist here", e.span), }; - hir::Expr { hir_id: expr_hir_id, kind, span: self.lower_span(e.span) } + hir::Expr { hir_id: expr_hir_id, kind, span } }) } - /// Create an `ExprKind::Ret` that is optionally wrapped by a call to check - /// a contract ensures clause, if it exists. - fn checked_return(&mut self, opt_expr: Option<&'hir hir::Expr<'hir>>) -> hir::ExprKind<'hir> { - let checked_ret = - if let Some((check_span, check_ident, check_hir_id)) = self.contract_ensures { - let expr = opt_expr.unwrap_or_else(|| self.expr_unit(check_span)); - Some(self.inject_ensures_check(expr, check_span, check_ident, check_hir_id)) - } else { - opt_expr - }; - hir::ExprKind::Ret(checked_ret) - } - - /// Wraps an expression with a call to the ensures check before it gets returned. - pub(crate) fn inject_ensures_check( - &mut self, - expr: &'hir hir::Expr<'hir>, - span: Span, - cond_ident: Ident, - cond_hir_id: HirId, - ) -> &'hir hir::Expr<'hir> { - let cond_fn = self.expr_ident(span, cond_ident, cond_hir_id); - let call_expr = self.expr_call_lang_item_fn_mut( - span, - hir::LangItem::ContractCheckEnsures, - arena_vec![self; *cond_fn, *expr], - ); - self.arena.alloc(call_expr) - } - pub(crate) fn lower_const_block(&mut self, c: &AnonConst) -> hir::ConstBlock { self.with_new_scopes(c.value.span, |this| { let def_id = this.local_def_id(c.id); @@ -455,7 +427,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_legacy_const_generics( &mut self, mut f: Expr, - args: ThinVec>, + args: ThinVec>, legacy_args_idx: &[usize], ) -> hir::ExprKind<'hir> { let ExprKind::Path(None, path) = &mut f.kind else { @@ -492,10 +464,16 @@ impl<'hir> LoweringContext<'_, 'hir> { for (idx, arg) in args.iter().cloned().enumerate() { if legacy_args_idx.contains(&idx) { let node_id = self.next_node_id(); - self.create_def(node_id, None, DefKind::AnonConst, f.span); + self.create_def( + node_id, + None, + DefKind::AnonConst, + DefPathData::LateAnonConst, + f.span, + ); let mut visitor = WillCreateDefIdsVisitor {}; let const_value = if let ControlFlow::Break(span) = visitor.visit_expr(&arg) { - AstP(Expr { + Box::new(Expr { id: self.next_node_id(), kind: ExprKind::Err(invalid_expr_error(self.tcx, span)), span: f.span, @@ -516,7 +494,7 @@ impl<'hir> LoweringContext<'_, 'hir> { // Add generic args to the last element of the path. let last_segment = path.segments.last_mut().unwrap(); assert!(last_segment.args.is_none()); - last_segment.args = Some(AstP(GenericArgs::AngleBracketed(AngleBracketedArgs { + last_segment.args = Some(Box::new(GenericArgs::AngleBracketed(AngleBracketedArgs { span: DUMMY_SP, args: generic_args, }))); @@ -640,7 +618,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let guard = arm.guard.as_ref().map(|cond| self.lower_expr(cond)); let hir_id = self.next_id(); let span = self.lower_span(arm.span); - self.lower_attrs(hir_id, &arm.attrs, arm.span); + self.lower_attrs(hir_id, &arm.attrs, arm.span, Target::Arm); let is_never_pattern = pat.is_never_pattern(); // We need to lower the body even if it's unneeded for never pattern in match, // ensure that we can get HirId for DefId if need (issue #137708). @@ -812,7 +790,7 @@ impl<'hir> LoweringContext<'_, 'hir> { self.lower_attrs( inner_hir_id, &[Attribute { - kind: AttrKind::Normal(ptr::P(NormalAttr::from_ident(Ident::new( + kind: AttrKind::Normal(Box::new(NormalAttr::from_ident(Ident::new( sym::track_caller, span, )))), @@ -821,6 +799,7 @@ impl<'hir> LoweringContext<'_, 'hir> { span: unstable_span, }], span, + Target::Fn, ); } } @@ -886,6 +865,7 @@ impl<'hir> LoweringContext<'_, 'hir> { }; let features = match await_kind { + FutureKind::Future if is_async_gen => Some(Arc::clone(&self.allow_async_gen)), FutureKind::Future => None, FutureKind::AsyncIterator => Some(Arc::clone(&self.allow_for_await)), }; @@ -1052,10 +1032,10 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_expr_closure( &mut self, + attrs: &[rustc_hir::Attribute], binder: &ClosureBinder, capture_clause: CaptureBy, closure_id: NodeId, - closure_hir_id: hir::HirId, constness: Const, movability: Movability, decl: &FnDecl, @@ -1067,15 +1047,9 @@ impl<'hir> LoweringContext<'_, 'hir> { let (binder_clause, generic_params) = self.lower_closure_binder(binder); let (body_id, closure_kind) = self.with_new_scopes(fn_decl_span, move |this| { - let mut coroutine_kind = if this - .attrs - .get(&closure_hir_id.local_id) - .is_some_and(|attrs| attrs.iter().any(|attr| attr.has_name(sym::coroutine))) - { - Some(hir::CoroutineKind::Coroutine(Movability::Movable)) - } else { - None - }; + + let mut coroutine_kind = find_attr!(attrs, AttributeKind::Coroutine(_) => hir::CoroutineKind::Coroutine(Movability::Movable)); + // FIXME(contracts): Support contracts on closures? let body_id = this.lower_fn_body(decl, None, |this| { this.coroutine_kind = coroutine_kind; @@ -1269,13 +1243,8 @@ impl<'hir> LoweringContext<'_, 'hir> { let rhs = self.lower_expr(rhs); // Introduce a `let` for destructuring: `let (lhs1, lhs2) = t`. - let destructure_let = self.stmt_let_pat( - None, - whole_span, - Some(rhs), - pat, - hir::LocalSource::AssignDesugar(self.lower_span(eq_sign_span)), - ); + let destructure_let = + self.stmt_let_pat(None, whole_span, Some(rhs), pat, hir::LocalSource::AssignDesugar); // `a = lhs1; b = lhs2;`. let stmts = self.arena.alloc_from_iter(std::iter::once(destructure_let).chain(assignments)); @@ -1291,7 +1260,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn extract_tuple_struct_path<'a>( &mut self, expr: &'a Expr, - ) -> Option<(&'a Option>, &'a Path)> { + ) -> Option<(&'a Option>, &'a Path)> { if let ExprKind::Path(qself, path) = &expr.kind { // Does the path resolve to something disallowed in a tuple struct/variant pattern? if let Some(partial_res) = self.resolver.get_partial_res(expr.id) { @@ -1313,7 +1282,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn extract_unit_struct_path<'a>( &mut self, expr: &'a Expr, - ) -> Option<(&'a Option>, &'a Path)> { + ) -> Option<(&'a Option>, &'a Path)> { if let ExprKind::Path(qself, path) = &expr.kind { // Does the path resolve to something disallowed in a unit struct/variant pattern? if let Some(partial_res) = self.resolver.get_partial_res(expr.id) { @@ -1440,10 +1409,10 @@ impl<'hir> LoweringContext<'_, 'hir> { self.dcx().emit_err(FunctionalRecordUpdateDestructuringAssignment { span: e.span, }); - true + Some(self.lower_span(e.span)) } - StructRest::Rest(_) => true, - StructRest::None => false, + StructRest::Rest(span) => Some(self.lower_span(*span)), + StructRest::None => None, }; let struct_pat = hir::PatKind::Struct(qpath, field_pats, fields_omitted); return self.pat_without_dbm(lhs.span, struct_pat); @@ -1484,7 +1453,7 @@ impl<'hir> LoweringContext<'_, 'hir> { /// Each sub-assignment is recorded in `assignments`. fn destructure_sequence( &mut self, - elements: &[AstP], + elements: &[Box], ctx: &str, eq_sign_span: Span, assignments: &mut Vec>, @@ -1511,7 +1480,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_expr_range_closed(&mut self, span: Span, e1: &Expr, e2: &Expr) -> hir::ExprKind<'hir> { let e1 = self.lower_expr_mut(e1); let e2 = self.lower_expr_mut(e2); - let fn_path = hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span)); + let fn_path = self.make_lang_item_qpath(hir::LangItem::RangeInclusiveNew, span, None); let fn_expr = self.arena.alloc(self.expr(span, hir::ExprKind::Path(fn_path))); hir::ExprKind::Call(fn_expr, arena_vec![self; e1, e2]) } @@ -1542,7 +1511,13 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::LangItem::Range } } - (None, Some(..), Closed) => hir::LangItem::RangeToInclusive, + (None, Some(..), Closed) => { + if self.tcx.features().new_range() { + hir::LangItem::RangeToInclusiveCopy + } else { + hir::LangItem::RangeToInclusive + } + } (Some(e1), Some(e2), Closed) => { if self.tcx.features().new_range() { hir::LangItem::RangeInclusiveCopy @@ -1566,17 +1541,32 @@ impl<'hir> LoweringContext<'_, 'hir> { }; let fields = self.arena.alloc_from_iter( - e1.iter().map(|e| (sym::start, e)).chain(e2.iter().map(|e| (sym::end, e))).map( - |(s, e)| { + e1.iter() + .map(|e| (sym::start, e)) + .chain(e2.iter().map(|e| { + ( + if matches!( + lang_item, + hir::LangItem::RangeInclusiveCopy | hir::LangItem::RangeToInclusiveCopy + ) { + sym::last + } else { + sym::end + }, + e, + ) + })) + .map(|(s, e)| { + let span = self.lower_span(e.span); + let span = self.mark_span_with_reason(DesugaringKind::RangeExpr, span, None); let expr = self.lower_expr(e); - let ident = Ident::new(s, self.lower_span(e.span)); - self.expr_field(ident, expr, e.span) - }, - ), + let ident = Ident::new(s, span); + self.expr_field(ident, expr, span) + }), ); hir::ExprKind::Struct( - self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span))), + self.arena.alloc(self.make_lang_item_qpath(lang_item, span, None)), fields, hir::StructTailExpr::None, ) @@ -1661,7 +1651,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_expr_field(&mut self, f: &ExprField) -> hir::ExprField<'hir> { let hir_id = self.lower_node_id(f.id); - self.lower_attrs(hir_id, &f.attrs, f.span); + self.lower_attrs(hir_id, &f.attrs, f.span, Target::ExprField); hir::ExprField { hir_id, ident: self.lower_ident(f.ident), @@ -1726,8 +1716,13 @@ impl<'hir> LoweringContext<'_, 'hir> { // `yield $expr` is transformed into `task_context = yield async_gen_ready($expr)`. // This ensures that we store our resumed `ResumeContext` correctly, and also that // the apparent value of the `yield` expression is `()`. - let wrapped_yielded = self.expr_call_lang_item_fn( + let desugar_span = self.mark_span_with_reason( + DesugaringKind::Async, span, + Some(Arc::clone(&self.allow_async_gen)), + ); + let wrapped_yielded = self.expr_call_lang_item_fn( + desugar_span, hir::LangItem::AsyncGenReady, std::slice::from_ref(yielded), ); @@ -1739,7 +1734,7 @@ impl<'hir> LoweringContext<'_, 'hir> { unreachable!("use of `await` outside of an async context."); }; let task_context_ident = Ident::with_dummy_span(sym::_task_context); - let lhs = self.expr_ident(span, task_context_ident, task_context_hid); + let lhs = self.expr_ident(desugar_span, task_context_ident, task_context_hid); hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)) } else { @@ -1917,7 +1912,7 @@ impl<'hir> LoweringContext<'_, 'hir> { // // Also, add the attributes to the outer returned expr node. let expr = self.expr_drop_temps_mut(for_span, match_expr); - self.lower_attrs(expr.hir_id, &e.attrs, e.span); + self.lower_attrs(expr.hir_id, &e.attrs, e.span, Target::from_expr(e)); expr } @@ -1958,23 +1953,14 @@ impl<'hir> LoweringContext<'_, 'hir> { ) }; - // `#[allow(unreachable_code)]` - let attr = attr::mk_attr_nested_word( - &self.tcx.sess.psess.attr_id_generator, - AttrStyle::Outer, - Safety::Default, - sym::allow, - sym::unreachable_code, - try_span, - ); - let attrs: AttrVec = thin_vec![attr]; + let attrs: AttrVec = thin_vec![self.unreachable_code_attr(try_span)]; // `ControlFlow::Continue(val) => #[allow(unreachable_code)] val,` let continue_arm = { let val_ident = Ident::with_dummy_span(sym::val); let (val_pat, val_pat_nid) = self.pat_ident(span, val_ident); let val_expr = self.expr_ident(span, val_ident, val_pat_nid); - self.lower_attrs(val_expr.hir_id, &attrs, span); + self.lower_attrs(val_expr.hir_id, &attrs, span, Target::Expression); let continue_pat = self.pat_cf_continue(unstable_span, val_pat); self.arm(continue_pat, val_expr) }; @@ -2005,7 +1991,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let ret_expr = self.checked_return(Some(from_residual_expr)); self.arena.alloc(self.expr(try_span, ret_expr)) }; - self.lower_attrs(ret_expr.hir_id, &attrs, span); + self.lower_attrs(ret_expr.hir_id, &attrs, span, Target::Expression); let break_pat = self.pat_cf_break(try_span, residual_local); self.arm(break_pat, ret_expr) @@ -2107,7 +2093,7 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(span, hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Mut, e)) } - fn expr_unit(&mut self, sp: Span) -> &'hir hir::Expr<'hir> { + pub(super) fn expr_unit(&mut self, sp: Span) -> &'hir hir::Expr<'hir> { self.arena.alloc(self.expr(sp, hir::ExprKind::Tup(&[]))) } @@ -2148,6 +2134,43 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(span, hir::ExprKind::Call(e, args)) } + pub(super) fn expr_struct( + &mut self, + span: Span, + path: &'hir hir::QPath<'hir>, + fields: &'hir [hir::ExprField<'hir>], + ) -> hir::Expr<'hir> { + self.expr(span, hir::ExprKind::Struct(path, fields, rustc_hir::StructTailExpr::None)) + } + + pub(super) fn expr_enum_variant( + &mut self, + span: Span, + path: &'hir hir::QPath<'hir>, + fields: &'hir [hir::Expr<'hir>], + ) -> hir::Expr<'hir> { + let fields = self.arena.alloc_from_iter(fields.into_iter().enumerate().map(|(i, f)| { + hir::ExprField { + hir_id: self.next_id(), + ident: Ident::from_str(&i.to_string()), + expr: f, + span: f.span, + is_shorthand: false, + } + })); + self.expr_struct(span, path, fields) + } + + pub(super) fn expr_enum_variant_lang_item( + &mut self, + span: Span, + lang_item: hir::LangItem, + fields: &'hir [hir::Expr<'hir>], + ) -> hir::Expr<'hir> { + let path = self.arena.alloc(self.make_lang_item_qpath(lang_item, span, None)); + self.expr_enum_variant(span, path, fields) + } + pub(super) fn expr_call( &mut self, span: Span, @@ -2176,8 +2199,13 @@ impl<'hir> LoweringContext<'_, 'hir> { self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args)) } - fn expr_lang_item_path(&mut self, span: Span, lang_item: hir::LangItem) -> hir::Expr<'hir> { - self.expr(span, hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span)))) + pub(super) fn expr_lang_item_path( + &mut self, + span: Span, + lang_item: hir::LangItem, + ) -> hir::Expr<'hir> { + let qpath = self.make_lang_item_qpath(lang_item, self.lower_span(span), None); + self.expr(span, hir::ExprKind::Path(qpath)) } /// `::name` @@ -2257,6 +2285,17 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(b.span, hir::ExprKind::Block(b, None)) } + /// Wrap an expression in a block, and wrap that block in an expression again. + /// Useful for constructing if-expressions, which require expressions of + /// kind block. + pub(super) fn block_expr_block( + &mut self, + expr: &'hir hir::Expr<'hir>, + ) -> &'hir hir::Expr<'hir> { + let b = self.block_expr(expr); + self.arena.alloc(self.expr_block(b)) + } + pub(super) fn expr_array_ref( &mut self, span: Span, @@ -2270,6 +2309,10 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(span, hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, expr)) } + pub(super) fn expr_bool_literal(&mut self, span: Span, val: bool) -> hir::Expr<'hir> { + self.expr(span, hir::ExprKind::Lit(Spanned { node: LitKind::Bool(val), span })) + } + pub(super) fn expr(&mut self, span: Span, kind: hir::ExprKind<'hir>) -> hir::Expr<'hir> { let hir_id = self.next_id(); hir::Expr { hir_id, kind, span: self.lower_span(span) } @@ -2303,6 +2346,19 @@ impl<'hir> LoweringContext<'_, 'hir> { body: expr, } } + + /// `#[allow(unreachable_code)]` + pub(super) fn unreachable_code_attr(&mut self, span: Span) -> Attribute { + let attr = attr::mk_attr_nested_word( + &self.tcx.sess.psess.attr_id_generator, + AttrStyle::Outer, + Safety::Default, + sym::allow, + sym::unreachable_code, + span, + ); + attr + } } /// Used by [`LoweringContext::make_lowered_await`] to customize the desugaring based on what kind diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs index ec9d26eb33f4..c5bbc2b81188 100644 --- a/compiler/rustc_ast_lowering/src/format.rs +++ b/compiler/rustc_ast_lowering/src/format.rs @@ -391,7 +391,8 @@ fn make_format_spec<'hir>( let flags = ctx.expr_field(Ident::new(sym::flags, sp), ctx.arena.alloc(flags), sp); let precision = ctx.expr_field(Ident::new(sym::precision, sp), ctx.arena.alloc(precision), sp); let width = ctx.expr_field(Ident::new(sym::width, sp), ctx.arena.alloc(width), sp); - let placeholder = ctx.arena.alloc(hir::QPath::LangItem(hir::LangItem::FormatPlaceholder, sp)); + let placeholder = + ctx.arena.alloc(ctx.make_lang_item_qpath(hir::LangItem::FormatPlaceholder, sp, None)); let fields = ctx.arena.alloc_from_iter([position, flags, precision, width]); ctx.expr(sp, hir::ExprKind::Struct(placeholder, fields, hir::StructTailExpr::None)) } @@ -487,26 +488,6 @@ fn expand_format_args<'hir>( // Generate: // [] (vec![], ctx.arena.alloc(ctx.expr(macsp, hir::ExprKind::Array(&[])))) - } else if argmap.len() == 1 && arguments.len() == 1 { - // Only one argument, so we don't need to make the `args` tuple. - // - // Generate: - // super let args = [::new_display(&arg)]; - let args = ctx.arena.alloc_from_iter(argmap.iter().map( - |(&(arg_index, ty), &placeholder_span)| { - let arg = &arguments[arg_index]; - let placeholder_span = - placeholder_span.unwrap_or(arg.expr.span).with_ctxt(macsp.ctxt()); - let arg = ctx.lower_expr(&arg.expr); - let ref_arg = ctx.arena.alloc(ctx.expr_ref(arg.span.with_ctxt(macsp.ctxt()), arg)); - make_argument(ctx, placeholder_span, ref_arg, ty) - }, - )); - let args = ctx.arena.alloc(ctx.expr(macsp, hir::ExprKind::Array(args))); - let args_ident = Ident::new(sym::args, macsp); - let (args_pat, args_hir_id) = ctx.pat_ident(macsp, args_ident); - let let_statement = ctx.stmt_super_let_pat(macsp, args_pat, Some(args)); - (vec![let_statement], ctx.arena.alloc(ctx.expr_ident_mut(macsp, args_ident, args_hir_id))) } else { // Generate: // super let args = (&arg0, &arg1, &…); diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs index 5b63206d7d62..5f8933aa2beb 100644 --- a/compiler/rustc_ast_lowering/src/index.rs +++ b/compiler/rustc_ast_lowering/src/index.rs @@ -311,7 +311,7 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { ); self.with_parent(const_arg.hir_id, |this| { - intravisit::walk_ambig_const_arg(this, const_arg); + intravisit::walk_const_arg(this, const_arg); }); } diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index 1899dcda3619..177f33c0d8f1 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -1,12 +1,13 @@ use rustc_abi::ExternAbi; -use rustc_ast::ptr::P; use rustc_ast::visit::AssocCtxt; use rustc_ast::*; -use rustc_attr_data_structures::{AttributeKind, find_attr}; use rustc_errors::{E0570, ErrorGuaranteed, struct_span_code_err}; +use rustc_hir::attrs::AttributeKind; use rustc_hir::def::{DefKind, PerNS, Res}; use rustc_hir::def_id::{CRATE_DEF_ID, LocalDefId}; -use rustc_hir::{self as hir, HirId, LifetimeSource, PredicateOrigin}; +use rustc_hir::{ + self as hir, HirId, ImplItemImplKind, LifetimeSource, PredicateOrigin, Target, find_attr, +}; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::span_bug; use rustc_middle::ty::{ResolverAstLowering, TyCtxt}; @@ -35,20 +36,18 @@ pub(super) struct ItemLowerer<'a, 'hir> { /// clause if it exists. fn add_ty_alias_where_clause( generics: &mut ast::Generics, - mut where_clauses: TyAliasWhereClauses, + after_where_clause: &ast::WhereClause, prefer_first: bool, ) { + generics.where_clause.predicates.extend_from_slice(&after_where_clause.predicates); + + let mut before = (generics.where_clause.has_where_token, generics.where_clause.span); + let mut after = (after_where_clause.has_where_token, after_where_clause.span); if !prefer_first { - (where_clauses.before, where_clauses.after) = (where_clauses.after, where_clauses.before); + (before, after) = (after, before); } - let where_clause = - if where_clauses.before.has_where_token || !where_clauses.after.has_where_token { - where_clauses.before - } else { - where_clauses.after - }; - generics.where_clause.has_where_token = where_clause.has_where_token; - generics.where_clause.span = where_clause.span; + (generics.where_clause.has_where_token, generics.where_clause.span) = + if before.0 || !after.0 { before } else { after }; } impl<'a, 'hir> ItemLowerer<'a, 'hir> { @@ -81,7 +80,7 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> { self.with_lctx(CRATE_NODE_ID, |lctx| { let module = lctx.lower_mod(&c.items, &c.spans); // FIXME(jdonszelman): is dummy span ever a problem here? - lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, DUMMY_SP); + lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, DUMMY_SP, Target::Crate); hir::OwnerNode::Crate(module) }) } @@ -102,7 +101,7 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> { impl<'hir> LoweringContext<'_, 'hir> { pub(super) fn lower_mod( &mut self, - items: &[P], + items: &[Box], spans: &ModSpans, ) -> &'hir hir::Mod<'hir> { self.arena.alloc(hir::Mod { @@ -137,7 +136,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_item(&mut self, i: &Item) -> &'hir hir::Item<'hir> { let vis_span = self.lower_span(i.vis.span); let hir_id = hir::HirId::make_owner(self.current_hir_id_owner.def_id); - let attrs = self.lower_attrs(hir_id, &i.attrs, i.span); + let attrs = self.lower_attrs(hir_id, &i.attrs, i.span, Target::from_ast_item(i)); let kind = self.lower_item_kind(i.span, i.id, hir_id, attrs, vis_span, &i.kind); let item = hir::Item { owner_id: hir_id.expect_owner(), @@ -252,7 +251,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ItemKind::Mod(_, ident, mod_kind) => { let ident = self.lower_ident(*ident); match mod_kind { - ModKind::Loaded(items, _, spans, _) => { + ModKind::Loaded(items, _, spans) => { hir::ItemKind::Mod(ident, self.lower_mod(items, spans)) } ModKind::Unloaded => panic!("`mod` items should have been loaded by now"), @@ -270,7 +269,7 @@ impl<'hir> LoweringContext<'_, 'hir> { self.lower_body(|this| (&[], this.expr(span, hir::ExprKind::InlineAsm(asm)))); hir::ItemKind::GlobalAsm { asm, fake_body } } - ItemKind::TyAlias(box TyAlias { ident, generics, where_clauses, ty, .. }) => { + ItemKind::TyAlias(box TyAlias { ident, generics, after_where_clause, ty, .. }) => { // We lower // // type Foo = impl Trait @@ -281,7 +280,7 @@ impl<'hir> LoweringContext<'_, 'hir> { // opaque type Foo1: Trait let ident = self.lower_ident(*ident); let mut generics = generics.clone(); - add_ty_alias_where_clause(&mut generics, *where_clauses, true); + add_ty_alias_where_clause(&mut generics, after_where_clause, true); let (generics, ty) = self.lower_generics( &generics, id, @@ -341,13 +340,9 @@ impl<'hir> LoweringContext<'_, 'hir> { ); hir::ItemKind::Union(ident, generics, vdata) } - ItemKind::Impl(box Impl { - safety, - polarity, - defaultness, - constness, + ItemKind::Impl(Impl { generics: ast_generics, - of_trait: trait_ref, + of_trait, self_ty: ty, items: impl_items, }) => { @@ -365,54 +360,30 @@ impl<'hir> LoweringContext<'_, 'hir> { // lifetime to be added, but rather a reference to a // parent lifetime. let itctx = ImplTraitContext::Universal; - let (generics, (trait_ref, lowered_ty)) = + let (generics, (of_trait, lowered_ty)) = self.lower_generics(ast_generics, id, itctx, |this| { - let modifiers = TraitBoundModifiers { - constness: BoundConstness::Never, - asyncness: BoundAsyncness::Normal, - // we don't use this in bound lowering - polarity: BoundPolarity::Positive, - }; - - let trait_ref = trait_ref.as_ref().map(|trait_ref| { - this.lower_trait_ref( - modifiers, - trait_ref, - ImplTraitContext::Disallowed(ImplTraitPosition::Trait), - ) - }); + let of_trait = of_trait + .as_deref() + .map(|of_trait| this.lower_trait_impl_header(of_trait)); let lowered_ty = this.lower_ty( ty, ImplTraitContext::Disallowed(ImplTraitPosition::ImplSelf), ); - (trait_ref, lowered_ty) + (of_trait, lowered_ty) }); let new_impl_items = self .arena .alloc_from_iter(impl_items.iter().map(|item| self.lower_impl_item_ref(item))); - // `defaultness.has_value()` is never called for an `impl`, always `true` in order - // to not cause an assertion failure inside the `lower_defaultness` function. - let has_val = true; - let (defaultness, defaultness_span) = self.lower_defaultness(*defaultness, has_val); - let polarity = match polarity { - ImplPolarity::Positive => ImplPolarity::Positive, - ImplPolarity::Negative(s) => ImplPolarity::Negative(self.lower_span(*s)), - }; - hir::ItemKind::Impl(self.arena.alloc(hir::Impl { - constness: self.lower_constness(*constness), - safety: self.lower_safety(*safety, hir::Safety::Safe), - polarity, - defaultness, - defaultness_span, + hir::ItemKind::Impl(hir::Impl { generics, - of_trait: trait_ref, + of_trait, self_ty: lowered_ty, items: new_impl_items, - })) + }) } ItemKind::Trait(box Trait { constness, @@ -444,7 +415,8 @@ impl<'hir> LoweringContext<'_, 'hir> { ); hir::ItemKind::Trait(constness, *is_auto, safety, ident, generics, bounds, items) } - ItemKind::TraitAlias(ident, generics, bounds) => { + ItemKind::TraitAlias(box TraitAlias { constness, ident, generics, bounds }) => { + let constness = self.lower_constness(*constness); let ident = self.lower_ident(*ident); let (generics, bounds) = self.lower_generics( generics, @@ -453,26 +425,26 @@ impl<'hir> LoweringContext<'_, 'hir> { |this| { this.lower_param_bounds( bounds, - RelaxedBoundPolicy::Allowed, + RelaxedBoundPolicy::Forbidden(RelaxedBoundForbiddenReason::TraitAlias), ImplTraitContext::Disallowed(ImplTraitPosition::Bound), ) }, ); - hir::ItemKind::TraitAlias(ident, generics, bounds) + hir::ItemKind::TraitAlias(constness, ident, generics, bounds) } ItemKind::MacroDef(ident, MacroDef { body, macro_rules }) => { let ident = self.lower_ident(*ident); - let body = P(self.lower_delim_args(body)); + let body = Box::new(self.lower_delim_args(body)); let def_id = self.local_def_id(id); let def_kind = self.tcx.def_kind(def_id); - let DefKind::Macro(macro_kind) = def_kind else { + let DefKind::Macro(macro_kinds) = def_kind else { unreachable!( "expected DefKind::Macro for macro item, found {}", def_kind.descr(def_id.to_def_id()) ); }; let macro_def = self.arena.alloc(ast::MacroDef { body, macro_rules: *macro_rules }); - hir::ItemKind::Macro(ident, macro_def, macro_kind) + hir::ItemKind::Macro(ident, macro_def, macro_kinds) } ItemKind::Delegation(box delegation) => { let delegation_results = self.lower_delegation(delegation, id, false); @@ -650,7 +622,8 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> { let hir_id = hir::HirId::make_owner(self.current_hir_id_owner.def_id); let owner_id = hir_id.expect_owner(); - let attrs = self.lower_attrs(hir_id, &i.attrs, i.span); + let attrs = + self.lower_attrs(hir_id, &i.attrs, i.span, Target::from_foreign_item_kind(&i.kind)); let (ident, kind) = match &i.kind { ForeignItemKind::Fn(box Fn { sig, ident, generics, define_opaque, .. }) => { let fdec = &sig.decl; @@ -719,7 +692,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_variant(&mut self, item_kind: &ItemKind, v: &Variant) -> hir::Variant<'hir> { let hir_id = self.lower_node_id(v.id); - self.lower_attrs(hir_id, &v.attrs, v.span); + self.lower_attrs(hir_id, &v.attrs, v.span, Target::Variant); hir::Variant { hir_id, def_id: self.local_def_id(v.id), @@ -802,7 +775,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ) -> hir::FieldDef<'hir> { let ty = self.lower_ty(&f.ty, ImplTraitContext::Disallowed(ImplTraitPosition::FieldTy)); let hir_id = self.lower_node_id(f.id); - self.lower_attrs(hir_id, &f.attrs, f.span); + self.lower_attrs(hir_id, &f.attrs, f.span, Target::Field); hir::FieldDef { span: self.lower_span(f.span), hir_id, @@ -821,7 +794,12 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_trait_item(&mut self, i: &AssocItem) -> &'hir hir::TraitItem<'hir> { let hir_id = hir::HirId::make_owner(self.current_hir_id_owner.def_id); - let attrs = self.lower_attrs(hir_id, &i.attrs, i.span); + let attrs = self.lower_attrs( + hir_id, + &i.attrs, + i.span, + Target::from_assoc_item_kind(&i.kind, AssocCtxt::Trait), + ); let trait_item_def_id = hir_id.expect_owner(); let (ident, generics, kind, has_default) = match &i.kind { @@ -922,10 +900,15 @@ impl<'hir> LoweringContext<'_, 'hir> { ) } AssocItemKind::Type(box TyAlias { - ident, generics, where_clauses, bounds, ty, .. + ident, + generics, + after_where_clause, + bounds, + ty, + .. }) => { let mut generics = generics.clone(); - add_ty_alias_where_clause(&mut generics, *where_clauses, false); + add_ty_alias_where_clause(&mut generics, after_where_clause, false); let (generics, kind) = self.lower_generics( &generics, i.id, @@ -983,6 +966,44 @@ impl<'hir> LoweringContext<'_, 'hir> { self.expr(span, hir::ExprKind::Err(guar)) } + fn lower_trait_impl_header( + &mut self, + trait_impl_header: &TraitImplHeader, + ) -> &'hir hir::TraitImplHeader<'hir> { + let TraitImplHeader { constness, safety, polarity, defaultness, ref trait_ref } = + *trait_impl_header; + let constness = self.lower_constness(constness); + let safety = self.lower_safety(safety, hir::Safety::Safe); + let polarity = match polarity { + ImplPolarity::Positive => ImplPolarity::Positive, + ImplPolarity::Negative(s) => ImplPolarity::Negative(self.lower_span(s)), + }; + // `defaultness.has_value()` is never called for an `impl`, always `true` in order + // to not cause an assertion failure inside the `lower_defaultness` function. + let has_val = true; + let (defaultness, defaultness_span) = self.lower_defaultness(defaultness, has_val); + let modifiers = TraitBoundModifiers { + constness: BoundConstness::Never, + asyncness: BoundAsyncness::Normal, + // we don't use this in bound lowering + polarity: BoundPolarity::Positive, + }; + let trait_ref = self.lower_trait_ref( + modifiers, + trait_ref, + ImplTraitContext::Disallowed(ImplTraitPosition::Trait), + ); + + self.arena.alloc(hir::TraitImplHeader { + constness, + safety, + polarity, + defaultness, + defaultness_span, + trait_ref, + }) + } + fn lower_impl_item( &mut self, i: &AssocItem, @@ -992,7 +1013,12 @@ impl<'hir> LoweringContext<'_, 'hir> { let has_value = true; let (defaultness, _) = self.lower_defaultness(i.kind.defaultness(), has_value); let hir_id = hir::HirId::make_owner(self.current_hir_id_owner.def_id); - let attrs = self.lower_attrs(hir_id, &i.attrs, i.span); + let attrs = self.lower_attrs( + hir_id, + &i.attrs, + i.span, + Target::from_assoc_item_kind(&i.kind, AssocCtxt::Impl { of_trait: is_in_trait_impl }), + ); let (ident, (generics, kind)) = match &i.kind { AssocItemKind::Const(box ConstItem { @@ -1048,9 +1074,11 @@ impl<'hir> LoweringContext<'_, 'hir> { (*ident, (generics, hir::ImplItemKind::Fn(sig, body_id))) } - AssocItemKind::Type(box TyAlias { ident, generics, where_clauses, ty, .. }) => { + AssocItemKind::Type(box TyAlias { + ident, generics, after_where_clause, ty, .. + }) => { let mut generics = generics.clone(); - add_ty_alias_where_clause(&mut generics, *where_clauses, false); + add_ty_alias_where_clause(&mut generics, after_where_clause, false); ( *ident, self.lower_generics( @@ -1097,20 +1125,31 @@ impl<'hir> LoweringContext<'_, 'hir> { } }; + let span = self.lower_span(i.span); let item = hir::ImplItem { owner_id: hir_id.expect_owner(), ident: self.lower_ident(ident), generics, + impl_kind: if is_in_trait_impl { + ImplItemImplKind::Trait { + defaultness, + trait_item_def_id: self + .resolver + .get_partial_res(i.id) + .and_then(|r| r.expect_full_res().opt_def_id()) + .ok_or_else(|| { + self.dcx().span_delayed_bug( + span, + "could not resolve trait item being implemented", + ) + }), + } + } else { + ImplItemImplKind::Inherent { vis_span: self.lower_span(i.vis.span) } + }, kind, - vis_span: self.lower_span(i.vis.span), - span: self.lower_span(i.span), - defaultness, + span, has_delayed_lints: !self.delayed_lints.is_empty(), - trait_item_def_id: self - .resolver - .get_partial_res(i.id) - .map(|r| r.expect_full_res().opt_def_id()) - .unwrap_or(None), }; self.arena.alloc(item) } @@ -1162,7 +1201,7 @@ impl<'hir> LoweringContext<'_, 'hir> { fn lower_param(&mut self, param: &Param) -> hir::Param<'hir> { let hir_id = self.lower_node_id(param.id); - self.lower_attrs(hir_id, ¶m.attrs, param.span); + self.lower_attrs(hir_id, ¶m.attrs, param.span, Target::Param); hir::Param { hir_id, pat: self.lower_pat(¶m.pat), @@ -1181,76 +1220,9 @@ impl<'hir> LoweringContext<'_, 'hir> { let params = this.arena.alloc_from_iter(decl.inputs.iter().map(|x| this.lower_param(x))); - // Optionally lower the fn contract, which turns: - // - // { body } - // - // into: - // - // { contract_requires(PRECOND); let __postcond = |ret_val| POSTCOND; postcond({ body }) } + // Optionally lower the fn contract if let Some(contract) = contract { - let precond = if let Some(req) = &contract.requires { - // Lower the precondition check intrinsic. - let lowered_req = this.lower_expr_mut(&req); - let req_span = this.mark_span_with_reason( - DesugaringKind::Contract, - lowered_req.span, - None, - ); - let precond = this.expr_call_lang_item_fn_mut( - req_span, - hir::LangItem::ContractCheckRequires, - &*arena_vec![this; lowered_req], - ); - Some(this.stmt_expr(req.span, precond)) - } else { - None - }; - let (postcond, body) = if let Some(ens) = &contract.ensures { - let ens_span = this.lower_span(ens.span); - let ens_span = - this.mark_span_with_reason(DesugaringKind::Contract, ens_span, None); - // Set up the postcondition `let` statement. - let check_ident: Ident = - Ident::from_str_and_span("__ensures_checker", ens_span); - let (checker_pat, check_hir_id) = this.pat_ident_binding_mode_mut( - ens_span, - check_ident, - hir::BindingMode::NONE, - ); - let lowered_ens = this.lower_expr_mut(&ens); - let postcond_checker = this.expr_call_lang_item_fn( - ens_span, - hir::LangItem::ContractBuildCheckEnsures, - &*arena_vec![this; lowered_ens], - ); - let postcond = this.stmt_let_pat( - None, - ens_span, - Some(postcond_checker), - this.arena.alloc(checker_pat), - hir::LocalSource::Contract, - ); - - // Install contract_ensures so we will intercept `return` statements, - // then lower the body. - this.contract_ensures = Some((ens_span, check_ident, check_hir_id)); - let body = this.arena.alloc(body(this)); - - // Finally, inject an ensures check on the implicit return of the body. - let body = this.inject_ensures_check(body, ens_span, check_ident, check_hir_id); - (Some(postcond), body) - } else { - let body = &*this.arena.alloc(body(this)); - (None, body) - }; - // Flatten the body into precond, then postcond, then wrapped body. - let wrapped_body = this.block_all( - body.span, - this.arena.alloc_from_iter([precond, postcond].into_iter().flatten()), - Some(body), - ); - (params, this.expr_block(wrapped_body)) + (params, this.lower_contract(body, contract)) } else { (params, body(this)) } @@ -1576,7 +1548,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let safety = self.lower_safety(h.safety, default_safety); // Treat safe `#[target_feature]` functions as unsafe, but also remember that we did so. - let safety = if find_attr!(attrs, AttributeKind::TargetFeature { .. }) + let safety = if find_attr!(attrs, AttributeKind::TargetFeature { was_forced: false, .. }) && safety.is_safe() && !self.tcx.sess.target.is_like_wasm { @@ -1842,7 +1814,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ) -> hir::WherePredicate<'hir> { let hir_id = self.lower_node_id(pred.id); let span = self.lower_span(pred.span); - self.lower_attrs(hir_id, &pred.attrs, span); + self.lower_attrs(hir_id, &pred.attrs, span, Target::WherePredicate); let kind = self.arena.alloc(match &pred.kind { WherePredicateKind::BoundPredicate(WhereBoundPredicate { bound_generic_params, diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 189c82b614c2..dd458ab1ea70 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -51,10 +51,11 @@ use rustc_data_structures::tagged_ptr::TaggedRef; use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle}; use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res}; use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId}; +use rustc_hir::definitions::{DefPathData, DisambiguatorState}; use rustc_hir::lints::DelayedLint; use rustc_hir::{ self as hir, AngleBrackets, ConstArg, GenericArg, HirId, ItemLocalMap, LifetimeSource, - LifetimeSyntax, ParamName, TraitCandidate, + LifetimeSyntax, ParamName, Target, TraitCandidate, }; use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_macros::extension; @@ -77,6 +78,7 @@ macro_rules! arena_vec { mod asm; mod block; +mod contract; mod delegation; mod errors; mod expr; @@ -92,6 +94,7 @@ rustc_fluent_macro::fluent_messages! { "../messages.ftl" } struct LoweringContext<'a, 'hir> { tcx: TyCtxt<'hir>, resolver: &'a mut ResolverAstLowering, + disambiguator: DisambiguatorState, /// Used to allocate HIR nodes. arena: &'hir hir::Arena<'hir>, @@ -135,9 +138,11 @@ struct LoweringContext<'a, 'hir> { #[cfg(debug_assertions)] node_id_to_local_id: NodeMap, + allow_contracts: Arc<[Symbol]>, allow_try_trait: Arc<[Symbol]>, allow_gen_future: Arc<[Symbol]>, allow_pattern_type: Arc<[Symbol]>, + allow_async_gen: Arc<[Symbol]>, allow_async_iterator: Arc<[Symbol]>, allow_for_await: Arc<[Symbol]>, allow_async_fn_traits: Arc<[Symbol]>, @@ -154,6 +159,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // Pseudo-globals. tcx, resolver, + disambiguator: DisambiguatorState::new(), arena: tcx.hir_arena, // HirId handling. @@ -179,6 +185,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { current_item: None, impl_trait_defs: Vec::new(), impl_trait_bounds: Vec::new(), + allow_contracts: [sym::contracts_internals].into(), allow_try_trait: [sym::try_trait_v2, sym::yeet_desugar_details].into(), allow_pattern_type: [sym::pattern_types, sym::pattern_type_range_trait].into(), allow_gen_future: if tcx.features().async_fn_track_caller() { @@ -186,8 +193,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } else { [sym::gen_future].into() }, - allow_for_await: [sym::async_iterator].into(), + allow_for_await: [sym::async_gen_internals, sym::async_iterator].into(), allow_async_fn_traits: [sym::async_fn_traits].into(), + allow_async_gen: [sym::async_gen_internals].into(), // FIXME(gen_blocks): how does `closure_track_caller`/`async_fn_track_caller` // interact with `gen`/`async gen` blocks allow_async_iterator: [sym::gen_future, sym::async_iterator].into(), @@ -296,6 +304,8 @@ enum RelaxedBoundPolicy<'a> { enum RelaxedBoundForbiddenReason { TraitObjectTy, SuperTrait, + TraitAlias, + AssocTyBounds, LateBoundVarsInScope, } @@ -543,6 +553,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { node_id: ast::NodeId, name: Option, def_kind: DefKind, + def_path_data: DefPathData, span: Span, ) -> LocalDefId { let parent = self.current_hir_id_owner.def_id; @@ -558,7 +569,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let def_id = self .tcx .at(span) - .create_def(parent, name, def_kind, None, &mut self.resolver.disambiguator) + .create_def(parent, name, def_kind, Some(def_path_data), &mut self.disambiguator) .def_id(); debug!("create_def: def_id_to_node_id[{:?}] <-> {:?}", def_id, node_id); @@ -675,7 +686,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let bodies = SortedMap::from_presorted_elements(bodies); // Don't hash unless necessary, because it's expensive. - let (opt_hash_including_bodies, attrs_hash, delayed_lints_hash) = + let rustc_middle::hir::Hashes { opt_hash_including_bodies, attrs_hash, delayed_lints_hash } = self.tcx.hash_owner_nodes(node, &bodies, &attrs, &delayed_lints, define_opaque); let num_nodes = self.item_local_id_counter.as_usize(); let (nodes, parenting) = index::index_hir(self.tcx, node, &bodies, num_nodes); @@ -843,6 +854,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { param, Some(kw::UnderscoreLifetime), DefKind::LifetimeParam, + DefPathData::DesugaredAnonymousLifetime, ident.span, ); debug!(?_def_id); @@ -942,11 +954,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { id: HirId, attrs: &[Attribute], target_span: Span, + target: Target, ) -> &'hir [hir::Attribute] { if attrs.is_empty() { &[] } else { - let lowered_attrs = self.lower_attrs_vec(attrs, self.lower_span(target_span), id); + let lowered_attrs = + self.lower_attrs_vec(attrs, self.lower_span(target_span), id, target); assert_eq!(id.owner, self.current_hir_id_owner); let ret = self.arena.alloc_from_iter(lowered_attrs); @@ -971,12 +985,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { attrs: &[Attribute], target_span: Span, target_hir_id: HirId, + target: Target, ) -> Vec { let l = self.span_lowerer(); self.attribute_parser.parse_attribute_list( attrs, target_span, target_hir_id, + target, OmitDoc::Lower, |s| l.lower(s), |l| { @@ -1109,9 +1125,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { &*self.arena.alloc(self.ty(constraint.span, hir::TyKind::Err(guar))); hir::AssocItemConstraintKind::Equality { term: err_ty.into() } } else { - // FIXME(#135229): These should be forbidden! - let bounds = - self.lower_param_bounds(bounds, RelaxedBoundPolicy::Allowed, itctx); + let bounds = self.lower_param_bounds( + bounds, + RelaxedBoundPolicy::Forbidden(RelaxedBoundForbiddenReason::AssocTyBounds), + itctx, + ); hir::AssocItemConstraintKind::Bound { bounds } } } @@ -1217,7 +1235,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { fn lower_path_ty( &mut self, t: &Ty, - qself: &Option>, + qself: &Option>, path: &Path, param_mode: ParamMode, itctx: ImplTraitContext, @@ -1939,7 +1957,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let (name, kind) = self.lower_generic_param_kind(param, source); let hir_id = self.lower_node_id(param.id); - self.lower_attrs(hir_id, ¶m.attrs, param.span()); + self.lower_attrs(hir_id, ¶m.attrs, param.span(), Target::Param); hir::GenericParam { hir_id, def_id: self.local_def_id(param.id), @@ -2021,7 +2039,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { ( hir::ParamName::Plain(self.lower_ident(param.ident)), - hir::GenericParamKind::Const { ty, default, synthetic: false }, + hir::GenericParamKind::Const { ty, default }, ) } } @@ -2078,12 +2096,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { span: Span, rbp: RelaxedBoundPolicy<'_>, ) { - // Even though feature `more_maybe_bounds` bypasses the given policy and (currently) enables - // relaxed bounds in every conceivable position[^1], we don't want to advertise it to the user - // (via a feature gate) since it's super internal. Besides this, it'd be quite distracting. + // Even though feature `more_maybe_bounds` enables the user to relax all default bounds + // other than `Sized` in a lot more positions (thereby bypassing the given policy), we don't + // want to advertise it to the user (via a feature gate error) since it's super internal. // - // [^1]: Strictly speaking, this is incorrect (at the very least for `Sized`) because it's - // no longer fully consistent with default trait elaboration in HIR ty lowering. + // FIXME(more_maybe_bounds): Moreover, if we actually were to add proper default traits + // (like a hypothetical `Move` or `Leak`) we would want to validate the location according + // to default trait elaboration in HIR ty lowering (which depends on the specific trait in + // question: E.g., `?Sized` & `?Move` most likely won't be allowed in all the same places). match rbp { RelaxedBoundPolicy::Allowed => return, @@ -2094,37 +2114,47 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { { return; } - if self.tcx.features().more_maybe_bounds() { - return; - } } RelaxedBoundPolicy::Forbidden(reason) => { - if self.tcx.features().more_maybe_bounds() { - return; - } + let gate = |context, subject| { + let extended = self.tcx.features().more_maybe_bounds(); + let is_sized = trait_ref + .trait_def_id() + .is_some_and(|def_id| self.tcx.is_lang_item(def_id, hir::LangItem::Sized)); + + if extended && !is_sized { + return; + } + + let prefix = if extended { "`Sized` " } else { "" }; + let mut diag = self.dcx().struct_span_err( + span, + format!("relaxed {prefix}bounds are not permitted in {context}"), + ); + if is_sized { + diag.note(format!( + "{subject} are not implicitly bounded by `Sized`, \ + so there is nothing to relax" + )); + } + diag.emit(); + }; match reason { RelaxedBoundForbiddenReason::TraitObjectTy => { - self.dcx().span_err( - span, - "relaxed bounds are not permitted in trait object types", - ); + gate("trait object types", "trait object types"); return; } RelaxedBoundForbiddenReason::SuperTrait => { - let mut diag = self.dcx().struct_span_err( - span, - "relaxed bounds are not permitted in supertrait bounds", - ); - if let Some(def_id) = trait_ref.trait_def_id() - && self.tcx.is_lang_item(def_id, hir::LangItem::Sized) - { - diag.note("traits are `?Sized` by default"); - } - diag.emit(); + gate("supertrait bounds", "traits"); return; } - RelaxedBoundForbiddenReason::LateBoundVarsInScope => {} + RelaxedBoundForbiddenReason::TraitAlias => { + gate("trait alias bounds", "trait aliases"); + return; + } + RelaxedBoundForbiddenReason::AssocTyBounds + | RelaxedBoundForbiddenReason::LateBoundVarsInScope => {} }; } } @@ -2133,7 +2163,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { .struct_span_err(span, "this relaxed bound is not permitted here") .with_note( "in this context, relaxed bounds are only allowed on \ - type parameters defined by the closest item", + type parameters defined on the closest item", ) .emit(); } @@ -2269,7 +2299,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // We're lowering a const argument that was originally thought to be a type argument, // so the def collector didn't create the def ahead of time. That's why we have to do // it here. - let def_id = self.create_def(node_id, None, DefKind::AnonConst, span); + let def_id = self.create_def( + node_id, + None, + DefKind::AnonConst, + DefPathData::LateAnonConst, + span, + ); let hir_id = self.lower_node_id(node_id); let path_expr = Expr { @@ -2499,8 +2535,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { lang_item: hir::LangItem, fields: &'hir [hir::PatField<'hir>], ) -> &'hir hir::Pat<'hir> { - let qpath = hir::QPath::LangItem(lang_item, self.lower_span(span)); - self.pat(span, hir::PatKind::Struct(qpath, fields, false)) + let path = self.make_lang_item_qpath(lang_item, self.lower_span(span), None); + self.pat(span, hir::PatKind::Struct(path, fields, None)) } fn pat_ident(&mut self, span: Span, ident: Ident) -> (&'hir hir::Pat<'hir>, HirId) { diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs index e44406210481..815338c84fa6 100644 --- a/compiler/rustc_ast_lowering/src/pat.rs +++ b/compiler/rustc_ast_lowering/src/pat.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use rustc_ast::ptr::P; use rustc_ast::*; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir::def::{DefKind, Res}; -use rustc_hir::{self as hir, LangItem}; +use rustc_hir::definitions::DefPathData; +use rustc_hir::{self as hir, LangItem, Target}; use rustc_middle::span_bug; use rustc_span::source_map::{Spanned, respan}; use rustc_span::{DesugaringKind, Ident, Span}; @@ -94,7 +94,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { let fs = self.arena.alloc_from_iter(fields.iter().map(|f| { let hir_id = self.lower_node_id(f.id); - self.lower_attrs(hir_id, &f.attrs, f.span); + self.lower_attrs(hir_id, &f.attrs, f.span, Target::PatField); hir::PatField { hir_id, @@ -107,10 +107,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { break hir::PatKind::Struct( qpath, fs, - matches!( - etc, - ast::PatFieldsRest::Rest | ast::PatFieldsRest::Recovered(_) - ), + match etc { + ast::PatFieldsRest::Rest(sp) => Some(self.lower_span(*sp)), + ast::PatFieldsRest::Recovered(_) => Some(Span::default()), + _ => None, + }, ); } PatKind::Tuple(pats) => { @@ -143,7 +144,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } // return inner to be processed in next loop PatKind::Paren(inner) => pattern = inner, - PatKind::MacCall(_) => panic!("{:?} shouldn't exist here", pattern.span), + PatKind::MacCall(_) => { + panic!("{pattern:#?} shouldn't exist here") + } PatKind::Err(guar) => break hir::PatKind::Err(*guar), } }; @@ -154,7 +157,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { fn lower_pat_tuple( &mut self, - pats: &[P], + pats: &[Pat], ctx: &str, ) -> (&'hir [hir::Pat<'hir>], hir::DotDotPos) { let mut elems = Vec::with_capacity(pats.len()); @@ -209,7 +212,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { /// When encountering `($binding_mode $ident @)? ..` (`slice`), /// this is interpreted as a sub-slice pattern semantically. /// Patterns that follow, which are not like `slice` -- or an error occurs, are in `after`. - fn lower_pat_slice(&mut self, pats: &[P]) -> hir::PatKind<'hir> { + fn lower_pat_slice(&mut self, pats: &[Pat]) -> hir::PatKind<'hir> { let mut before = Vec::new(); let mut after = Vec::new(); let mut slice = None; @@ -460,6 +463,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { ) }), ), + TyPatKind::NotNull => hir::TyPatKind::NotNull, TyPatKind::Or(variants) => { hir::TyPatKind::Or(self.arena.alloc_from_iter( variants.iter().map(|pat| self.lower_ty_pat_mut(pat, base_type)), @@ -524,7 +528,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { // We're generating a range end that didn't exist in the AST, // so the def collector didn't create the def ahead of time. That's why we have to do // it here. - let def_id = self.create_def(node_id, None, DefKind::AnonConst, span); + let def_id = + self.create_def(node_id, None, DefKind::AnonConst, DefPathData::LateAnonConst, span); let hir_id = self.lower_node_id(node_id); let unstable_span = self.mark_span_with_reason( diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs index c80ef275c801..3322e0fb66b4 100644 --- a/compiler/rustc_ast_lowering/src/path.rs +++ b/compiler/rustc_ast_lowering/src/path.rs @@ -24,7 +24,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { pub(crate) fn lower_qpath( &mut self, id: NodeId, - qself: &Option>, + qself: &Option>, p: &Path, param_mode: ParamMode, allow_return_type_notation: AllowReturnTypeNotation, diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml index 1940628b44a5..3e04f8b11ec9 100644 --- a/compiler/rustc_ast_passes/Cargo.toml +++ b/compiler/rustc_ast_passes/Cargo.toml @@ -15,7 +15,6 @@ rustc_errors = { path = "../rustc_errors" } rustc_feature = { path = "../rustc_feature" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } rustc_macros = { path = "../rustc_macros" } -rustc_parse = { path = "../rustc_parse" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_target = { path = "../rustc_target" } diff --git a/compiler/rustc_ast_passes/messages.ftl b/compiler/rustc_ast_passes/messages.ftl index af93d55c8982..5e10c5a77d97 100644 --- a/compiler/rustc_ast_passes/messages.ftl +++ b/compiler/rustc_ast_passes/messages.ftl @@ -17,9 +17,13 @@ ast_passes_abi_must_not_have_parameters_or_return_type= ast_passes_abi_must_not_have_return_type= invalid signature for `extern {$abi}` function - .note = functions with the "custom" ABI cannot have a return type + .note = functions with the {$abi} ABI cannot have a return type .help = remove the return type +ast_passes_abi_x86_interrupt = + invalid signature for `extern "x86-interrupt"` function + .note = functions with the "x86-interrupt" ABI must be have either 1 or 2 parameters (but found {$param_count}) + ast_passes_assoc_const_without_body = associated constant in `impl` without body .suggestion = provide a definition for the constant @@ -32,6 +36,13 @@ ast_passes_assoc_type_without_body = associated type in `impl` without body .suggestion = provide a definition for the type +ast_passes_async_fn_in_const_trait_or_trait_impl = + async functions are not allowed in `const` {$in_impl -> + [true] trait impls + *[false] traits + } + .label = associated functions of `const` cannot be declared `async` + ast_passes_at_least_one_trait = at least one trait must be specified ast_passes_auto_generic = auto traits cannot have generic parameters @@ -40,14 +51,12 @@ ast_passes_auto_generic = auto traits cannot have generic parameters ast_passes_auto_items = auto traits cannot have associated items .label = {ast_passes_auto_items} - .suggestion = remove these associated items + .suggestion = remove the associated items ast_passes_auto_super_lifetime = auto traits cannot have super traits or lifetime bounds .label = {ast_passes_auto_super_lifetime} .suggestion = remove the super traits or lifetime bounds -ast_passes_bad_c_variadic = only foreign, `unsafe extern "C"`, or `unsafe extern "C-unwind"` functions may have a C-variadic arg - ast_passes_body_in_extern = incorrect `{$kind}` inside `extern` block .cannot_have = cannot have a body .invalid = the invalid body @@ -55,6 +64,17 @@ ast_passes_body_in_extern = incorrect `{$kind}` inside `extern` block ast_passes_bound_in_context = bounds on `type`s in {$ctx} have no effect +ast_passes_c_variadic_bad_extern = `...` is not supported for `extern "{$abi}"` functions + .label = `extern "{$abi}"` because of this + .help = only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list + +ast_passes_c_variadic_must_be_unsafe = + functions with a C variable argument list must be unsafe + .suggestion = add the `unsafe` keyword to this definition + +ast_passes_c_variadic_no_extern = `...` is not supported for non-extern functions + .help = only `extern "C"` and `extern "C-unwind"` functions may have a C variable argument list + ast_passes_const_and_c_variadic = functions cannot be both `const` and C-variadic .const = `const` because of this .variadic = C-variadic because of this @@ -73,6 +93,10 @@ ast_passes_const_without_body = ast_passes_constraint_on_negative_bound = associated type constraints not allowed on negative bounds +ast_passes_coroutine_and_c_variadic = functions cannot be both `{$coroutine_kind}` and C-variadic + .const = `{$coroutine_kind}` because of this + .variadic = C-variadic because of this + ast_passes_equality_in_where = equality constraints are not yet supported in `where` clauses .label = not supported .suggestion = if `{$ident}` is an associated type you're trying to set, use the associated type binding syntax @@ -102,6 +126,10 @@ ast_passes_extern_without_abi = `extern` declarations without an explicit ABI ar .suggestion = specify an ABI .help = prior to Rust 2024, a default ABI was inferred +ast_passes_extern_without_abi_sugg = `extern` declarations without an explicit ABI are deprecated + .label = ABI should be specified here + .suggestion = explicitly specify the {$default_abi} ABI + ast_passes_feature_on_non_nightly = `#![feature]` may not be used on the {$channel} release channel .suggestion = remove the attribute .stable_since = the feature `{$name}` has been stable since `{$since}` and no longer requires an attribute to enable @@ -175,11 +203,6 @@ ast_passes_generic_default_trailing = generic parameters with a default must be ast_passes_incompatible_features = `{$f1}` and `{$f2}` are incompatible, using them at the same time is not allowed .help = remove one of these features -ast_passes_inherent_cannot_be = inherent impls cannot be {$annotation} - .because = {$annotation} because of this - .type = inherent impl for this type - .only_trait = only trait implementations may be annotated with {$annotation} - ast_passes_item_invalid_safety = items outside of `unsafe extern {"{ }"}` cannot be declared with `safe` safety qualifier .suggestion = remove safe from this item @@ -191,6 +214,10 @@ ast_passes_match_arm_with_no_body = .suggestion = add a body after the pattern ast_passes_missing_unsafe_on_extern = extern blocks must be unsafe + .suggestion = needs `unsafe` before the extern keyword + +ast_passes_missing_unsafe_on_extern_lint = extern blocks should be unsafe + .suggestion = needs `unsafe` before the extern keyword ast_passes_module_nonascii = trying to load file for module `{$name}` with non-ascii identifier name .help = consider using the `#[path]` attribute to specify filesystem path @@ -241,6 +268,10 @@ ast_passes_tilde_const_disallowed = `[const]` is not allowed here .trait_assoc_ty = associated types in non-`const` traits cannot have `[const]` trait bounds .trait_impl_assoc_ty = associated types in non-const impls cannot have `[const]` trait bounds .inherent_assoc_ty = inherent associated types cannot have `[const]` trait bounds + .struct = structs cannot have `[const]` trait bounds + .enum = enums cannot have `[const]` trait bounds + .union = unions cannot have `[const]` trait bounds + .anon_const = anonymous constants cannot have `[const]` trait bounds .object = trait objects cannot have `[const]` trait bounds .item = this item cannot have `[const]` trait bounds diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs index 895a457ec1d5..6b218f34363d 100644 --- a/compiler/rustc_ast_passes/src/ast_validation.rs +++ b/compiler/rustc_ast_passes/src/ast_validation.rs @@ -22,20 +22,19 @@ use std::str::FromStr; use itertools::{Either, Itertools}; use rustc_abi::{CanonAbi, ExternAbi, InterruptKind}; -use rustc_ast::ptr::P; use rustc_ast::visit::{AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor, walk_list}; use rustc_ast::*; use rustc_ast_pretty::pprust::{self, State}; +use rustc_attr_parsing::validate_attr; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{DiagCtxtHandle, LintBuffer}; use rustc_feature::Features; -use rustc_parse::validate_attr; use rustc_session::Session; +use rustc_session::lint::BuiltinLintDiag; use rustc_session::lint::builtin::{ DEPRECATED_WHERE_CLAUSE_LOCATION, MISSING_ABI, MISSING_UNSAFE_ON_EXTERN, PATTERNS_IN_FNS_WITHOUT_BODY, }; -use rustc_session::lint::{BuiltinLintDiag, LintBuffer}; use rustc_span::{Ident, Span, kw, sym}; use rustc_target::spec::{AbiMap, AbiMapping}; use thin_vec::thin_vec; @@ -146,25 +145,24 @@ impl<'a> AstValidator<'a> { &mut self, ty_alias: &TyAlias, ) -> Result<(), errors::WhereClauseBeforeTypeAlias> { - if ty_alias.ty.is_none() || !ty_alias.where_clauses.before.has_where_token { + if ty_alias.ty.is_none() || !ty_alias.generics.where_clause.has_where_token { return Ok(()); } - let (before_predicates, after_predicates) = - ty_alias.generics.where_clause.predicates.split_at(ty_alias.where_clauses.split); - let span = ty_alias.where_clauses.before.span; + let span = ty_alias.generics.where_clause.span; - let sugg = if !before_predicates.is_empty() || !ty_alias.where_clauses.after.has_where_token + let sugg = if !ty_alias.generics.where_clause.predicates.is_empty() + || !ty_alias.after_where_clause.has_where_token { let mut state = State::new(); - if !ty_alias.where_clauses.after.has_where_token { + if !ty_alias.after_where_clause.has_where_token { state.space(); state.word_space("where"); } - let mut first = after_predicates.is_empty(); - for p in before_predicates { + let mut first = ty_alias.after_where_clause.predicates.is_empty(); + for p in &ty_alias.generics.where_clause.predicates { if !first { state.word_space(","); } @@ -175,7 +173,7 @@ impl<'a> AstValidator<'a> { errors::WhereClauseBeforeTypeAliasSugg::Move { left: span, snippet: state.s.eof(), - right: ty_alias.where_clauses.after.span.shrink_to_hi(), + right: ty_alias.after_where_clause.span.shrink_to_hi(), } } else { errors::WhereClauseBeforeTypeAliasSugg::Remove { span } @@ -294,6 +292,21 @@ impl<'a> AstValidator<'a> { }); } + fn check_async_fn_in_const_trait_or_impl(&self, sig: &FnSig, parent: &TraitOrTraitImpl) { + let Some(const_keyword) = parent.constness() else { return }; + + let Some(CoroutineKind::Async { span: async_keyword, .. }) = sig.header.coroutine_kind + else { + return; + }; + + self.dcx().emit_err(errors::AsyncFnInConstTraitOrTraitImpl { + async_keyword, + in_impl: matches!(parent, TraitOrTraitImpl::TraitImpl { .. }), + const_keyword, + }); + } + fn check_fn_decl(&self, fn_decl: &FnDecl, self_semantic: SelfSemantic) { self.check_decl_num_args(fn_decl); self.check_decl_cvariadic_pos(fn_decl); @@ -391,7 +404,24 @@ impl<'a> AstValidator<'a> { if let InterruptKind::X86 = interrupt_kind { // "x86-interrupt" is special because it does have arguments. // FIXME(workingjubilee): properly lint on acceptable input types. - if let FnRetTy::Ty(ref ret_ty) = sig.decl.output { + let inputs = &sig.decl.inputs; + let param_count = inputs.len(); + if !matches!(param_count, 1 | 2) { + let mut spans: Vec = + inputs.iter().map(|arg| arg.span).collect(); + if spans.is_empty() { + spans = vec![sig.span]; + } + self.dcx().emit_err(errors::AbiX86Interrupt { spans, param_count }); + } + + if let FnRetTy::Ty(ref ret_ty) = sig.decl.output + && match &ret_ty.kind { + TyKind::Never => false, + TyKind::Tup(tup) if tup.is_empty() => false, + _ => true, + } + { self.dcx().emit_err(errors::AbiMustNotHaveReturnType { span: ret_ty.span, abi, @@ -450,12 +480,18 @@ impl<'a> AstValidator<'a> { fn reject_params_or_return(&self, abi: ExternAbi, ident: &Ident, sig: &FnSig) { let mut spans: Vec<_> = sig.decl.inputs.iter().map(|p| p.span).collect(); - if let FnRetTy::Ty(ref ret_ty) = sig.decl.output { + if let FnRetTy::Ty(ref ret_ty) = sig.decl.output + && match &ret_ty.kind { + TyKind::Never => false, + TyKind::Tup(tup) if tup.is_empty() => false, + _ => true, + } + { spans.push(ret_ty.span); } if !spans.is_empty() { - let header_span = sig.header.span().unwrap_or(sig.span.shrink_to_lo()); + let header_span = sig.header_span(); let suggestion_span = header_span.shrink_to_hi().to(sig.decl.output.span()); let padding = if header_span.is_empty() { "" } else { " " }; @@ -529,11 +565,7 @@ impl<'a> AstValidator<'a> { self.dcx().emit_err(errors::BoundInContext { span, ctx }); } - fn check_foreign_ty_genericless( - &self, - generics: &Generics, - where_clauses: &TyAliasWhereClauses, - ) { + fn check_foreign_ty_genericless(&self, generics: &Generics, after_where_clause: &WhereClause) { let cannot_have = |span, descr, remove_descr| { self.dcx().emit_err(errors::ExternTypesCannotHave { span, @@ -547,14 +579,14 @@ impl<'a> AstValidator<'a> { cannot_have(generics.span, "generic parameters", "generic parameters"); } - let check_where_clause = |where_clause: TyAliasWhereClause| { + let check_where_clause = |where_clause: &WhereClause| { if where_clause.has_where_token { cannot_have(where_clause.span, "`where` clauses", "`where` clause"); } }; - check_where_clause(where_clauses.before); - check_where_clause(where_clauses.after); + check_where_clause(&generics.where_clause); + check_where_clause(&after_where_clause); } fn check_foreign_kind_bodyless(&self, ident: Ident, kind: &str, body_span: Option) { @@ -628,46 +660,68 @@ impl<'a> AstValidator<'a> { /// - Non-const /// - Either foreign, or free and `unsafe extern "C"` semantically fn check_c_variadic_type(&self, fk: FnKind<'a>) { - let variadic_spans: Vec<_> = fk - .decl() - .inputs - .iter() - .filter(|arg| matches!(arg.ty.kind, TyKind::CVarArgs)) - .map(|arg| arg.span) - .collect(); + // `...` is already rejected when it is not the final parameter. + let variadic_param = match fk.decl().inputs.last() { + Some(param) if matches!(param.ty.kind, TyKind::CVarArgs) => param, + _ => return, + }; - if variadic_spans.is_empty() { - return; - } + let FnKind::Fn(fn_ctxt, _, Fn { sig, .. }) = fk else { + // Unreachable because the parser already rejects `...` in closures. + unreachable!("C variable argument list cannot be used in closures") + }; - if let Some(header) = fk.header() - && let Const::Yes(const_span) = header.constness - { - let mut spans = variadic_spans.clone(); - spans.push(const_span); + // C-variadics are not yet implemented in const evaluation. + if let Const::Yes(const_span) = sig.header.constness { self.dcx().emit_err(errors::ConstAndCVariadic { - spans, + spans: vec![const_span, variadic_param.span], const_span, - variadic_spans: variadic_spans.clone(), + variadic_span: variadic_param.span, }); } - match (fk.ctxt(), fk.header()) { - (Some(FnCtxt::Foreign), _) => return, - (Some(FnCtxt::Free), Some(header)) => match header.ext { - Extern::Explicit(StrLit { symbol_unescaped: sym::C, .. }, _) - | Extern::Explicit(StrLit { symbol_unescaped: sym::C_dash_unwind, .. }, _) - | Extern::Implicit(_) - if matches!(header.safety, Safety::Unsafe(_)) => - { - return; - } - _ => {} - }, - _ => {} - }; + if let Some(coroutine_kind) = sig.header.coroutine_kind { + self.dcx().emit_err(errors::CoroutineAndCVariadic { + spans: vec![coroutine_kind.span(), variadic_param.span], + coroutine_kind: coroutine_kind.as_str(), + coroutine_span: coroutine_kind.span(), + variadic_span: variadic_param.span, + }); + } - self.dcx().emit_err(errors::BadCVariadic { span: variadic_spans }); + match fn_ctxt { + FnCtxt::Foreign => return, + FnCtxt::Free | FnCtxt::Assoc(_) => match sig.header.ext { + Extern::Implicit(_) => { + if !matches!(sig.header.safety, Safety::Unsafe(_)) { + self.dcx().emit_err(errors::CVariadicMustBeUnsafe { + span: variadic_param.span, + unsafe_span: sig.safety_span(), + }); + } + } + Extern::Explicit(StrLit { symbol_unescaped, .. }, _) => { + if !matches!(symbol_unescaped, sym::C | sym::C_dash_unwind) { + self.dcx().emit_err(errors::CVariadicBadExtern { + span: variadic_param.span, + abi: symbol_unescaped, + extern_span: sig.extern_span(), + }); + } + + if !matches!(sig.header.safety, Safety::Unsafe(_)) { + self.dcx().emit_err(errors::CVariadicMustBeUnsafe { + span: variadic_param.span, + unsafe_span: sig.safety_span(), + }); + } + } + Extern::None => { + let err = errors::CVariadicNoExtern { span: variadic_param.span }; + self.dcx().emit_err(err); + } + }, + } } fn check_item_named(&self, ident: Ident, kind: &str) { @@ -699,23 +753,27 @@ impl<'a> AstValidator<'a> { } } - fn deny_super_traits(&self, bounds: &GenericBounds, ident_span: Span) { + fn deny_super_traits(&self, bounds: &GenericBounds, ident: Span) { if let [.., last] = &bounds[..] { - let span = ident_span.shrink_to_hi().to(last.span()); - self.dcx().emit_err(errors::AutoTraitBounds { span, ident: ident_span }); + let span = bounds.iter().map(|b| b.span()).collect(); + let removal = ident.shrink_to_hi().to(last.span()); + self.dcx().emit_err(errors::AutoTraitBounds { span, removal, ident }); } } - fn deny_where_clause(&self, where_clause: &WhereClause, ident_span: Span) { + fn deny_where_clause(&self, where_clause: &WhereClause, ident: Span) { if !where_clause.predicates.is_empty() { // FIXME: The current diagnostic is misleading since it only talks about // super trait and lifetime bounds while we should just say “bounds”. - self.dcx() - .emit_err(errors::AutoTraitBounds { span: where_clause.span, ident: ident_span }); + self.dcx().emit_err(errors::AutoTraitBounds { + span: vec![where_clause.span], + removal: where_clause.span, + ident, + }); } } - fn deny_items(&self, trait_items: &[P], ident_span: Span) { + fn deny_items(&self, trait_items: &[Box], ident_span: Span) { if !trait_items.is_empty() { let spans: Vec<_> = trait_items.iter().map(|i| i.kind.ident().unwrap().span).collect(); let total = trait_items.first().unwrap().span.to(trait_items.last().unwrap().span); @@ -835,7 +893,7 @@ impl<'a> AstValidator<'a> { MISSING_ABI, id, span, - BuiltinLintDiag::MissingAbi(span, ExternAbi::FALLBACK), + errors::MissingAbiSugg { span, default_abi: ExternAbi::FALLBACK }, ) } } @@ -951,13 +1009,16 @@ impl<'a> Visitor<'a> for AstValidator<'a> { } match &item.kind { - ItemKind::Impl(box Impl { - safety, - polarity, - defaultness: _, - constness, + ItemKind::Impl(Impl { generics, - of_trait: Some(t), + of_trait: + Some(box TraitImplHeader { + safety, + polarity, + defaultness: _, + constness, + trait_ref: t, + }), self_ty, items, }) => { @@ -989,46 +1050,12 @@ impl<'a> Visitor<'a> for AstValidator<'a> { walk_list!(this, visit_assoc_item, items, AssocCtxt::Impl { of_trait: true }); }); } - ItemKind::Impl(box Impl { - safety, - polarity, - defaultness, - constness, - generics, - of_trait: None, - self_ty, - items, - }) => { - let error = |annotation_span, annotation, only_trait| errors::InherentImplCannot { - span: self_ty.span, - annotation_span, - annotation, - self_ty: self_ty.span, - only_trait, - }; - + ItemKind::Impl(Impl { generics, of_trait: None, self_ty, items }) => { self.visit_attrs_vis(&item.attrs, &item.vis); self.visibility_not_permitted( &item.vis, errors::VisibilityNotPermittedNote::IndividualImplItems, ); - if let &Safety::Unsafe(span) = safety { - self.dcx().emit_err(errors::InherentImplCannotUnsafe { - span: self_ty.span, - annotation_span: span, - annotation: "unsafe", - self_ty: self_ty.span, - }); - } - if let &ImplPolarity::Negative(span) = polarity { - self.dcx().emit_err(error(span, "negative", false)); - } - if let &Defaultness::Default(def_span) = defaultness { - self.dcx().emit_err(error(def_span, "`default`", true)); - } - if let &Const::Yes(span) = constness { - self.dcx().emit_err(error(span, "`const`", true)); - } self.with_tilde_const(Some(TildeConstReason::Impl { span: item.span }), |this| { this.visit_generics(generics) @@ -1094,7 +1121,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { MISSING_UNSAFE_ON_EXTERN, item.id, item.span, - BuiltinLintDiag::MissingUnsafeOnExtern { + errors::MissingUnsafeOnExternLint { suggestion: item.span.shrink_to_lo(), }, ); @@ -1124,7 +1151,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> { ); } } - visit::walk_item(self, item) + self.with_tilde_const(Some(TildeConstReason::Enum { span: item.span }), |this| { + visit::walk_item(this, item) + }); } ItemKind::Trait(box Trait { constness, @@ -1163,38 +1192,52 @@ impl<'a> Visitor<'a> for AstValidator<'a> { walk_list!(this, visit_assoc_item, items, AssocCtxt::Trait); }); } + ItemKind::TraitAlias(box TraitAlias { constness, generics, bounds, .. }) => { + let disallowed = matches!(constness, ast::Const::No) + .then(|| TildeConstReason::Trait { span: item.span }); + self.with_tilde_const(disallowed, |this| { + this.visit_generics(generics); + walk_list!(this, visit_param_bound, bounds, BoundKind::SuperTraits) + }); + } ItemKind::Mod(safety, ident, mod_kind) => { if let &Safety::Unsafe(span) = safety { self.dcx().emit_err(errors::UnsafeItem { span, kind: "module" }); } // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584). - if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _, _)) + if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _)) && !attr::contains_name(&item.attrs, sym::path) { self.check_mod_file_item_asciionly(*ident); } visit::walk_item(self, item) } - ItemKind::Struct(ident, generics, vdata) => match vdata { - VariantData::Struct { fields, .. } => { - self.visit_attrs_vis_ident(&item.attrs, &item.vis, ident); - self.visit_generics(generics); - walk_list!(self, visit_field_def, fields); - } - _ => visit::walk_item(self, item), - }, + ItemKind::Struct(ident, generics, vdata) => { + self.with_tilde_const(Some(TildeConstReason::Struct { span: item.span }), |this| { + match vdata { + VariantData::Struct { fields, .. } => { + this.visit_attrs_vis_ident(&item.attrs, &item.vis, ident); + this.visit_generics(generics); + walk_list!(this, visit_field_def, fields); + } + _ => visit::walk_item(this, item), + } + }) + } ItemKind::Union(ident, generics, vdata) => { if vdata.fields().is_empty() { self.dcx().emit_err(errors::FieldlessUnion { span: item.span }); } - match vdata { - VariantData::Struct { fields, .. } => { - self.visit_attrs_vis_ident(&item.attrs, &item.vis, ident); - self.visit_generics(generics); - walk_list!(self, visit_field_def, fields); + self.with_tilde_const(Some(TildeConstReason::Union { span: item.span }), |this| { + match vdata { + VariantData::Struct { fields, .. } => { + this.visit_attrs_vis_ident(&item.attrs, &item.vis, ident); + this.visit_generics(generics); + walk_list!(this, visit_field_def, fields); + } + _ => visit::walk_item(this, item), } - _ => visit::walk_item(self, item), - } + }); } ItemKind::Const(box ConstItem { defaultness, expr, .. }) => { self.check_defaultness(item.span, *defaultness); @@ -1221,7 +1264,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { visit::walk_item(self, item); } ItemKind::TyAlias( - ty_alias @ box TyAlias { defaultness, bounds, where_clauses, ty, .. }, + ty_alias @ box TyAlias { defaultness, bounds, after_where_clause, ty, .. }, ) => { self.check_defaultness(item.span, *defaultness); if ty.is_none() { @@ -1236,9 +1279,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> { if let Err(err) = self.check_type_alias_where_clause_location(ty_alias) { self.dcx().emit_err(err); } - } else if where_clauses.after.has_where_token { + } else if after_where_clause.has_where_token { self.dcx().emit_err(errors::WhereClauseAfterTypeAlias { - span: where_clauses.after.span, + span: after_where_clause.span, help: self.sess.is_nightly_build(), }); } @@ -1268,7 +1311,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { defaultness, ident, generics, - where_clauses, + after_where_clause, bounds, ty, .. @@ -1276,7 +1319,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { self.check_defaultness(fi.span, *defaultness); self.check_foreign_kind_bodyless(*ident, "type", ty.as_ref().map(|b| b.span)); self.check_type_no_bounds(bounds, "`extern` blocks"); - self.check_foreign_ty_genericless(generics, where_clauses); + self.check_foreign_ty_genericless(generics, after_where_clause); self.check_foreign_item_ascii_only(*ident); } ForeignItemKind::Static(box StaticItem { ident, safety, expr, .. }) => { @@ -1586,6 +1629,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> { self.visibility_not_permitted(&item.vis, errors::VisibilityNotPermittedNote::TraitImpl); if let AssocItemKind::Fn(box Fn { sig, .. }) = &item.kind { self.check_trait_fn_not_const(sig.header.constness, parent); + self.check_async_fn_in_const_trait_or_impl(sig, parent); } } @@ -1623,6 +1667,13 @@ impl<'a> Visitor<'a> for AstValidator<'a> { _ => self.with_in_trait_impl(None, |this| visit::walk_assoc_item(this, item, ctxt)), } } + + fn visit_anon_const(&mut self, anon_const: &'a AnonConst) { + self.with_tilde_const( + Some(TildeConstReason::AnonConst { span: anon_const.value.span }), + |this| visit::walk_anon_const(this, anon_const), + ) + } } /// When encountering an equality constraint in a `where` clause, emit an error. If the code seems @@ -1754,7 +1805,7 @@ fn deny_equality_constraints( .map(|segment| segment.ident.name) .zip(poly.trait_ref.path.segments.iter().map(|segment| segment.ident.name)) .all(|(a, b)| a == b) - && let Some(potential_assoc) = full_path.segments.iter().last() + && let Some(potential_assoc) = full_path.segments.last() { suggest(poly, potential_assoc, predicate); } diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs index fd4b2528541e..fd75e999d138 100644 --- a/compiler/rustc_ast_passes/src/errors.rs +++ b/compiler/rustc_ast_passes/src/errors.rs @@ -4,7 +4,7 @@ use rustc_abi::ExternAbi; use rustc_ast::ParamKindOrd; use rustc_errors::codes::*; use rustc_errors::{Applicability, Diag, EmissionGuarantee, Subdiagnostic}; -use rustc_macros::{Diagnostic, Subdiagnostic}; +use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_span::{Ident, Span, Symbol}; use crate::fluent_generated as fluent; @@ -62,6 +62,16 @@ pub(crate) struct TraitFnConst { pub make_trait_const_sugg: Option, } +#[derive(Diagnostic)] +#[diag(ast_passes_async_fn_in_const_trait_or_trait_impl)] +pub(crate) struct AsyncFnInConstTraitOrTraitImpl { + #[primary_span] + pub async_keyword: Span, + pub in_impl: bool, + #[label] + pub const_keyword: Span, +} + #[derive(Diagnostic)] #[diag(ast_passes_forbidden_bound)] pub(crate) struct ForbiddenBound { @@ -309,10 +319,37 @@ pub(crate) struct ExternItemAscii { } #[derive(Diagnostic)] -#[diag(ast_passes_bad_c_variadic)] -pub(crate) struct BadCVariadic { +#[diag(ast_passes_c_variadic_no_extern)] +#[help] +pub(crate) struct CVariadicNoExtern { #[primary_span] - pub span: Vec, + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(ast_passes_c_variadic_must_be_unsafe)] +pub(crate) struct CVariadicMustBeUnsafe { + #[primary_span] + pub span: Span, + + #[suggestion( + ast_passes_suggestion, + applicability = "maybe-incorrect", + code = "unsafe ", + style = "verbose" + )] + pub unsafe_span: Span, +} + +#[derive(Diagnostic)] +#[diag(ast_passes_c_variadic_bad_extern)] +#[help] +pub(crate) struct CVariadicBadExtern { + #[primary_span] + pub span: Span, + pub abi: Symbol, + #[label] + pub extern_span: Span, } #[derive(Diagnostic)] @@ -344,7 +381,7 @@ pub(crate) struct ModuleNonAscii { #[diag(ast_passes_auto_generic, code = E0567)] pub(crate) struct AutoTraitGeneric { #[primary_span] - #[suggestion(code = "", applicability = "machine-applicable")] + #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")] pub span: Span, #[label] pub ident: Span, @@ -354,8 +391,9 @@ pub(crate) struct AutoTraitGeneric { #[diag(ast_passes_auto_super_lifetime, code = E0568)] pub(crate) struct AutoTraitBounds { #[primary_span] - #[suggestion(code = "", applicability = "machine-applicable")] - pub span: Span, + pub span: Vec, + #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")] + pub removal: Span, #[label] pub ident: Span, } @@ -365,7 +403,7 @@ pub(crate) struct AutoTraitBounds { pub(crate) struct AutoTraitItems { #[primary_span] pub spans: Vec, - #[suggestion(code = "", applicability = "machine-applicable")] + #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")] pub total: Span, #[label] pub ident: Span, @@ -463,32 +501,6 @@ pub(crate) struct UnsafeNegativeImpl { pub r#unsafe: Span, } -#[derive(Diagnostic)] -#[diag(ast_passes_inherent_cannot_be)] -pub(crate) struct InherentImplCannot<'a> { - #[primary_span] - pub span: Span, - #[label(ast_passes_because)] - pub annotation_span: Span, - pub annotation: &'a str, - #[label(ast_passes_type)] - pub self_ty: Span, - #[note(ast_passes_only_trait)] - pub only_trait: bool, -} - -#[derive(Diagnostic)] -#[diag(ast_passes_inherent_cannot_be, code = E0197)] -pub(crate) struct InherentImplCannotUnsafe<'a> { - #[primary_span] - pub span: Span, - #[label(ast_passes_because)] - pub annotation_span: Span, - pub annotation: &'a str, - #[label(ast_passes_type)] - pub self_ty: Span, -} - #[derive(Diagnostic)] #[diag(ast_passes_unsafe_item)] pub(crate) struct UnsafeItem { @@ -504,6 +516,13 @@ pub(crate) struct MissingUnsafeOnExtern { pub span: Span, } +#[derive(LintDiagnostic)] +#[diag(ast_passes_missing_unsafe_on_extern_lint)] +pub(crate) struct MissingUnsafeOnExternLint { + #[suggestion(code = "unsafe ", applicability = "machine-applicable")] + pub suggestion: Span, +} + #[derive(Diagnostic)] #[diag(ast_passes_fieldless_union)] pub(crate) struct FieldlessUnion { @@ -623,6 +642,26 @@ pub(crate) enum TildeConstReason { #[primary_span] span: Span, }, + #[note(ast_passes_struct)] + Struct { + #[primary_span] + span: Span, + }, + #[note(ast_passes_enum)] + Enum { + #[primary_span] + span: Span, + }, + #[note(ast_passes_union)] + Union { + #[primary_span] + span: Span, + }, + #[note(ast_passes_anon_const)] + AnonConst { + #[primary_span] + span: Span, + }, #[note(ast_passes_object)] TraitObject, #[note(ast_passes_item)] @@ -651,7 +690,19 @@ pub(crate) struct ConstAndCVariadic { #[label(ast_passes_const)] pub const_span: Span, #[label(ast_passes_variadic)] - pub variadic_spans: Vec, + pub variadic_span: Span, +} + +#[derive(Diagnostic)] +#[diag(ast_passes_coroutine_and_c_variadic)] +pub(crate) struct CoroutineAndCVariadic { + #[primary_span] + pub spans: Vec, + pub coroutine_kind: &'static str, + #[label(ast_passes_const)] + pub coroutine_span: Span, + #[label(ast_passes_variadic)] + pub variadic_span: Span, } #[derive(Diagnostic)] @@ -810,6 +861,14 @@ pub(crate) struct MissingAbi { pub span: Span, } +#[derive(LintDiagnostic)] +#[diag(ast_passes_extern_without_abi_sugg)] +pub(crate) struct MissingAbiSugg { + #[suggestion(code = "extern {default_abi}", applicability = "machine-applicable")] + pub span: Span, + pub default_abi: ExternAbi, +} + #[derive(Diagnostic)] #[diag(ast_passes_abi_custom_safe_foreign_function)] pub(crate) struct AbiCustomSafeForeignFunction { @@ -886,3 +945,12 @@ pub(crate) struct AbiMustNotHaveReturnType { pub span: Span, pub abi: ExternAbi, } + +#[derive(Diagnostic)] +#[diag(ast_passes_abi_x86_interrupt)] +#[note] +pub(crate) struct AbiX86Interrupt { + #[primary_span] + pub spans: Vec, + pub param_count: usize, +} diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs index 662357ce8841..b8a29a9a08f4 100644 --- a/compiler/rustc_ast_passes/src/feature_gate.rs +++ b/compiler/rustc_ast_passes/src/feature_gate.rs @@ -183,11 +183,12 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { gate_doc!( "experimental" { cfg => doc_cfg - cfg_hide => doc_cfg_hide + auto_cfg => doc_cfg masked => doc_masked notable_trait => doc_notable_trait } "meant for internal use only" { + attribute => rustdoc_internals keyword => rustdoc_internals fake_variadic => rustdoc_internals search_unbox => rustdoc_internals @@ -217,18 +218,18 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } } - ast::ItemKind::Impl(box ast::Impl { polarity, defaultness, of_trait, .. }) => { - if let &ast::ImplPolarity::Negative(span) = polarity { + ast::ItemKind::Impl(ast::Impl { of_trait: Some(of_trait), .. }) => { + if let ast::ImplPolarity::Negative(span) = of_trait.polarity { gate!( &self, negative_impls, - span.to(of_trait.as_ref().map_or(span, |t| t.path.span)), + span.to(of_trait.trait_ref.path.span), "negative trait bounds are not fully implemented; \ use marker types for now" ); } - if let ast::Defaultness::Default(_) = defaultness { + if let ast::Defaultness::Default(_) = of_trait.defaultness { gate!(&self, specialization, i.span, "specialization is unstable"); } } @@ -524,6 +525,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) { gate_all!(where_clause_attrs, "attributes in `where` clause are unstable"); gate_all!(super_let, "`super let` is experimental"); gate_all!(frontmatter, "frontmatters are experimental"); + gate_all!(coroutines, "coroutine syntax is experimental"); if !visitor.features.never_patterns() { if let Some(spans) = spans.get(&sym::never_patterns) { @@ -620,11 +622,7 @@ fn maybe_stage_features(sess: &Session, features: &Features, krate: &ast::Crate) } fn check_incompatible_features(sess: &Session, features: &Features) { - let enabled_lang_features = - features.enabled_lang_features().iter().map(|feat| (feat.gate_name, feat.attr_sp)); - let enabled_lib_features = - features.enabled_lib_features().iter().map(|feat| (feat.gate_name, feat.attr_sp)); - let enabled_features = enabled_lang_features.chain(enabled_lib_features); + let enabled_features = features.enabled_features_iter_stable_order(); for (f1, f2) in rustc_feature::INCOMPATIBLE_FEATURES .iter() diff --git a/compiler/rustc_ast_pretty/src/pp.rs b/compiler/rustc_ast_pretty/src/pp.rs index 8a0dbadf18cd..4108671a3629 100644 --- a/compiler/rustc_ast_pretty/src/pp.rs +++ b/compiler/rustc_ast_pretty/src/pp.rs @@ -298,7 +298,7 @@ impl Printer { } } - // This is is where `BoxMarker`s are produced. + // This is where `BoxMarker`s are produced. fn scan_begin(&mut self, token: BeginToken) -> BoxMarker { if self.scan_stack.is_empty() { self.left_total = 1; @@ -310,7 +310,7 @@ impl Printer { BoxMarker } - // This is is where `BoxMarker`s are consumed. + // This is where `BoxMarker`s are consumed. fn scan_end(&mut self, b: BoxMarker) { if self.scan_stack.is_empty() { self.print_end(); diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index f0cf0c1487f7..93f4e47342f1 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -10,8 +10,7 @@ use std::borrow::Cow; use std::sync::Arc; use rustc_ast::attr::AttrIdGenerator; -use rustc_ast::ptr::P; -use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind}; +use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree}; use rustc_ast::util::classify; use rustc_ast::util::comments::{Comment, CommentStyle}; @@ -442,7 +441,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool); fn print_ident(&mut self, ident: Ident) { - self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string()); + self.word(IdentPrinter::for_ast_ident(ident, ident.guess_print_mode()).to_string()); self.ann_post(ident) } @@ -1016,17 +1015,16 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere /* Name components */ token::Ident(name, is_raw) => { - IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into() + IdentPrinter::new(name, is_raw.to_print_mode_ident(), convert_dollar_crate) + .to_string() + .into() } token::NtIdent(ident, is_raw) => { - IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into() + IdentPrinter::for_ast_ident(ident, is_raw.to_print_mode_ident()).to_string().into() } - token::Lifetime(name, IdentIsRaw::No) - | token::NtLifetime(Ident { name, .. }, IdentIsRaw::No) => name.to_string().into(), - token::Lifetime(name, IdentIsRaw::Yes) - | token::NtLifetime(Ident { name, .. }, IdentIsRaw::Yes) => { - format!("'r#{}", &name.as_str()[1..]).into() + token::Lifetime(name, is_raw) | token::NtLifetime(Ident { name, .. }, is_raw) => { + IdentPrinter::new(name, is_raw.to_print_mode_lifetime(), None).to_string().into() } /* Other */ @@ -1178,7 +1176,7 @@ impl<'a> State<'a> { self.end(rb); } - fn commasep_exprs(&mut self, b: Breaks, exprs: &[P]) { + fn commasep_exprs(&mut self, b: Breaks, exprs: &[Box]) { self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e, FixupContext::default()), |e| e.span) } @@ -1234,6 +1232,7 @@ impl<'a> State<'a> { self.print_expr_anon_const(end, &[]); } } + rustc_ast::TyPatKind::NotNull => self.word("!null"), rustc_ast::TyPatKind::Or(variants) => { let mut first = true; for pat in variants { @@ -1713,10 +1712,15 @@ impl<'a> State<'a> { if mutbl.is_mut() { self.word_nbsp("mut"); } - if let ByRef::Yes(rmutbl) = by_ref { + if let ByRef::Yes(pinnedness, rmutbl) = by_ref { self.word_nbsp("ref"); + if pinnedness.is_pinned() { + self.word_nbsp("pin"); + } if rmutbl.is_mut() { self.word_nbsp("mut"); + } else if pinnedness.is_pinned() { + self.word_nbsp("const"); } } self.print_ident(*ident); @@ -1771,7 +1775,7 @@ impl<'a> State<'a> { }, |f| f.pat.span, ); - if let ast::PatFieldsRest::Rest | ast::PatFieldsRest::Recovered(_) = etc { + if let ast::PatFieldsRest::Rest(_) | ast::PatFieldsRest::Recovered(_) = etc { if !fields.is_empty() { self.word_space(","); } diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs index 8a2cb64b2a08..bdf73ac32f0d 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs @@ -2,7 +2,6 @@ use std::fmt::Write; use ast::{ForLoopKind, MatchKind}; use itertools::{Itertools, Position}; -use rustc_ast::ptr::P; use rustc_ast::util::classify; use rustc_ast::util::literal::escape_byte_str_symbol; use rustc_ast::util::parser::{self, ExprPrecedence, Fixity}; @@ -54,7 +53,7 @@ impl<'a> State<'a> { self.print_else(elseopt) } - fn print_call_post(&mut self, args: &[P]) { + fn print_call_post(&mut self, args: &[Box]) { self.popen(); self.commasep_exprs(Inconsistent, args); self.pclose() @@ -111,7 +110,7 @@ impl<'a> State<'a> { } } - fn print_expr_vec(&mut self, exprs: &[P]) { + fn print_expr_vec(&mut self, exprs: &[Box]) { let ib = self.ibox(INDENT_UNIT); self.word("["); self.commasep_exprs(Inconsistent, exprs); @@ -149,7 +148,7 @@ impl<'a> State<'a> { fn print_expr_struct( &mut self, - qself: &Option>, + qself: &Option>, path: &ast::Path, fields: &[ast::ExprField], rest: &ast::StructRest, @@ -204,7 +203,7 @@ impl<'a> State<'a> { self.word("}"); } - fn print_expr_tup(&mut self, exprs: &[P]) { + fn print_expr_tup(&mut self, exprs: &[Box]) { self.popen(); self.commasep_exprs(Inconsistent, exprs); if exprs.len() == 1 { @@ -213,7 +212,7 @@ impl<'a> State<'a> { self.pclose() } - fn print_expr_call(&mut self, func: &ast::Expr, args: &[P], fixup: FixupContext) { + fn print_expr_call(&mut self, func: &ast::Expr, args: &[Box], fixup: FixupContext) { // Independent of parenthesization related to precedence, we must // parenthesize `func` if this is a statement context in which without // parentheses, a statement boundary would occur inside `func` or @@ -247,7 +246,7 @@ impl<'a> State<'a> { &mut self, segment: &ast::PathSegment, receiver: &ast::Expr, - base_args: &[P], + base_args: &[Box], fixup: FixupContext, ) { // The fixup here is different than in `print_expr_call` because diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs index 11c97a552c69..294d03a83b16 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs @@ -1,8 +1,6 @@ use ast::StaticItem; use itertools::{Itertools, Position}; -use rustc_ast as ast; -use rustc_ast::ModKind; -use rustc_ast::ptr::P; +use rustc_ast::{self as ast, ModKind, TraitAlias}; use rustc_span::Ident; use crate::pp::BoxMarker; @@ -60,14 +58,14 @@ impl<'a> State<'a> { defaultness, ident, generics, - where_clauses, + after_where_clause, bounds, ty, }) => { self.print_associated_type( *ident, generics, - *where_clauses, + after_where_clause, bounds, ty.as_deref(), vis, @@ -128,14 +126,12 @@ impl<'a> State<'a> { &mut self, ident: Ident, generics: &ast::Generics, - where_clauses: ast::TyAliasWhereClauses, + after_where_clause: &ast::WhereClause, bounds: &ast::GenericBounds, ty: Option<&ast::Ty>, vis: &ast::Visibility, defaultness: ast::Defaultness, ) { - let (before_predicates, after_predicates) = - generics.where_clause.predicates.split_at(where_clauses.split); let (cb, ib) = self.head(""); self.print_visibility(vis); self.print_defaultness(defaultness); @@ -146,13 +142,13 @@ impl<'a> State<'a> { self.word_nbsp(":"); self.print_type_bounds(bounds); } - self.print_where_clause_parts(where_clauses.before.has_where_token, before_predicates); + self.print_where_clause(&generics.where_clause); if let Some(ty) = ty { self.space(); self.word_space("="); self.print_type(ty); } - self.print_where_clause_parts(where_clauses.after.has_where_token, after_predicates); + self.print_where_clause(&after_where_clause); self.word(";"); self.end(ib); self.end(cb); @@ -284,14 +280,14 @@ impl<'a> State<'a> { defaultness, ident, generics, - where_clauses, + after_where_clause, bounds, ty, }) => { self.print_associated_type( *ident, generics, - *where_clauses, + after_where_clause, bounds, ty.as_deref(), &item.vis, @@ -309,39 +305,41 @@ impl<'a> State<'a> { let (cb, ib) = self.head(visibility_qualified(&item.vis, "union")); self.print_struct(struct_def, generics, *ident, item.span, true, cb, ib); } - ast::ItemKind::Impl(box ast::Impl { - safety, - polarity, - defaultness, - constness, - generics, - of_trait, - self_ty, - items, - }) => { + ast::ItemKind::Impl(ast::Impl { generics, of_trait, self_ty, items }) => { let (cb, ib) = self.head(""); self.print_visibility(&item.vis); - self.print_defaultness(*defaultness); - self.print_safety(*safety); - self.word("impl"); - if generics.params.is_empty() { - self.nbsp(); - } else { - self.print_generic_params(&generics.params); - self.space(); - } + let impl_generics = |this: &mut Self| { + this.word("impl"); - self.print_constness(*constness); + if generics.params.is_empty() { + this.nbsp(); + } else { + this.print_generic_params(&generics.params); + this.space(); + } + }; - if let ast::ImplPolarity::Negative(_) = polarity { - self.word("!"); - } - - if let Some(t) = of_trait { - self.print_trait_ref(t); + if let Some(box of_trait) = of_trait { + let ast::TraitImplHeader { + defaultness, + safety, + constness, + polarity, + ref trait_ref, + } = *of_trait; + self.print_defaultness(defaultness); + self.print_safety(safety); + impl_generics(self); + self.print_constness(constness); + if let ast::ImplPolarity::Negative(_) = polarity { + self.word("!"); + } + self.print_trait_ref(trait_ref); self.space(); self.word_space("for"); + } else { + impl_generics(self); } self.print_type(self_ty); @@ -387,8 +385,11 @@ impl<'a> State<'a> { let empty = item.attrs.is_empty() && items.is_empty(); self.bclose(item.span, empty, cb); } - ast::ItemKind::TraitAlias(ident, generics, bounds) => { - let (cb, ib) = self.head(visibility_qualified(&item.vis, "trait")); + ast::ItemKind::TraitAlias(box TraitAlias { constness, ident, generics, bounds }) => { + let (cb, ib) = self.head(""); + self.print_visibility(&item.vis); + self.print_constness(*constness); + self.word_nbsp("trait"); self.print_ident(*ident); self.print_generic_params(&generics.params); self.nbsp(); @@ -584,14 +585,14 @@ impl<'a> State<'a> { defaultness, ident, generics, - where_clauses, + after_where_clause, bounds, ty, }) => { self.print_associated_type( *ident, generics, - *where_clauses, + after_where_clause, bounds, ty.as_deref(), vis, @@ -628,10 +629,10 @@ impl<'a> State<'a> { &mut self, attrs: &[ast::Attribute], vis: &ast::Visibility, - qself: &Option>, + qself: &Option>, path: &ast::Path, kind: DelegationKind<'_>, - body: &Option>, + body: &Option>, ) { let body_cb_ib = body.as_ref().map(|body| (body, self.head(""))); self.print_visibility(vis); @@ -758,14 +759,7 @@ impl<'a> State<'a> { } fn print_where_clause(&mut self, where_clause: &ast::WhereClause) { - self.print_where_clause_parts(where_clause.has_where_token, &where_clause.predicates); - } - - fn print_where_clause_parts( - &mut self, - has_where_token: bool, - predicates: &[ast::WherePredicate], - ) { + let ast::WhereClause { has_where_token, ref predicates, span: _ } = *where_clause; if predicates.is_empty() && !has_where_token { return; } diff --git a/compiler/rustc_attr_data_structures/Cargo.toml b/compiler/rustc_attr_data_structures/Cargo.toml deleted file mode 100644 index b18923c337ff..000000000000 --- a/compiler/rustc_attr_data_structures/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "rustc_attr_data_structures" -version = "0.0.0" -edition = "2024" - -[dependencies] -# tidy-alphabetical-start -rustc_abi = {path = "../rustc_abi"} -rustc_ast = {path = "../rustc_ast"} -rustc_ast_pretty = {path = "../rustc_ast_pretty"} -rustc_data_structures = {path = "../rustc_data_structures"} -rustc_macros = {path = "../rustc_macros"} -rustc_serialize = {path = "../rustc_serialize"} -rustc_span = {path = "../rustc_span"} -thin-vec = "0.2.12" -# tidy-alphabetical-end diff --git a/compiler/rustc_attr_data_structures/src/attributes.rs b/compiler/rustc_attr_data_structures/src/attributes.rs deleted file mode 100644 index 55019cd57a71..000000000000 --- a/compiler/rustc_attr_data_structures/src/attributes.rs +++ /dev/null @@ -1,472 +0,0 @@ -use rustc_abi::Align; -use rustc_ast::token::CommentKind; -use rustc_ast::{self as ast, AttrStyle}; -use rustc_macros::{Decodable, Encodable, HashStable_Generic, PrintAttribute}; -use rustc_span::hygiene::Transparency; -use rustc_span::{Ident, Span, Symbol}; -use thin_vec::ThinVec; - -use crate::{DefaultBodyStability, PartialConstStability, PrintAttribute, RustcVersion, Stability}; - -#[derive(Copy, Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic, PrintAttribute)] -pub enum InlineAttr { - None, - Hint, - Always, - Never, - /// `#[rustc_force_inline]` forces inlining to happen in the MIR inliner - it reports an error - /// if the inlining cannot happen. It is limited to only free functions so that the calls - /// can always be resolved. - Force { - attr_span: Span, - reason: Option, - }, -} - -impl InlineAttr { - pub fn always(&self) -> bool { - match self { - InlineAttr::Always | InlineAttr::Force { .. } => true, - InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false, - } - } -} - -#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq, HashStable_Generic)] -pub enum InstructionSetAttr { - ArmA32, - ArmT32, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq, Default, PrintAttribute)] -#[derive(Encodable, Decodable, HashStable_Generic)] -pub enum OptimizeAttr { - /// No `#[optimize(..)]` attribute - #[default] - Default, - /// `#[optimize(none)]` - DoNotOptimize, - /// `#[optimize(speed)]` - Speed, - /// `#[optimize(size)]` - Size, -} - -impl OptimizeAttr { - pub fn do_not_optimize(&self) -> bool { - matches!(self, Self::DoNotOptimize) - } -} - -#[derive(PartialEq, Debug, Encodable, Decodable, Copy, Clone, HashStable_Generic, PrintAttribute)] -pub enum ReprAttr { - ReprInt(IntType), - ReprRust, - ReprC, - ReprPacked(Align), - ReprSimd, - ReprTransparent, - ReprAlign(Align), -} -pub use ReprAttr::*; -use rustc_span::def_id::DefId; - -pub enum TransparencyError { - UnknownTransparency(Symbol, Span), - MultipleTransparencyAttrs(Span, Span), -} - -#[derive(Eq, PartialEq, Debug, Copy, Clone)] -#[derive(Encodable, Decodable, HashStable_Generic, PrintAttribute)] -pub enum IntType { - SignedInt(ast::IntTy), - UnsignedInt(ast::UintTy), -} - -#[derive(Copy, Debug, Encodable, Decodable, Clone, HashStable_Generic, PrintAttribute)] -pub struct Deprecation { - pub since: DeprecatedSince, - /// The note to issue a reason. - pub note: Option, - /// A text snippet used to completely replace any use of the deprecated item in an expression. - /// - /// This is currently unstable. - pub suggestion: Option, -} - -/// Release in which an API is deprecated. -#[derive(Copy, Debug, Encodable, Decodable, Clone, HashStable_Generic, PrintAttribute)] -pub enum DeprecatedSince { - RustcVersion(RustcVersion), - /// Deprecated in the future ("to be determined"). - Future, - /// `feature(staged_api)` is off. Deprecation versions outside the standard - /// library are allowed to be arbitrary strings, for better or worse. - NonStandard(Symbol), - /// Deprecation version is unspecified but optional. - Unspecified, - /// Failed to parse a deprecation version, or the deprecation version is - /// unspecified and required. An error has already been emitted. - Err, -} - -/// Successfully-parsed value of a `#[coverage(..)]` attribute. -#[derive(Copy, Debug, Eq, PartialEq, Encodable, Decodable, Clone)] -#[derive(HashStable_Generic, PrintAttribute)] -pub enum CoverageAttrKind { - On, - Off, -} - -impl Deprecation { - /// Whether an item marked with #[deprecated(since = "X")] is currently - /// deprecated (i.e., whether X is not greater than the current rustc - /// version). - pub fn is_in_effect(&self) -> bool { - match self.since { - DeprecatedSince::RustcVersion(since) => since <= RustcVersion::CURRENT, - DeprecatedSince::Future => false, - // The `since` field doesn't have semantic purpose without `#![staged_api]`. - DeprecatedSince::NonStandard(_) => true, - // Assume deprecation is in effect if "since" field is absent or invalid. - DeprecatedSince::Unspecified | DeprecatedSince::Err => true, - } - } - - pub fn is_since_rustc_version(&self) -> bool { - matches!(self.since, DeprecatedSince::RustcVersion(_)) - } -} - -/// There are three valid forms of the attribute: -/// `#[used]`, which is semantically equivalent to `#[used(linker)]` except that the latter is currently unstable. -/// `#[used(compiler)]` -/// `#[used(linker)]` -#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, Hash)] -#[derive(HashStable_Generic, PrintAttribute)] -pub enum UsedBy { - Compiler, - Linker, -} - -#[derive(Encodable, Decodable, Clone, Debug, PartialEq, Eq, Hash)] -#[derive(HashStable_Generic, PrintAttribute)] -pub enum MacroUseArgs { - UseAll, - UseSpecific(ThinVec), -} - -impl Default for MacroUseArgs { - fn default() -> Self { - Self::UseSpecific(ThinVec::new()) - } -} - -#[derive(Debug, Clone, Encodable, Decodable, HashStable_Generic)] -pub struct StrippedCfgItem { - pub parent_module: ModId, - pub ident: Ident, - pub cfg: (CfgEntry, Span), -} - -impl StrippedCfgItem { - pub fn map_mod_id(self, f: impl FnOnce(ModId) -> New) -> StrippedCfgItem { - StrippedCfgItem { parent_module: f(self.parent_module), ident: self.ident, cfg: self.cfg } - } -} - -#[derive(Encodable, Decodable, Clone, Debug, PartialEq, Eq, Hash)] -#[derive(HashStable_Generic, PrintAttribute)] -pub enum CfgEntry { - All(ThinVec, Span), - Any(ThinVec, Span), - Not(Box, Span), - Bool(bool, Span), - NameValue { name: Symbol, name_span: Span, value: Option<(Symbol, Span)>, span: Span }, - Version(Option, Span), -} - -/// Represents parsed *built-in* inert attributes. -/// -/// ## Overview -/// These attributes are markers that guide the compilation process and are never expanded into other code. -/// They persist throughout the compilation phases, from AST to HIR and beyond. -/// -/// ## Attribute Processing -/// While attributes are initially parsed by [`rustc_parse`] into [`ast::Attribute`], they still contain raw token streams -/// because different attributes have different internal structures. This enum represents the final, -/// fully parsed form of these attributes, where each variant contains all the information and -/// structure relevant for the specific attribute. -/// -/// Some attributes can be applied multiple times to the same item, and they are "collapsed" into a single -/// semantic attribute. For example: -/// ```rust -/// #[repr(C)] -/// #[repr(packed)] -/// struct S { } -/// ``` -/// This is equivalent to `#[repr(C, packed)]` and results in a single [`AttributeKind::Repr`] containing -/// both `C` and `packed` annotations. This collapsing happens during parsing and is reflected in the -/// data structures defined in this enum. -/// -/// ## Usage -/// These parsed attributes are used throughout the compiler to: -/// - Control code generation (e.g., `#[repr]`) -/// - Mark API stability (`#[stable]`, `#[unstable]`) -/// - Provide documentation (`#[doc]`) -/// - Guide compiler behavior (e.g., `#[allow_internal_unstable]`) -/// -/// ## Note on Attribute Organization -/// Some attributes like `InlineAttr`, `OptimizeAttr`, and `InstructionSetAttr` are defined separately -/// from this enum because they are used in specific compiler phases (like code generation) and don't -/// need to persist throughout the entire compilation process. They are typically processed and -/// converted into their final form earlier in the compilation pipeline. -/// -/// For example: -/// - `InlineAttr` is used during code generation to control function inlining -/// - `OptimizeAttr` is used to control optimization levels -/// - `InstructionSetAttr` is used for target-specific code generation -/// -/// These attributes are handled by their respective compiler passes in the [`rustc_codegen_ssa`] crate -/// and don't need to be preserved in the same way as the attributes in this enum. -/// -/// For more details on attribute parsing, see the [`rustc_attr_parsing`] crate. -/// -/// [`rustc_parse`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_parse/index.html -/// [`rustc_codegen_ssa`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/index.html -/// [`rustc_attr_parsing`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_parsing/index.html -#[derive(Clone, Debug, HashStable_Generic, Encodable, Decodable, PrintAttribute)] -pub enum AttributeKind { - // tidy-alphabetical-start - /// Represents `#[align(N)]`. - // FIXME(#82232, #143834): temporarily renamed to mitigate `#[align]` nameres ambiguity - Align { align: Align, span: Span }, - - /// Represents `#[rustc_allow_const_fn_unstable]`. - AllowConstFnUnstable(ThinVec, Span), - - /// Represents `#[rustc_allow_incoherent_impl]`. - AllowIncoherentImpl(Span), - - /// Represents `#[allow_internal_unstable]`. - AllowInternalUnstable(ThinVec<(Symbol, Span)>, Span), - - /// Represents `#[rustc_as_ptr]` (used by the `dangling_pointers_from_temporaries` lint). - AsPtr(Span), - - /// Represents `#[automatically_derived]` - AutomaticallyDerived(Span), - - /// Represents `#[rustc_default_body_unstable]`. - BodyStability { - stability: DefaultBodyStability, - /// Span of the `#[rustc_default_body_unstable(...)]` attribute - span: Span, - }, - - /// Represents `#[rustc_coherence_is_core]`. - CoherenceIsCore, - - /// Represents `#[rustc_coinductive]`. - Coinductive(Span), - - /// Represents `#[cold]`. - Cold(Span), - - /// Represents `#[rustc_confusables]`. - Confusables { - symbols: ThinVec, - // FIXME(jdonszelmann): remove when target validation code is moved - first_span: Span, - }, - - /// Represents `#[const_continue]`. - ConstContinue(Span), - - /// Represents `#[rustc_const_stable]` and `#[rustc_const_unstable]`. - ConstStability { - stability: PartialConstStability, - /// Span of the `#[rustc_const_stable(...)]` or `#[rustc_const_unstable(...)]` attribute - span: Span, - }, - - /// Represents `#[rustc_const_stable_indirect]`. - ConstStabilityIndirect, - - /// Represents `#[const_trait]`. - ConstTrait(Span), - - /// Represents `#[coverage(..)]`. - Coverage(Span, CoverageAttrKind), - - ///Represents `#[rustc_deny_explicit_impl]`. - DenyExplicitImpl(Span), - - /// Represents [`#[deprecated]`](https://doc.rust-lang.org/stable/reference/attributes/diagnostics.html#the-deprecated-attribute). - Deprecation { deprecation: Deprecation, span: Span }, - - /// Represents `#[rustc_do_not_implement_via_object]`. - DoNotImplementViaObject(Span), - - /// Represents [`#[doc]`](https://doc.rust-lang.org/stable/rustdoc/write-documentation/the-doc-attribute.html). - DocComment { style: AttrStyle, kind: CommentKind, span: Span, comment: Symbol }, - - /// Represents `#[rustc_dummy]`. - Dummy, - - /// Represents [`#[export_name]`](https://doc.rust-lang.org/reference/abi.html#the-export_name-attribute). - ExportName { - /// The name to export this item with. - /// It may not contain \0 bytes as it will be converted to a null-terminated string. - name: Symbol, - span: Span, - }, - - /// Represents `#[export_stable]`. - ExportStable, - - /// Represents `#[ffi_const]`. - FfiConst(Span), - - /// Represents `#[ffi_pure]`. - FfiPure(Span), - - /// Represents `#[fundamental]`. - Fundamental, - - /// Represents `#[ignore]` - Ignore { - span: Span, - /// ignore can optionally have a reason: `#[ignore = "reason this is ignored"]` - reason: Option, - }, - - /// Represents `#[inline]` and `#[rustc_force_inline]`. - Inline(InlineAttr, Span), - - /// Represents `#[link_name]`. - LinkName { name: Symbol, span: Span }, - - /// Represents `#[link_ordinal]`. - LinkOrdinal { ordinal: u16, span: Span }, - - /// Represents [`#[link_section]`](https://doc.rust-lang.org/reference/abi.html#the-link_section-attribute) - LinkSection { name: Symbol, span: Span }, - - /// Represents `#[loop_match]`. - LoopMatch(Span), - - /// Represents `#[macro_escape]`. - MacroEscape(Span), - - /// Represents `#[rustc_macro_transparency]`. - MacroTransparency(Transparency), - - /// Represents `#[macro_use]`. - MacroUse { span: Span, arguments: MacroUseArgs }, - - /// Represents `#[marker]`. - Marker(Span), - - /// Represents [`#[may_dangle]`](https://std-dev-guide.rust-lang.org/tricky/may-dangle.html). - MayDangle(Span), - - /// Represents `#[must_use]`. - MustUse { - span: Span, - /// must_use can optionally have a reason: `#[must_use = "reason this must be used"]` - reason: Option, - }, - - /// Represents `#[naked]` - Naked(Span), - - /// Represents `#[no_implicit_prelude]` - NoImplicitPrelude(Span), - - /// Represents `#[no_mangle]` - NoMangle(Span), - - /// Represents `#[non_exhaustive]` - NonExhaustive(Span), - - /// Represents `#[omit_gdb_pretty_printer_section]` - OmitGdbPrettyPrinterSection, - - /// Represents `#[optimize(size|speed)]` - Optimize(OptimizeAttr, Span), - - /// Represents `#[rustc_paren_sugar]`. - ParenSugar(Span), - - /// Represents `#[rustc_pass_by_value]` (used by the `rustc_pass_by_value` lint). - PassByValue(Span), - - /// Represents `#[path]` - Path(Symbol, Span), - - /// Represents `#[pointee]` - Pointee(Span), - - /// Represents `#[proc_macro]` - ProcMacro(Span), - - /// Represents `#[proc_macro_attribute]` - ProcMacroAttribute(Span), - - /// Represents `#[proc_macro_derive]` - ProcMacroDerive { trait_name: Symbol, helper_attrs: ThinVec, span: Span }, - - /// Represents `#[rustc_pub_transparent]` (used by the `repr_transparent_external_private_fields` lint). - PubTransparent(Span), - - /// Represents [`#[repr]`](https://doc.rust-lang.org/stable/reference/type-layout.html#representations). - Repr { reprs: ThinVec<(ReprAttr, Span)>, first_span: Span }, - - /// Represents `#[rustc_builtin_macro]`. - RustcBuiltinMacro { builtin_name: Option, helper_attrs: ThinVec, span: Span }, - - /// Represents `#[rustc_layout_scalar_valid_range_end]`. - RustcLayoutScalarValidRangeEnd(Box, Span), - - /// Represents `#[rustc_layout_scalar_valid_range_start]`. - RustcLayoutScalarValidRangeStart(Box, Span), - - /// Represents `#[rustc_object_lifetime_default]`. - RustcObjectLifetimeDefault, - - /// Represents `#[rustc_skip_during_method_dispatch]`. - SkipDuringMethodDispatch { array: bool, boxed_slice: bool, span: Span }, - - /// Represents `#[rustc_specialization_trait]`. - SpecializationTrait(Span), - - /// Represents `#[stable]`, `#[unstable]` and `#[rustc_allowed_through_unstable_modules]`. - Stability { - stability: Stability, - /// Span of the attribute. - span: Span, - }, - - /// Represents `#[rustc_std_internal_symbol]`. - StdInternalSymbol(Span), - - /// Represents `#[target_feature(enable = "...")]` - TargetFeature(ThinVec<(Symbol, Span)>, Span), - - /// Represents `#[track_caller]` - TrackCaller(Span), - - /// Represents `#[type_const]`. - TypeConst(Span), - - /// Represents `#[rustc_unsafe_specialization_marker]`. - UnsafeSpecializationMarker(Span), - - /// Represents `#[unstable_feature_bound]`. - UnstableFeatureBound(ThinVec<(Symbol, Span)>), - - /// Represents `#[used]` - Used { used_by: UsedBy, span: Span }, - // tidy-alphabetical-end -} diff --git a/compiler/rustc_attr_data_structures/src/encode_cross_crate.rs b/compiler/rustc_attr_data_structures/src/encode_cross_crate.rs deleted file mode 100644 index af2d46d0bc71..000000000000 --- a/compiler/rustc_attr_data_structures/src/encode_cross_crate.rs +++ /dev/null @@ -1,86 +0,0 @@ -use crate::AttributeKind; - -#[derive(PartialEq)] -pub enum EncodeCrossCrate { - Yes, - No, -} - -impl AttributeKind { - pub fn encode_cross_crate(&self) -> EncodeCrossCrate { - use AttributeKind::*; - use EncodeCrossCrate::*; - - match self { - // tidy-alphabetical-start - Align { .. } => No, - AllowConstFnUnstable(..) => No, - AllowIncoherentImpl(..) => No, - AllowInternalUnstable(..) => Yes, - AsPtr(..) => Yes, - AutomaticallyDerived(..) => Yes, - BodyStability { .. } => No, - CoherenceIsCore => No, - Coinductive(..) => No, - Cold(..) => No, - Confusables { .. } => Yes, - ConstContinue(..) => No, - ConstStability { .. } => Yes, - ConstStabilityIndirect => No, - ConstTrait(..) => No, - Coverage(..) => No, - DenyExplicitImpl(..) => No, - Deprecation { .. } => Yes, - DoNotImplementViaObject(..) => No, - DocComment { .. } => Yes, - Dummy => No, - ExportName { .. } => Yes, - ExportStable => No, - FfiConst(..) => No, - FfiPure(..) => No, - Fundamental { .. } => Yes, - Ignore { .. } => No, - Inline(..) => No, - LinkName { .. } => Yes, // Needed for rustdoc - LinkOrdinal { .. } => No, - LinkSection { .. } => Yes, // Needed for rustdoc - LoopMatch(..) => No, - MacroEscape(..) => No, - MacroTransparency(..) => Yes, - MacroUse { .. } => No, - Marker(..) => No, - MayDangle(..) => No, - MustUse { .. } => Yes, - Naked(..) => No, - NoImplicitPrelude(..) => No, - NoMangle(..) => Yes, // Needed for rustdoc - NonExhaustive(..) => Yes, // Needed for rustdoc - OmitGdbPrettyPrinterSection => No, - Optimize(..) => No, - ParenSugar(..) => No, - PassByValue(..) => Yes, - Path(..) => No, - Pointee(..) => No, - ProcMacro(..) => No, - ProcMacroAttribute(..) => No, - ProcMacroDerive { .. } => No, - PubTransparent(..) => Yes, - Repr { .. } => No, - RustcBuiltinMacro { .. } => Yes, - RustcLayoutScalarValidRangeEnd(..) => Yes, - RustcLayoutScalarValidRangeStart(..) => Yes, - RustcObjectLifetimeDefault => No, - SkipDuringMethodDispatch { .. } => No, - SpecializationTrait(..) => No, - Stability { .. } => Yes, - StdInternalSymbol(..) => No, - TargetFeature(..) => No, - TrackCaller(..) => Yes, - TypeConst(..) => Yes, - UnsafeSpecializationMarker(..) => No, - UnstableFeatureBound(..) => No, - Used { .. } => No, - // tidy-alphabetical-end - } - } -} diff --git a/compiler/rustc_attr_data_structures/src/lib.rs b/compiler/rustc_attr_data_structures/src/lib.rs deleted file mode 100644 index 4c5af805ca90..000000000000 --- a/compiler/rustc_attr_data_structures/src/lib.rs +++ /dev/null @@ -1,216 +0,0 @@ -//! Data structures for representing parsed attributes in the Rust compiler. -//! For detailed documentation about attribute processing, -//! see [rustc_attr_parsing](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_parsing/index.html). - -// tidy-alphabetical-start -#![allow(internal_features)] -#![doc(rust_logo)] -#![feature(rustdoc_internals)] -// tidy-alphabetical-end - -mod attributes; -mod encode_cross_crate; -mod stability; -mod version; - -pub mod lints; - -use std::num::NonZero; - -pub use attributes::*; -pub use encode_cross_crate::EncodeCrossCrate; -use rustc_abi::Align; -use rustc_ast::token::CommentKind; -use rustc_ast::{AttrStyle, IntTy, UintTy}; -use rustc_ast_pretty::pp::Printer; -use rustc_span::hygiene::Transparency; -use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol}; -pub use stability::*; -use thin_vec::ThinVec; -pub use version::*; - -/// Requirements for a `StableHashingContext` to be used in this crate. -/// This is a hack to allow using the `HashStable_Generic` derive macro -/// instead of implementing everything in `rustc_middle`. -pub trait HashStableContext: rustc_ast::HashStableContext + rustc_abi::HashStableContext {} - -/// This trait is used to print attributes in `rustc_hir_pretty`. -/// -/// For structs and enums it can be derived using [`rustc_macros::PrintAttribute`]. -/// The output will look a lot like a `Debug` implementation, but fields of several types -/// like [`Span`]s and empty tuples, are gracefully skipped so they don't clutter the -/// representation much. -pub trait PrintAttribute { - /// Whether or not this will render as something meaningful, or if it's skipped - /// (which will force the containing struct to also skip printing a comma - /// and the field name). - fn should_render(&self) -> bool; - - fn print_attribute(&self, p: &mut Printer); -} - -impl PrintAttribute for u128 { - fn should_render(&self) -> bool { - true - } - - fn print_attribute(&self, p: &mut Printer) { - p.word(self.to_string()) - } -} - -impl PrintAttribute for &T { - fn should_render(&self) -> bool { - T::should_render(self) - } - - fn print_attribute(&self, p: &mut Printer) { - T::print_attribute(self, p) - } -} -impl PrintAttribute for Option { - fn should_render(&self) -> bool { - self.as_ref().is_some_and(|x| x.should_render()) - } - - fn print_attribute(&self, p: &mut Printer) { - if let Some(i) = self { - T::print_attribute(i, p) - } - } -} -impl PrintAttribute for ThinVec { - fn should_render(&self) -> bool { - self.is_empty() || self[0].should_render() - } - - fn print_attribute(&self, p: &mut Printer) { - let mut last_printed = false; - p.word("["); - for i in self { - if last_printed { - p.word_space(","); - } - i.print_attribute(p); - last_printed = i.should_render(); - } - p.word("]"); - } -} -macro_rules! print_skip { - ($($t: ty),* $(,)?) => {$( - impl PrintAttribute for $t { - fn should_render(&self) -> bool { false } - fn print_attribute(&self, _: &mut Printer) { } - })* - }; -} - -macro_rules! print_disp { - ($($t: ty),* $(,)?) => {$( - impl PrintAttribute for $t { - fn should_render(&self) -> bool { true } - fn print_attribute(&self, p: &mut Printer) { - p.word(format!("{}", self)); - } - } - )*}; -} -macro_rules! print_debug { - ($($t: ty),* $(,)?) => {$( - impl PrintAttribute for $t { - fn should_render(&self) -> bool { true } - fn print_attribute(&self, p: &mut Printer) { - p.word(format!("{:?}", self)); - } - } - )*}; -} - -macro_rules! print_tup { - (num_should_render $($ts: ident)*) => { 0 $(+ $ts.should_render() as usize)* }; - () => {}; - ($t: ident $($ts: ident)*) => { - #[allow(non_snake_case, unused)] - impl<$t: PrintAttribute, $($ts: PrintAttribute),*> PrintAttribute for ($t, $($ts),*) { - fn should_render(&self) -> bool { - let ($t, $($ts),*) = self; - print_tup!(num_should_render $t $($ts)*) != 0 - } - - fn print_attribute(&self, p: &mut Printer) { - let ($t, $($ts),*) = self; - let parens = print_tup!(num_should_render $t $($ts)*) > 1; - if parens { - p.popen(); - } - - let mut printed_anything = $t.should_render(); - - $t.print_attribute(p); - - $( - if $ts.should_render() { - if printed_anything { - p.word_space(","); - } - printed_anything = true; - } - $ts.print_attribute(p); - )* - - if parens { - p.pclose(); - } - } - } - - print_tup!($($ts)*); - }; -} - -print_tup!(A B C D E F G H); -print_skip!(Span, (), ErrorGuaranteed); -print_disp!(u16, bool, NonZero); -print_debug!(Symbol, Ident, UintTy, IntTy, Align, AttrStyle, CommentKind, Transparency); - -/// Finds attributes in sequences of attributes by pattern matching. -/// -/// A little like `matches` but for attributes. -/// -/// ```rust,ignore (illustrative) -/// // finds the repr attribute -/// if let Some(r) = find_attr!(attrs, AttributeKind::Repr(r) => r) { -/// -/// } -/// -/// // checks if one has matched -/// if find_attr!(attrs, AttributeKind::Repr(_)) { -/// -/// } -/// ``` -/// -/// Often this requires you to first end up with a list of attributes. -/// A common way to get those is through `tcx.get_all_attrs(did)` -#[macro_export] -macro_rules! find_attr { - ($attributes_list: expr, $pattern: pat $(if $guard: expr)?) => {{ - $crate::find_attr!($attributes_list, $pattern $(if $guard)? => ()).is_some() - }}; - - ($attributes_list: expr, $pattern: pat $(if $guard: expr)? => $e: expr) => {{ - 'done: { - for i in $attributes_list { - let i: &rustc_hir::Attribute = i; - match i { - rustc_hir::Attribute::Parsed($pattern) $(if $guard)? => { - break 'done Some($e); - } - _ => {} - } - } - - None - } - }}; -} diff --git a/compiler/rustc_attr_data_structures/src/lints.rs b/compiler/rustc_attr_data_structures/src/lints.rs deleted file mode 100644 index 60ca4d43ce9b..000000000000 --- a/compiler/rustc_attr_data_structures/src/lints.rs +++ /dev/null @@ -1,16 +0,0 @@ -use rustc_macros::HashStable_Generic; -use rustc_span::Span; - -#[derive(Clone, Debug, HashStable_Generic)] -pub struct AttributeLint { - pub id: Id, - pub span: Span, - pub kind: AttributeLintKind, -} - -#[derive(Clone, Debug, HashStable_Generic)] -pub enum AttributeLintKind { - UnusedDuplicate { this: Span, other: Span, warning: bool }, - IllFormedAttributeInput { suggestions: Vec }, - EmptyAttribute { first_span: Span }, -} diff --git a/compiler/rustc_attr_data_structures/src/version.rs b/compiler/rustc_attr_data_structures/src/version.rs deleted file mode 100644 index 030e95209405..000000000000 --- a/compiler/rustc_attr_data_structures/src/version.rs +++ /dev/null @@ -1,47 +0,0 @@ -use std::fmt::{self, Display}; -use std::sync::OnceLock; - -use rustc_macros::{ - Decodable, Encodable, HashStable_Generic, PrintAttribute, current_rustc_version, -}; - -use crate::PrintAttribute; - -#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -#[derive(HashStable_Generic, PrintAttribute)] -pub struct RustcVersion { - pub major: u16, - pub minor: u16, - pub patch: u16, -} - -impl RustcVersion { - pub const CURRENT: Self = current_rustc_version!(); - pub fn current_overridable() -> Self { - *CURRENT_OVERRIDABLE.get_or_init(|| { - if let Ok(override_var) = std::env::var("RUSTC_OVERRIDE_VERSION_STRING") - && let Some(override_) = Self::parse_str(&override_var) - { - override_ - } else { - Self::CURRENT - } - }) - } - fn parse_str(value: &str) -> Option { - // Ignore any suffixes such as "-dev" or "-nightly". - let mut components = value.split('-').next().unwrap().splitn(3, '.'); - let major = components.next()?.parse().ok()?; - let minor = components.next()?.parse().ok()?; - let patch = components.next().unwrap_or("0").parse().ok()?; - Some(RustcVersion { major, minor, patch }) - } -} - -static CURRENT_OVERRIDABLE: OnceLock = OnceLock::new(); - -impl Display for RustcVersion { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "{}.{}.{}", self.major, self.minor, self.patch) - } -} diff --git a/compiler/rustc_attr_parsing/Cargo.toml b/compiler/rustc_attr_parsing/Cargo.toml index 32029137268b..79193f394fe4 100644 --- a/compiler/rustc_attr_parsing/Cargo.toml +++ b/compiler/rustc_attr_parsing/Cargo.toml @@ -8,14 +8,15 @@ edition = "2024" rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_ast_pretty = { path = "../rustc_ast_pretty" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_feature = { path = "../rustc_feature" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } rustc_hir = { path = "../rustc_hir" } rustc_lexer = { path = "../rustc_lexer" } rustc_macros = { path = "../rustc_macros" } +rustc_parse = { path = "../rustc_parse" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } +rustc_target = { path = "../rustc_target" } thin-vec = "0.2.12" # tidy-alphabetical-end diff --git a/compiler/rustc_attr_parsing/messages.ftl b/compiler/rustc_attr_parsing/messages.ftl index 35ff48cb5f24..a2a5f8ab1423 100644 --- a/compiler/rustc_attr_parsing/messages.ftl +++ b/compiler/rustc_attr_parsing/messages.ftl @@ -1,3 +1,11 @@ +attr_parsing_as_needed_compatibility = + linking modifier `as-needed` is only compatible with `dylib`, `framework` and `raw-dylib` linking kinds + +attr_parsing_bundle_needs_static = + linking modifier `bundle` is only compatible with `static` linking kind + +attr_parsing_cfg_attr_bad_delim = wrong `cfg_attr` delimiters + attr_parsing_cfg_predicate_identifier = `cfg` predicate key must be an identifier @@ -8,10 +16,22 @@ attr_parsing_deprecated_item_suggestion = attr_parsing_empty_attribute = unused attribute - .suggestion = remove this attribute + .suggestion = {$valid_without_list -> + [true] remove these parentheses + *[other] remove this attribute + } + .note = {$valid_without_list -> + [true] using `{$attr_path}` with an empty list is equivalent to not using a list at all + *[other] using `{$attr_path}` with an empty list has no effect + } + attr_parsing_empty_confusables = expected at least one confusable name +attr_parsing_empty_link_name = + link name must not be empty + .label = empty link name + attr_parsing_expected_one_cfg_pattern = expected 1 cfg-pattern @@ -32,6 +52,15 @@ attr_parsing_ill_formed_attribute_input = {$num_suggestions -> *[other] valid forms for the attribute are {$suggestions} } +attr_parsing_import_name_type_raw = + import name type can only be used with link kind `raw-dylib` + +attr_parsing_import_name_type_x86 = + import name type is only supported on x86 + +attr_parsing_incompatible_wasm_link = + `wasm_import_module` is incompatible with other arguments in `#[link]` attributes + attr_parsing_incorrect_repr_format_align_one_arg = incorrect `repr(align)` attribute format: `align` takes exactly one argument in parentheses @@ -51,6 +80,11 @@ attr_parsing_incorrect_repr_format_packed_one_or_zero_arg = attr_parsing_invalid_alignment_value = invalid alignment value: {$error_part} +attr_parsing_invalid_attr_unsafe = `{$name}` is not an unsafe attribute + .label = this is not an unsafe attribute + .suggestion = remove the `unsafe(...)` + .note = extraneous unsafe is not allowed in attributes + attr_parsing_invalid_issue_string = `issue` must be a non-zero numeric string or "none" .must_not_be_zero = `issue` must not be "0", use "none" instead @@ -59,6 +93,13 @@ attr_parsing_invalid_issue_string = .pos_overflow = number too large to fit in target type .neg_overflow = number too small to fit in target type +attr_parsing_invalid_link_modifier = + invalid linking modifier syntax, expected '+' or '-' prefix before one of: bundle, verbatim, whole-archive, as-needed + +attr_parsing_invalid_meta_item = expected a literal (`1u8`, `1.0f32`, `"string"`, etc.) here, found {$descr} + .remove_neg_sugg = negative numbers are not literals, try removing the `-` sign + .quote_ident_sugg = surround the identifier with quotation marks to make it into a string literal + attr_parsing_invalid_predicate = invalid predicate `{$predicate}` @@ -78,9 +119,42 @@ attr_parsing_invalid_repr_hint_no_value = attr_parsing_invalid_since = 'since' must be a Rust version number, such as "1.31.0" +attr_parsing_invalid_style = {$is_used_as_inner -> + [false] crate-level attribute should be an inner attribute: add an exclamation mark: `#![{$name}]` + *[other] the `#![{$name}]` attribute can only be used at the crate root + } + .note = This attribute does not have an `!`, which means it is applied to this {$target} + +attr_parsing_invalid_target = `#[{$name}]` attribute cannot be used on {$target} + .help = `#[{$name}]` can {$only}be applied to {$applied} + .suggestion = remove the attribute +attr_parsing_invalid_target_lint = `#[{$name}]` attribute cannot be used on {$target} + .warn = {-attr_parsing_previously_accepted} + .help = `#[{$name}]` can {$only}be applied to {$applied} + .suggestion = remove the attribute + +attr_parsing_limit_invalid = + `limit` must be a non-negative integer + .label = {$error_str} +attr_parsing_link_arg_unstable = + link kind `link-arg` is unstable + +attr_parsing_link_cfg_unstable = + link cfg is unstable + +attr_parsing_link_framework_apple = + link kind `framework` is only supported on Apple targets + attr_parsing_link_ordinal_out_of_range = ordinal value in `link_ordinal` is too large: `{$ordinal}` .note = the value may not exceed `u16::MAX` +attr_parsing_link_requires_name = + `#[link]` attribute requires a `name = "string"` argument + .label = missing `name` argument + +attr_parsing_meta_bad_delim = wrong meta list delimiters +attr_parsing_meta_bad_delim_suggestion = the delimiters should be `(` and `)` + attr_parsing_missing_feature = missing 'feature' @@ -93,6 +167,9 @@ attr_parsing_missing_note = attr_parsing_missing_since = missing 'since' +attr_parsing_multiple_modifiers = + multiple `{$modifier}` modifiers in a single `modifiers` argument + attr_parsing_multiple_stability_levels = multiple stability levels @@ -108,6 +185,23 @@ attr_parsing_null_on_export = `export_name` may not contain null characters attr_parsing_null_on_link_section = `link_section` may not contain null characters +attr_parsing_null_on_objc_class = `objc::class!` may not contain null characters + +attr_parsing_null_on_objc_selector = `objc::selector!` may not contain null characters + +attr_parsing_objc_class_expected_string_literal = `objc::class!` expected a string literal + +attr_parsing_objc_selector_expected_string_literal = `objc::selector!` expected a string literal + +attr_parsing_raw_dylib_elf_unstable = + link kind `raw-dylib` is unstable on ELF platforms + +attr_parsing_raw_dylib_no_nul = + link name must not contain NUL characters if link kind is `raw-dylib` + +attr_parsing_raw_dylib_only_windows = + link kind `raw-dylib` is only supported on Windows targets + attr_parsing_repr_ident = meta item in `repr` must be an identifier @@ -122,6 +216,9 @@ attr_parsing_soft_no_args = attr_parsing_stability_outside_std = stability attributes may not be used outside of the standard library +attr_parsing_suffixed_literal_in_attribute = suffixed literals are not allowed in attributes + .help = instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), use an unsuffixed version (`1`, `1.0`, etc.) + attr_parsing_unknown_meta_item = unknown meta item '{$item}' .label = expected one of {$expected} @@ -132,11 +229,16 @@ attr_parsing_unknown_version_literal = attr_parsing_unrecognized_repr_hint = unrecognized representation hint .help = valid reprs are `Rust` (default), `C`, `align`, `packed`, `transparent`, `simd`, `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize` + .note = for more information, visit + +attr_parsing_unsafe_attr_outside_unsafe = unsafe attribute used without unsafe + .label = usage of unsafe attribute +attr_parsing_unsafe_attr_outside_unsafe_suggestion = wrap the attribute in `unsafe(...)` attr_parsing_unstable_cfg_target_compact = compact `cfg(target(..))` is experimental and subject to change -attr_parsing_unstable_feature_bound_incompatible_stability = Item annotated with `#[unstable_feature_bound]` should not be stable +attr_parsing_unstable_feature_bound_incompatible_stability = item annotated with `#[unstable_feature_bound]` should not be stable .help = If this item is meant to be stable, do not use any functions annotated with `#[unstable_feature_bound]`. Otherwise, mark this item as unstable with `#[unstable]` attr_parsing_unsupported_literal_cfg_boolean = @@ -161,3 +263,6 @@ attr_parsing_unused_multiple = -attr_parsing_previously_accepted = this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! + +attr_parsing_whole_archive_needs_static = + linking modifier `whole-archive` is only compatible with `static` linking kind diff --git a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs index a6bd2306ec50..088fa73d7427 100644 --- a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs +++ b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs @@ -1,12 +1,6 @@ use std::iter; -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; - -use super::{CombineAttributeParser, ConvertFn}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; +use super::prelude::*; use crate::session_diagnostics; pub(crate) struct AllowInternalUnstableParser; @@ -15,7 +9,13 @@ impl CombineAttributeParser for AllowInternalUnstableParser { type Item = (Symbol, Span); const CONVERT: ConvertFn = |items, span| AttributeKind::AllowInternalUnstable(items, span); - const TEMPLATE: AttributeTemplate = template!(Word, List: "feat1, feat2, ..."); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::MacroDef), + Allow(Target::Fn), + Warn(Target::Field), + Warn(Target::Arm), + ]); + const TEMPLATE: AttributeTemplate = template!(Word, List: &["feat1, feat2, ..."]); fn extend<'c>( cx: &'c mut AcceptContext<'_, '_, S>, @@ -32,7 +32,12 @@ impl CombineAttributeParser for UnstableFeatureBoundParser { const PATH: &'static [rustc_span::Symbol] = &[sym::unstable_feature_bound]; type Item = (Symbol, Span); const CONVERT: ConvertFn = |items, _| AttributeKind::UnstableFeatureBound(items); - const TEMPLATE: AttributeTemplate = template!(Word, List: "feat1, feat2, ..."); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Impl { of_trait: true }), + Allow(Target::Trait), + ]); + const TEMPLATE: AttributeTemplate = template!(Word, List: &["feat1, feat2, ..."]); fn extend<'c>( cx: &'c mut AcceptContext<'_, '_, S>, @@ -53,7 +58,14 @@ impl CombineAttributeParser for AllowConstFnUnstableParser { type Item = Symbol; const CONVERT: ConvertFn = |items, first_span| AttributeKind::AllowConstFnUnstable(items, first_span); - const TEMPLATE: AttributeTemplate = template!(Word, List: "feat1, feat2, ..."); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + const TEMPLATE: AttributeTemplate = template!(Word, List: &["feat1, feat2, ..."]); fn extend<'c>( cx: &'c mut AcceptContext<'_, '_, S>, diff --git a/compiler/rustc_attr_parsing/src/attributes/body.rs b/compiler/rustc_attr_parsing/src/attributes/body.rs new file mode 100644 index 000000000000..a1492d761946 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/body.rs @@ -0,0 +1,12 @@ +//! Attributes that can be found in function body. + +use super::prelude::*; + +pub(crate) struct CoroutineParser; + +impl NoArgsAttributeParser for CoroutineParser { + const PATH: &[Symbol] = &[sym::coroutine]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Closure)]); + const CREATE: fn(rustc_span::Span) -> AttributeKind = |span| AttributeKind::Coroutine(span); +} diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg.rs b/compiler/rustc_attr_parsing/src/attributes/cfg.rs index 6373cf6e08ad..dab9e7666c23 100644 --- a/compiler/rustc_attr_parsing/src/attributes/cfg.rs +++ b/compiler/rustc_attr_parsing/src/attributes/cfg.rs @@ -1,23 +1,41 @@ -use rustc_ast::{LitKind, NodeId}; -use rustc_attr_data_structures::{CfgEntry, RustcVersion}; +use rustc_ast::token::Delimiter; +use rustc_ast::tokenstream::DelimSpan; +use rustc_ast::{AttrItem, Attribute, CRATE_NODE_ID, LitKind, NodeId, ast, token}; +use rustc_errors::{Applicability, PResult}; use rustc_feature::{AttributeTemplate, Features, template}; +use rustc_hir::attrs::CfgEntry; +use rustc_hir::{AttrPath, RustcVersion}; +use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_parse::{exp, parse_in}; use rustc_session::Session; use rustc_session::config::ExpectedValues; use rustc_session::lint::BuiltinLintDiag; use rustc_session::lint::builtin::UNEXPECTED_CFGS; -use rustc_session::parse::feature_err; +use rustc_session::parse::{ParseSess, feature_err}; use rustc_span::{Span, Symbol, sym}; use thin_vec::ThinVec; use crate::context::{AcceptContext, ShouldEmit, Stage}; use crate::parser::{ArgParser, MetaItemListParser, MetaItemOrLitParser, NameValueParser}; +use crate::session_diagnostics::{ + AttributeParseError, AttributeParseErrorReason, CfgAttrBadDelim, MetaBadDelimSugg, +}; use crate::{ - CfgMatchesLintEmitter, fluent_generated, parse_version, session_diagnostics, try_gate_cfg, + AttributeParser, CfgMatchesLintEmitter, fluent_generated, parse_version, session_diagnostics, + try_gate_cfg, }; -pub const CFG_TEMPLATE: AttributeTemplate = template!(List: "predicate"); +pub const CFG_TEMPLATE: AttributeTemplate = template!( + List: &["predicate"], + "https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg-attribute" +); -pub fn parse_cfg_attr<'c, S: Stage>( +const CFG_ATTR_TEMPLATE: AttributeTemplate = template!( + List: &["predicate, attr1, attr2, ..."], + "https://doc.rust-lang.org/reference/conditional-compilation.html#the-cfg_attr-attribute" +); + +pub fn parse_cfg<'c, S: Stage>( cx: &'c mut AcceptContext<'_, '_, S>, args: &'c ArgParser<'_>, ) -> Option { @@ -32,7 +50,7 @@ pub fn parse_cfg_attr<'c, S: Stage>( parse_cfg_entry(cx, single) } -fn parse_cfg_entry( +pub(crate) fn parse_cfg_entry( cx: &mut AcceptContext<'_, '_, S>, item: &MetaItemOrLitParser<'_>, ) -> Option { @@ -66,9 +84,7 @@ fn parse_cfg_entry( }, a @ (ArgParser::NoArgs | ArgParser::NameValue(_)) => { let Some(name) = meta.path().word_sym() else { - cx.emit_err(session_diagnostics::CfgPredicateIdentifier { - span: meta.path().span(), - }); + cx.expected_identifier(meta.path().span()); return None; }; parse_name_value(name, meta.path().span(), a.name_value(), meta.span(), cx)? @@ -77,7 +93,7 @@ fn parse_cfg_entry( MetaItemOrLitParser::Lit(lit) => match lit.kind { LitKind::Bool(b) => CfgEntry::Bool(b, lit.span), _ => { - cx.emit_err(session_diagnostics::CfgPredicateIdentifier { span: lit.span }); + cx.expected_identifier(lit.span); return None; } }, @@ -145,9 +161,7 @@ fn parse_cfg_entry_target( // Then, parse it as a name-value item let Some(name) = sub_item.path().word_sym() else { - cx.emit_err(session_diagnostics::CfgPredicateIdentifier { - span: sub_item.path().span(), - }); + cx.expected_identifier(sub_item.path().span()); return None; }; let name = Symbol::intern(&format!("target_{name}")); @@ -296,3 +310,122 @@ impl EvalConfigResult { } } } + +pub fn parse_cfg_attr( + cfg_attr: &Attribute, + sess: &Session, + features: Option<&Features>, +) -> Option<(CfgEntry, Vec<(AttrItem, Span)>)> { + match cfg_attr.get_normal_item().args { + ast::AttrArgs::Delimited(ast::DelimArgs { dspan, delim, ref tokens }) + if !tokens.is_empty() => + { + check_cfg_attr_bad_delim(&sess.psess, dspan, delim); + match parse_in(&sess.psess, tokens.clone(), "`cfg_attr` input", |p| { + parse_cfg_attr_internal(p, sess, features, cfg_attr) + }) { + Ok(r) => return Some(r), + Err(e) => { + let suggestions = + CFG_ATTR_TEMPLATE.suggestions(Some(cfg_attr.style), sym::cfg_attr); + e.with_span_suggestions( + cfg_attr.span, + "must be of the form", + suggestions, + Applicability::HasPlaceholders, + ) + .with_note(format!( + "for more information, visit <{}>", + CFG_ATTR_TEMPLATE.docs.expect("cfg_attr has docs") + )) + .emit(); + } + } + } + _ => { + let (span, reason) = if let ast::AttrArgs::Delimited(ast::DelimArgs { dspan, .. }) = + cfg_attr.get_normal_item().args + { + (dspan.entire(), AttributeParseErrorReason::ExpectedAtLeastOneArgument) + } else { + (cfg_attr.span, AttributeParseErrorReason::ExpectedList) + }; + + sess.dcx().emit_err(AttributeParseError { + span, + attr_span: cfg_attr.span, + template: CFG_ATTR_TEMPLATE, + attribute: AttrPath::from_ast(&cfg_attr.get_normal_item().path), + reason, + suggestions: CFG_ATTR_TEMPLATE.suggestions(Some(cfg_attr.style), sym::cfg_attr), + }); + } + } + None +} + +fn check_cfg_attr_bad_delim(psess: &ParseSess, span: DelimSpan, delim: Delimiter) { + if let Delimiter::Parenthesis = delim { + return; + } + psess.dcx().emit_err(CfgAttrBadDelim { + span: span.entire(), + sugg: MetaBadDelimSugg { open: span.open, close: span.close }, + }); +} + +/// Parses `cfg_attr(pred, attr_item_list)` where `attr_item_list` is comma-delimited. +fn parse_cfg_attr_internal<'a>( + parser: &mut Parser<'a>, + sess: &'a Session, + features: Option<&Features>, + attribute: &Attribute, +) -> PResult<'a, (CfgEntry, Vec<(ast::AttrItem, Span)>)> { + // Parse cfg predicate + let pred_start = parser.token.span; + let meta = MetaItemOrLitParser::parse_single(parser, ShouldEmit::ErrorsAndLints)?; + let pred_span = pred_start.with_hi(parser.token.span.hi()); + + let cfg_predicate = AttributeParser::parse_single_args( + sess, + attribute.span, + attribute.get_normal_item().span(), + attribute.style, + AttrPath { + segments: attribute + .ident_path() + .expect("cfg_attr is not a doc comment") + .into_boxed_slice(), + span: attribute.span, + }, + pred_span, + CRATE_NODE_ID, + features, + ShouldEmit::ErrorsAndLints, + &meta, + parse_cfg_entry, + &CFG_ATTR_TEMPLATE, + ) + .ok_or_else(|| { + let mut diag = sess.dcx().struct_err( + "cfg_entry parsing failing with `ShouldEmit::ErrorsAndLints` should emit a error.", + ); + diag.downgrade_to_delayed_bug(); + diag + })?; + + parser.expect(exp!(Comma))?; + + // Presumably, the majority of the time there will only be one attr. + let mut expanded_attrs = Vec::with_capacity(1); + while parser.token != token::Eof { + let lo = parser.token.span; + let item = parser.parse_attr_item(ForceCollect::Yes)?; + expanded_attrs.push((item, lo.to(parser.prev_token.span))); + if !parser.eat(exp!(Comma)) { + break; + } + } + + Ok((cfg_predicate, expanded_attrs)) +} diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg_old.rs b/compiler/rustc_attr_parsing/src/attributes/cfg_old.rs index c5025a8b6eac..3257d898eccb 100644 --- a/compiler/rustc_attr_parsing/src/attributes/cfg_old.rs +++ b/compiler/rustc_attr_parsing/src/attributes/cfg_old.rs @@ -1,7 +1,7 @@ use rustc_ast::{LitKind, MetaItem, MetaItemInner, MetaItemKind, MetaItemLit, NodeId}; use rustc_ast_pretty::pprust; -use rustc_attr_data_structures::RustcVersion; use rustc_feature::{Features, GatedCfg, find_gated_cfg}; +use rustc_hir::RustcVersion; use rustc_session::Session; use rustc_session::config::ExpectedValues; use rustc_session::lint::builtin::UNEXPECTED_CFGS; diff --git a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs index afa9abed0bb5..f09b02251e4d 100644 --- a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs @@ -1,15 +1,12 @@ -use rustc_attr_data_structures::{AttributeKind, CoverageAttrKind, OptimizeAttr, UsedBy}; -use rustc_feature::{AttributeTemplate, template}; +use rustc_hir::attrs::{CoverageAttrKind, OptimizeAttr, SanitizerSet, UsedBy}; use rustc_session::parse::feature_err; -use rustc_span::{Span, Symbol, sym}; -use super::{ - AcceptMapping, AttributeOrder, AttributeParser, CombineAttributeParser, ConvertFn, - NoArgsAttributeParser, OnDuplicate, SingleAttributeParser, +use super::prelude::*; +use crate::session_diagnostics::{ + NakedFunctionIncompatibleAttribute, NullOnExport, NullOnObjcClass, NullOnObjcSelector, + ObjcClassExpectedStringLiteral, ObjcSelectorExpectedStringLiteral, }; -use crate::context::{AcceptContext, FinalizeContext, Stage}; -use crate::parser::ArgParser; -use crate::session_diagnostics::{NakedFunctionIncompatibleAttribute, NullOnExport}; +use crate::target_checking::Policy::AllowSilent; pub(crate) struct OptimizeParser; @@ -17,7 +14,14 @@ impl SingleAttributeParser for OptimizeParser { const PATH: &[Symbol] = &[sym::optimize]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(List: "size|speed|none"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Closure), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Method(MethodKind::Inherent)), + ]); + const TEMPLATE: AttributeTemplate = template!(List: &["size", "speed", "none"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(list) = args.list() else { @@ -35,7 +39,7 @@ impl SingleAttributeParser for OptimizeParser { Some(sym::speed) => OptimizeAttr::Speed, Some(sym::none) => OptimizeAttr::DoNotOptimize, _ => { - cx.expected_specific_argument(single.span(), vec!["size", "speed", "none"]); + cx.expected_specific_argument(single.span(), &[sym::size, sym::speed, sym::none]); OptimizeAttr::Default } }; @@ -49,6 +53,15 @@ pub(crate) struct ColdParser; impl NoArgsAttributeParser for ColdParser { const PATH: &[Symbol] = &[sym::cold]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::ForeignFn), + Allow(Target::Closure), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::Cold; } @@ -58,11 +71,22 @@ impl SingleAttributeParser for CoverageParser { const PATH: &[Symbol] = &[sym::coverage]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Closure), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Impl { of_trait: true }), + Allow(Target::Impl { of_trait: false }), + Allow(Target::Mod), + Allow(Target::Crate), + ]); const TEMPLATE: AttributeTemplate = template!(OneOf: &[sym::off, sym::on]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(args) = args.list() else { - cx.expected_specific_argument_and_list(cx.attr_span, vec!["on", "off"]); + cx.expected_specific_argument_and_list(cx.attr_span, &[sym::on, sym::off]); return None; }; @@ -71,7 +95,8 @@ impl SingleAttributeParser for CoverageParser { return None; }; - let fail_incorrect_argument = |span| cx.expected_specific_argument(span, vec!["on", "off"]); + let fail_incorrect_argument = + |span| cx.expected_specific_argument(span, &[sym::on, sym::off]); let Some(arg) = arg.meta_item() else { fail_incorrect_argument(args.span); @@ -97,6 +122,17 @@ impl SingleAttributeParser for ExportNameParser { const PATH: &[rustc_span::Symbol] = &[sym::export_name]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Static), + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Warn(Target::Field), + Warn(Target::Arm), + Warn(Target::MacroDef), + Warn(Target::MacroCall), + ]); const TEMPLATE: AttributeTemplate = template!(NameValueStr: "name"); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { @@ -118,6 +154,70 @@ impl SingleAttributeParser for ExportNameParser { } } +pub(crate) struct ObjcClassParser; + +impl SingleAttributeParser for ObjcClassParser { + const PATH: &[rustc_span::Symbol] = &[sym::rustc_objc_class]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::ForeignStatic)]); + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "ClassName"); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(nv) = args.name_value() else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + let Some(classname) = nv.value_as_str() else { + // `#[rustc_objc_class = ...]` is expected to be used as an implementatioin detail + // inside a standard library macro, but `cx.expected_string_literal` exposes too much. + // Use a custom error message instead. + cx.emit_err(ObjcClassExpectedStringLiteral { span: nv.value_span }); + return None; + }; + if classname.as_str().contains('\0') { + // `#[rustc_objc_class = ...]` will be converted to a null-terminated string, + // so it may not contain any null characters. + cx.emit_err(NullOnObjcClass { span: nv.value_span }); + return None; + } + Some(AttributeKind::ObjcClass { classname, span: cx.attr_span }) + } +} + +pub(crate) struct ObjcSelectorParser; + +impl SingleAttributeParser for ObjcSelectorParser { + const PATH: &[rustc_span::Symbol] = &[sym::rustc_objc_selector]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::ForeignStatic)]); + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "methodName"); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(nv) = args.name_value() else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + let Some(methname) = nv.value_as_str() else { + // `#[rustc_objc_selector = ...]` is expected to be used as an implementatioin detail + // inside a standard library macro, but `cx.expected_string_literal` exposes too much. + // Use a custom error message instead. + cx.emit_err(ObjcSelectorExpectedStringLiteral { span: nv.value_span }); + return None; + }; + if methname.as_str().contains('\0') { + // `#[rustc_objc_selector = ...]` will be converted to a null-terminated string, + // so it may not contain any null characters. + cx.emit_err(NullOnObjcSelector { span: nv.value_span }); + return None; + } + Some(AttributeKind::ObjcSelector { methname, span: cx.attr_span }) + } +} + #[derive(Default)] pub(crate) struct NakedParser { span: Option, @@ -138,6 +238,13 @@ impl AttributeParser for NakedParser { this.span = Some(cx.attr_span); } })]; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Warn(Target::MacroCall), + ]); fn finalize(self, cx: &FinalizeContext<'_, '_, S>) -> Option { // FIXME(jdonszelmann): upgrade this list to *parsed* attributes @@ -179,6 +286,7 @@ impl AttributeParser for NakedParser { sym::rustc_std_internal_symbol, // FIXME(#82232, #143834): temporarily renamed to mitigate `#[align]` nameres ambiguity sym::rustc_align, + sym::rustc_align_static, // obviously compatible with self sym::naked, // documentation @@ -230,6 +338,19 @@ pub(crate) struct TrackCallerParser; impl NoArgsAttributeParser for TrackCallerParser { const PATH: &[Symbol] = &[sym::track_caller]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::ForeignFn), + Allow(Target::Closure), + Warn(Target::MacroDef), + Warn(Target::Arm), + Warn(Target::Field), + Warn(Target::MacroCall), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::TrackCaller; } @@ -237,6 +358,14 @@ pub(crate) struct NoMangleParser; impl NoArgsAttributeParser for NoMangleParser { const PATH: &[Symbol] = &[sym::no_mangle]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Fn), + Allow(Target::Static), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::TraitImpl)), + AllowSilent(Target::Const), // Handled in the `InvalidNoMangleItems` pass + Error(Target::Closure), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoMangle; } @@ -244,6 +373,7 @@ impl NoArgsAttributeParser for NoMangleParser { pub(crate) struct UsedParser { first_compiler: Option, first_linker: Option, + first_default: Option, } // A custom `AttributeParser` is used rather than a Simple attribute parser because @@ -253,10 +383,10 @@ pub(crate) struct UsedParser { impl AttributeParser for UsedParser { const ATTRIBUTES: AcceptMapping = &[( &[sym::used], - template!(Word, List: "compiler|linker"), + template!(Word, List: &["compiler", "linker"]), |group: &mut Self, cx, args| { let used_by = match args { - ArgParser::NoArgs => UsedBy::Linker, + ArgParser::NoArgs => UsedBy::Default, ArgParser::List(list) => { let Some(l) = list.single() else { cx.expected_single_argument(list.span); @@ -289,7 +419,7 @@ impl AttributeParser for UsedParser { UsedBy::Linker } _ => { - cx.expected_specific_argument(l.span(), vec!["compiler", "linker"]); + cx.expected_specific_argument(l.span(), &[sym::compiler, sym::linker]); return; } } @@ -297,12 +427,29 @@ impl AttributeParser for UsedParser { ArgParser::NameValue(_) => return, }; + let attr_span = cx.attr_span; + + // `#[used]` is interpreted as `#[used(linker)]` (though depending on target OS the + // circumstances are more complicated). While we're checking `used_by`, also report + // these cross-`UsedBy` duplicates to warn. let target = match used_by { UsedBy::Compiler => &mut group.first_compiler, - UsedBy::Linker => &mut group.first_linker, + UsedBy::Linker => { + if let Some(prev) = group.first_default { + cx.warn_unused_duplicate(prev, attr_span); + return; + } + &mut group.first_linker + } + UsedBy::Default => { + if let Some(prev) = group.first_linker { + cx.warn_unused_duplicate(prev, attr_span); + return; + } + &mut group.first_default + } }; - let attr_span = cx.attr_span; if let Some(prev) = *target { cx.warn_unused_duplicate(prev, attr_span); } else { @@ -310,75 +457,222 @@ impl AttributeParser for UsedParser { } }, )]; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Static), Warn(Target::MacroCall)]); fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { - // Ratcheting behaviour, if both `linker` and `compiler` are specified, use `linker` - Some(match (self.first_compiler, self.first_linker) { - (_, Some(span)) => AttributeKind::Used { used_by: UsedBy::Linker, span }, - (Some(span), _) => AttributeKind::Used { used_by: UsedBy::Compiler, span }, - (None, None) => return None, + // If a specific form of `used` is specified, it takes precedence over generic `#[used]`. + // If both `linker` and `compiler` are specified, use `linker`. + Some(match (self.first_compiler, self.first_linker, self.first_default) { + (_, Some(span), _) => AttributeKind::Used { used_by: UsedBy::Linker, span }, + (Some(span), _, _) => AttributeKind::Used { used_by: UsedBy::Compiler, span }, + (_, _, Some(span)) => AttributeKind::Used { used_by: UsedBy::Default, span }, + (None, None, None) => return None, }) } } +fn parse_tf_attribute<'c, S: Stage>( + cx: &'c mut AcceptContext<'_, '_, S>, + args: &'c ArgParser<'_>, +) -> impl IntoIterator + 'c { + let mut features = Vec::new(); + let ArgParser::List(list) = args else { + cx.expected_list(cx.attr_span); + return features; + }; + if list.is_empty() { + cx.warn_empty_attribute(cx.attr_span); + return features; + } + for item in list.mixed() { + let Some(name_value) = item.meta_item() else { + cx.expected_name_value(item.span(), Some(sym::enable)); + return features; + }; + + // Validate name + let Some(name) = name_value.path().word_sym() else { + cx.expected_name_value(name_value.path().span(), Some(sym::enable)); + return features; + }; + if name != sym::enable { + cx.expected_name_value(name_value.path().span(), Some(sym::enable)); + return features; + } + + // Use value + let Some(name_value) = name_value.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::enable)); + return features; + }; + let Some(value_str) = name_value.value_as_str() else { + cx.expected_string_literal(name_value.value_span, Some(name_value.value_as_lit())); + return features; + }; + for feature in value_str.as_str().split(",") { + features.push((Symbol::intern(feature), item.span())); + } + } + features +} + pub(crate) struct TargetFeatureParser; impl CombineAttributeParser for TargetFeatureParser { type Item = (Symbol, Span); const PATH: &[Symbol] = &[sym::target_feature]; - const CONVERT: ConvertFn = |items, span| AttributeKind::TargetFeature(items, span); - const TEMPLATE: AttributeTemplate = template!(List: "enable = \"feat1, feat2\""); + const CONVERT: ConvertFn = |items, span| AttributeKind::TargetFeature { + features: items, + attr_span: span, + was_forced: false, + }; + const TEMPLATE: AttributeTemplate = template!(List: &["enable = \"feat1, feat2\""]); fn extend<'c>( cx: &'c mut AcceptContext<'_, '_, S>, args: &'c ArgParser<'_>, ) -> impl IntoIterator + 'c { - let mut features = Vec::new(); - let ArgParser::List(list) = args else { - cx.expected_list(cx.attr_span); - return features; - }; - if list.is_empty() { - cx.warn_empty_attribute(cx.attr_span); - return features; - } - for item in list.mixed() { - let Some(name_value) = item.meta_item() else { - cx.expected_name_value(item.span(), Some(sym::enable)); - return features; - }; + parse_tf_attribute(cx, args) + } - // Validate name - let Some(name) = name_value.path().word_sym() else { - cx.expected_name_value(name_value.path().span(), Some(sym::enable)); - return features; - }; - if name != sym::enable { - cx.expected_name_value(name_value.path().span(), Some(sym::enable)); - return features; - } + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Warn(Target::Statement), + Warn(Target::Field), + Warn(Target::Arm), + Warn(Target::MacroDef), + Warn(Target::MacroCall), + ]); +} - // Use value - let Some(name_value) = name_value.args().name_value() else { - cx.expected_name_value(item.span(), Some(sym::enable)); - return features; - }; - let Some(value_str) = name_value.value_as_str() else { - cx.expected_string_literal(name_value.value_span, Some(name_value.value_as_lit())); - return features; - }; - for feature in value_str.as_str().split(",") { - features.push((Symbol::intern(feature), item.span())); - } - } - features +pub(crate) struct ForceTargetFeatureParser; + +impl CombineAttributeParser for ForceTargetFeatureParser { + type Item = (Symbol, Span); + const PATH: &[Symbol] = &[sym::force_target_feature]; + const CONVERT: ConvertFn = |items, span| AttributeKind::TargetFeature { + features: items, + attr_span: span, + was_forced: true, + }; + const TEMPLATE: AttributeTemplate = template!(List: &["enable = \"feat1, feat2\""]); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + + fn extend<'c>( + cx: &'c mut AcceptContext<'_, '_, S>, + args: &'c ArgParser<'_>, + ) -> impl IntoIterator + 'c { + parse_tf_attribute(cx, args) } } -pub(crate) struct OmitGdbPrettyPrinterSectionParser; +pub(crate) struct SanitizeParser; -impl NoArgsAttributeParser for OmitGdbPrettyPrinterSectionParser { - const PATH: &[Symbol] = &[sym::omit_gdb_pretty_printer_section]; +impl SingleAttributeParser for SanitizeParser { + const PATH: &[Symbol] = &[sym::sanitize]; + + // FIXME: still checked in check_attrs.rs + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); + + const TEMPLATE: AttributeTemplate = template!(List: &[ + r#"address = "on|off""#, + r#"kernel_address = "on|off""#, + r#"cfi = "on|off""#, + r#"hwaddress = "on|off""#, + r#"kcfi = "on|off""#, + r#"memory = "on|off""#, + r#"memtag = "on|off""#, + r#"shadow_call_stack = "on|off""#, + r#"thread = "on|off""# + ]); + + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::OmitGdbPrettyPrinterSection; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(list) = args.list() else { + cx.expected_list(cx.attr_span); + return None; + }; + + let mut on_set = SanitizerSet::empty(); + let mut off_set = SanitizerSet::empty(); + + for item in list.mixed() { + let Some(item) = item.meta_item() else { + cx.expected_name_value(item.span(), None); + continue; + }; + + let path = item.path().word_sym(); + let Some(value) = item.args().name_value() else { + cx.expected_name_value(item.span(), path); + continue; + }; + + let mut apply = |s: SanitizerSet| { + let is_on = match value.value_as_str() { + Some(sym::on) => true, + Some(sym::off) => false, + Some(_) => { + cx.expected_specific_argument_strings( + value.value_span, + &[sym::on, sym::off], + ); + return; + } + None => { + cx.expected_string_literal(value.value_span, Some(value.value_as_lit())); + return; + } + }; + + if is_on { + on_set |= s; + } else { + off_set |= s; + } + }; + + match path { + Some(sym::address) | Some(sym::kernel_address) => { + apply(SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS) + } + Some(sym::cfi) => apply(SanitizerSet::CFI), + Some(sym::kcfi) => apply(SanitizerSet::KCFI), + Some(sym::memory) => apply(SanitizerSet::MEMORY), + Some(sym::memtag) => apply(SanitizerSet::MEMTAG), + Some(sym::shadow_call_stack) => apply(SanitizerSet::SHADOWCALLSTACK), + Some(sym::thread) => apply(SanitizerSet::THREAD), + Some(sym::hwaddress) => apply(SanitizerSet::HWADDRESS), + _ => { + cx.expected_specific_argument_strings( + item.path().span(), + &[ + sym::address, + sym::cfi, + sym::kcfi, + sym::memory, + sym::memtag, + sym::shadow_call_stack, + sym::thread, + sym::hwaddress, + ], + ); + continue; + } + } + } + + Some(AttributeKind::Sanitize { on_set, off_set, span: cx.attr_span }) + } } diff --git a/compiler/rustc_attr_parsing/src/attributes/confusables.rs b/compiler/rustc_attr_parsing/src/attributes/confusables.rs index c911908dfb38..97e78dfb136b 100644 --- a/compiler/rustc_attr_parsing/src/attributes/confusables.rs +++ b/compiler/rustc_attr_parsing/src/attributes/confusables.rs @@ -1,11 +1,5 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::template; -use rustc_span::{Span, Symbol, sym}; -use thin_vec::ThinVec; - -use super::{AcceptMapping, AttributeParser}; -use crate::context::{FinalizeContext, Stage}; -use crate::session_diagnostics; +use super::prelude::*; +use crate::session_diagnostics::EmptyConfusables; #[derive(Default)] pub(crate) struct ConfusablesParser { @@ -16,7 +10,7 @@ pub(crate) struct ConfusablesParser { impl AttributeParser for ConfusablesParser { const ATTRIBUTES: AcceptMapping = &[( &[sym::rustc_confusables], - template!(List: r#""name1", "name2", ..."#), + template!(List: &[r#""name1", "name2", ..."#]), |this, cx, args| { let Some(list) = args.list() else { cx.expected_list(cx.attr_span); @@ -24,7 +18,7 @@ impl AttributeParser for ConfusablesParser { }; if list.is_empty() { - cx.emit_err(session_diagnostics::EmptyConfusables { span: cx.attr_span }); + cx.emit_err(EmptyConfusables { span: cx.attr_span }); } for param in list.mixed() { @@ -41,6 +35,8 @@ impl AttributeParser for ConfusablesParser { this.first_span.get_or_insert(cx.attr_span); }, )]; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Method(MethodKind::Inherent))]); fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { if self.confusables.is_empty() { diff --git a/compiler/rustc_attr_parsing/src/attributes/crate_level.rs b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs new file mode 100644 index 000000000000..1d3ab76e1499 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/crate_level.rs @@ -0,0 +1,144 @@ +use super::prelude::*; + +pub(crate) struct CrateNameParser; + +impl SingleAttributeParser for CrateNameParser { + const PATH: &[Symbol] = &[sym::crate_name]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "name"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(n) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + + let Some(name) = n.value_as_str() else { + cx.expected_string_literal(n.value_span, Some(n.value_as_lit())); + return None; + }; + + Some(AttributeKind::CrateName { name, name_span: n.value_span, attr_span: cx.attr_span }) + } +} + +pub(crate) struct RecursionLimitParser; + +impl SingleAttributeParser for RecursionLimitParser { + const PATH: &[Symbol] = &[sym::recursion_limit]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N", "https://doc.rust-lang.org/reference/attributes/limits.html#the-recursion_limit-attribute"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(nv) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + + Some(AttributeKind::RecursionLimit { + limit: cx.parse_limit_int(nv)?, + attr_span: cx.attr_span, + limit_span: nv.value_span, + }) + } +} + +pub(crate) struct MoveSizeLimitParser; + +impl SingleAttributeParser for MoveSizeLimitParser { + const PATH: &[Symbol] = &[sym::move_size_limit]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(nv) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + + Some(AttributeKind::MoveSizeLimit { + limit: cx.parse_limit_int(nv)?, + attr_span: cx.attr_span, + limit_span: nv.value_span, + }) + } +} + +pub(crate) struct TypeLengthLimitParser; + +impl SingleAttributeParser for TypeLengthLimitParser { + const PATH: &[Symbol] = &[sym::type_length_limit]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(nv) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + + Some(AttributeKind::TypeLengthLimit { + limit: cx.parse_limit_int(nv)?, + attr_span: cx.attr_span, + limit_span: nv.value_span, + }) + } +} + +pub(crate) struct PatternComplexityLimitParser; + +impl SingleAttributeParser for PatternComplexityLimitParser { + const PATH: &[Symbol] = &[sym::pattern_complexity_limit]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(nv) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + + Some(AttributeKind::PatternComplexityLimit { + limit: cx.parse_limit_int(nv)?, + attr_span: cx.attr_span, + limit_span: nv.value_span, + }) + } +} + +pub(crate) struct NoCoreParser; + +impl NoArgsAttributeParser for NoCoreParser { + const PATH: &[Symbol] = &[sym::no_core]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoCore; +} + +pub(crate) struct NoStdParser; + +impl NoArgsAttributeParser for NoStdParser { + const PATH: &[Symbol] = &[sym::no_std]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoStd; +} + +pub(crate) struct RustcCoherenceIsCoreParser; + +impl NoArgsAttributeParser for RustcCoherenceIsCoreParser { + const PATH: &[Symbol] = &[sym::rustc_coherence_is_core]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::CrateLevel; + const CREATE: fn(Span) -> AttributeKind = AttributeKind::RustcCoherenceIsCore; +} diff --git a/compiler/rustc_attr_parsing/src/attributes/debugger.rs b/compiler/rustc_attr_parsing/src/attributes/debugger.rs new file mode 100644 index 000000000000..56ff10be4264 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/debugger.rs @@ -0,0 +1,60 @@ +use rustc_hir::attrs::{DebugVisualizer, DebuggerVisualizerType}; + +use super::prelude::*; + +pub(crate) struct DebuggerViualizerParser; + +impl CombineAttributeParser for DebuggerViualizerParser { + const PATH: &[Symbol] = &[sym::debugger_visualizer]; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Mod), Allow(Target::Crate)]); + const TEMPLATE: AttributeTemplate = template!( + List: &[r#"natvis_file = "...", gdb_script_file = "...""#], + "https://doc.rust-lang.org/reference/attributes/debugger.html#the-debugger_visualizer-attribute" + ); + + type Item = DebugVisualizer; + const CONVERT: ConvertFn = |v, _| AttributeKind::DebuggerVisualizer(v); + + fn extend<'c>( + cx: &'c mut AcceptContext<'_, '_, S>, + args: &'c ArgParser<'_>, + ) -> impl IntoIterator + 'c { + let Some(l) = args.list() else { + cx.expected_list(args.span().unwrap_or(cx.attr_span)); + return None; + }; + let Some(single) = l.single() else { + cx.expected_single_argument(l.span); + return None; + }; + let Some(mi) = single.meta_item() else { + cx.expected_name_value(single.span(), None); + return None; + }; + let path = mi.path().word_sym(); + let visualizer_type = match path { + Some(sym::natvis_file) => DebuggerVisualizerType::Natvis, + Some(sym::gdb_script_file) => DebuggerVisualizerType::GdbPrettyPrinter, + _ => { + cx.expected_specific_argument( + mi.path().span(), + &[sym::natvis_file, sym::gdb_script_file], + ); + return None; + } + }; + + let Some(path) = mi.args().name_value() else { + cx.expected_name_value(single.span(), path); + return None; + }; + + let Some(path) = path.value_as_str() else { + cx.expected_string_literal(path.value_span, Some(path.value_as_lit())); + return None; + }; + + Some(DebugVisualizer { span: mi.span(), visualizer_type, path }) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/deprecation.rs b/compiler/rustc_attr_parsing/src/attributes/deprecation.rs index 08cf1ab5d190..f96477e28cd0 100644 --- a/compiler/rustc_attr_parsing/src/attributes/deprecation.rs +++ b/compiler/rustc_attr_parsing/src/attributes/deprecation.rs @@ -1,12 +1,10 @@ -use rustc_attr_data_structures::{AttributeKind, DeprecatedSince, Deprecation}; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; +use rustc_hir::attrs::{DeprecatedSince, Deprecation}; +use super::prelude::*; use super::util::parse_version; -use super::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; -use crate::session_diagnostics; +use crate::session_diagnostics::{ + DeprecatedItemSuggestion, InvalidSince, MissingNote, MissingSince, +}; pub(crate) struct DeprecationParser; @@ -38,9 +36,35 @@ impl SingleAttributeParser for DeprecationParser { const PATH: &[Symbol] = &[sym::deprecated]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Fn), + Allow(Target::Mod), + Allow(Target::Struct), + Allow(Target::Enum), + Allow(Target::Union), + Allow(Target::Const), + Allow(Target::Static), + Allow(Target::MacroDef), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::TyAlias), + Allow(Target::Use), + Allow(Target::ForeignFn), + Allow(Target::ForeignStatic), + Allow(Target::ForeignTy), + Allow(Target::Field), + Allow(Target::Trait), + Allow(Target::AssocTy), + Allow(Target::AssocConst), + Allow(Target::Variant), + Allow(Target::Impl { of_trait: false }), + Allow(Target::Crate), + Error(Target::WherePredicate), + ]); const TEMPLATE: AttributeTemplate = template!( Word, - List: r#"/*opt*/ since = "version", /*opt*/ note = "reason""#, + List: &[r#"since = "version""#, r#"note = "reason""#, r#"since = "version", note = "reason""#], NameValueStr: "reason" ); @@ -75,7 +99,7 @@ impl SingleAttributeParser for DeprecationParser { } Some(name @ sym::suggestion) => { if !features.deprecated_suggestion() { - cx.emit_err(session_diagnostics::DeprecatedItemSuggestion { + cx.emit_err(DeprecatedItemSuggestion { span: param.span(), is_nightly: cx.sess().is_nightly_build(), details: (), @@ -117,18 +141,18 @@ impl SingleAttributeParser for DeprecationParser { } else if let Some(version) = parse_version(since) { DeprecatedSince::RustcVersion(version) } else { - cx.emit_err(session_diagnostics::InvalidSince { span: cx.attr_span }); + cx.emit_err(InvalidSince { span: cx.attr_span }); DeprecatedSince::Err } } else if is_rustc { - cx.emit_err(session_diagnostics::MissingSince { span: cx.attr_span }); + cx.emit_err(MissingSince { span: cx.attr_span }); DeprecatedSince::Err } else { DeprecatedSince::Unspecified }; if is_rustc && note.is_none() { - cx.emit_err(session_diagnostics::MissingNote { span: cx.attr_span }); + cx.emit_err(MissingNote { span: cx.attr_span }); return None; } diff --git a/compiler/rustc_attr_parsing/src/attributes/dummy.rs b/compiler/rustc_attr_parsing/src/attributes/dummy.rs index e5e1c3bb6b6a..7293cee842c2 100644 --- a/compiler/rustc_attr_parsing/src/attributes/dummy.rs +++ b/compiler/rustc_attr_parsing/src/attributes/dummy.rs @@ -1,16 +1,18 @@ -use rustc_attr_data_structures::AttributeKind; use rustc_feature::{AttributeTemplate, template}; +use rustc_hir::attrs::AttributeKind; use rustc_span::{Symbol, sym}; use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser}; use crate::context::{AcceptContext, Stage}; use crate::parser::ArgParser; +use crate::target_checking::{ALL_TARGETS, AllowedTargets}; pub(crate) struct DummyParser; impl SingleAttributeParser for DummyParser { const PATH: &[Symbol] = &[sym::rustc_dummy]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Ignore; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); const TEMPLATE: AttributeTemplate = template!(Word); // Anything, really fn convert(_: &mut AcceptContext<'_, '_, S>, _: &ArgParser<'_>) -> Option { diff --git a/compiler/rustc_attr_parsing/src/attributes/inline.rs b/compiler/rustc_attr_parsing/src/attributes/inline.rs index fe812175218d..eda272fb7f2b 100644 --- a/compiler/rustc_attr_parsing/src/attributes/inline.rs +++ b/compiler/rustc_attr_parsing/src/attributes/inline.rs @@ -2,15 +2,9 @@ // note: need to model better how duplicate attr errors work when not using // SingleAttributeParser which is what we have two of here. -use rustc_attr_data_structures::lints::AttributeLintKind; -use rustc_attr_data_structures::{AttributeKind, InlineAttr}; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Symbol, sym}; +use rustc_hir::attrs::{AttributeKind, InlineAttr}; -use super::{AcceptContext, AttributeOrder, OnDuplicate}; -use crate::attributes::SingleAttributeParser; -use crate::context::Stage; -use crate::parser::ArgParser; +use super::prelude::*; pub(crate) struct InlineParser; @@ -18,7 +12,26 @@ impl SingleAttributeParser for InlineParser { const PATH: &'static [Symbol] = &[sym::inline]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(Word, List: "always|never"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Closure), + Allow(Target::Delegation { mac: false }), + Warn(Target::Method(MethodKind::Trait { body: false })), + Warn(Target::ForeignFn), + Warn(Target::Field), + Warn(Target::MacroDef), + Warn(Target::Arm), + Warn(Target::AssocConst), + Warn(Target::MacroCall), + ]); + const TEMPLATE: AttributeTemplate = template!( + Word, + List: &["always", "never"], + "https://doc.rust-lang.org/reference/attributes/codegen.html#the-inline-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { match args { @@ -37,14 +50,13 @@ impl SingleAttributeParser for InlineParser { Some(AttributeKind::Inline(InlineAttr::Never, cx.attr_span)) } _ => { - cx.expected_specific_argument(l.span(), vec!["always", "never"]); + cx.expected_specific_argument(l.span(), &[sym::always, sym::never]); return None; } } } ArgParser::NameValue(_) => { - let suggestions = - >::TEMPLATE.suggestions(false, "inline"); + let suggestions = cx.suggestions(); let span = cx.attr_span; cx.emit_lint(AttributeLintKind::IllFormedAttributeInput { suggestions }, span); return None; @@ -59,7 +71,12 @@ impl SingleAttributeParser for RustcForceInlineParser { const PATH: &'static [Symbol] = &[sym::rustc_force_inline]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(Word, List: "reason", NameValueStr: "reason"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + ]); + + const TEMPLATE: AttributeTemplate = template!(Word, List: &["reason"], NameValueStr: "reason"); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let reason = match args { diff --git a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs index 960cebd89259..797d04b914dd 100644 --- a/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/link_attrs.rs @@ -1,14 +1,21 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_attr_data_structures::AttributeKind::{LinkName, LinkOrdinal, LinkSection}; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; +use rustc_feature::Features; +use rustc_hir::attrs::AttributeKind::{LinkName, LinkOrdinal, LinkSection}; +use rustc_hir::attrs::*; +use rustc_session::Session; +use rustc_session::parse::feature_err; +use rustc_span::kw; +use rustc_target::spec::BinaryFormat; -use crate::attributes::{ - AttributeOrder, NoArgsAttributeParser, OnDuplicate, SingleAttributeParser, +use super::prelude::*; +use super::util::parse_single_integer; +use crate::attributes::cfg::parse_cfg_entry; +use crate::fluent_generated; +use crate::session_diagnostics::{ + AsNeededCompatibility, BundleNeedsStatic, EmptyLinkName, ImportNameTypeRaw, ImportNameTypeX86, + IncompatibleWasmLink, InvalidLinkModifier, LinkFrameworkApple, LinkOrdinalOutOfRange, + LinkRequiresName, MultipleModifiers, NullOnLinkSection, RawDylibNoNul, RawDylibOnlyWindows, + WholeArchiveNeedsStatic, }; -use crate::context::{AcceptContext, Stage, parse_single_integer}; -use crate::parser::ArgParser; -use crate::session_diagnostics::{LinkOrdinalOutOfRange, NullOnLinkSection}; pub(crate) struct LinkNameParser; @@ -16,7 +23,14 @@ impl SingleAttributeParser for LinkNameParser { const PATH: &[Symbol] = &[sym::link_name]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(NameValueStr: "name"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::ForeignFn), + Allow(Target::ForeignStatic), + ]); + const TEMPLATE: AttributeTemplate = template!( + NameValueStr: "name", + "https://doc.rust-lang.org/reference/items/external-blocks.html#the-link_name-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(nv) = args.name_value() else { @@ -32,13 +46,438 @@ impl SingleAttributeParser for LinkNameParser { } } +pub(crate) struct LinkParser; + +impl CombineAttributeParser for LinkParser { + type Item = LinkEntry; + const PATH: &[Symbol] = &[sym::link]; + const CONVERT: ConvertFn = AttributeKind::Link; + const TEMPLATE: AttributeTemplate = template!(List: &[ + r#"name = "...""#, + r#"name = "...", kind = "dylib|static|...""#, + r#"name = "...", wasm_import_module = "...""#, + r#"name = "...", import_name_type = "decorated|noprefix|undecorated""#, + r#"name = "...", kind = "dylib|static|...", wasm_import_module = "...", import_name_type = "decorated|noprefix|undecorated""#, + ], "https://doc.rust-lang.org/reference/items/external-blocks.html#the-link-attribute"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); //FIXME Still checked fully in `check_attr.rs` + + fn extend<'c>( + cx: &'c mut AcceptContext<'_, '_, S>, + args: &'c ArgParser<'_>, + ) -> impl IntoIterator + 'c { + let items = match args { + ArgParser::List(list) => list, + // This is an edgecase added because making this a hard error would break too many crates + // Specifically `#[link = "dl"]` is accepted with a FCW + // For more information, see https://github.com/rust-lang/rust/pull/143193 + ArgParser::NameValue(nv) if nv.value_as_str().is_some_and(|v| v == sym::dl) => { + let suggestions = cx.suggestions(); + let span = cx.attr_span; + cx.emit_lint(AttributeLintKind::IllFormedAttributeInput { suggestions }, span); + return None; + } + _ => { + cx.expected_list(cx.attr_span); + return None; + } + }; + + let sess = cx.sess(); + let features = cx.features(); + + let mut name = None; + let mut kind = None; + let mut modifiers = None; + let mut cfg = None; + let mut wasm_import_module = None; + let mut import_name_type = None; + for item in items.mixed() { + let Some(item) = item.meta_item() else { + cx.unexpected_literal(item.span()); + continue; + }; + + let cont = match item.path().word().map(|ident| ident.name) { + Some(sym::name) => Self::parse_link_name(item, &mut name, cx), + Some(sym::kind) => Self::parse_link_kind(item, &mut kind, cx, sess, features), + Some(sym::modifiers) => Self::parse_link_modifiers(item, &mut modifiers, cx), + Some(sym::cfg) => Self::parse_link_cfg(item, &mut cfg, cx, sess, features), + Some(sym::wasm_import_module) => { + Self::parse_link_wasm_import_module(item, &mut wasm_import_module, cx) + } + Some(sym::import_name_type) => { + Self::parse_link_import_name_type(item, &mut import_name_type, cx) + } + _ => { + cx.expected_specific_argument_strings( + item.span(), + &[ + sym::name, + sym::kind, + sym::modifiers, + sym::cfg, + sym::wasm_import_module, + sym::import_name_type, + ], + ); + true + } + }; + if !cont { + return None; + } + } + + // Do this outside the above loop so we don't depend on modifiers coming after kinds + let mut verbatim = None; + if let Some((modifiers, span)) = modifiers { + for modifier in modifiers.as_str().split(',') { + let (modifier, value): (Symbol, bool) = match modifier.strip_prefix(&['+', '-']) { + Some(m) => (Symbol::intern(m), modifier.starts_with('+')), + None => { + cx.emit_err(InvalidLinkModifier { span }); + continue; + } + }; + + macro report_unstable_modifier($feature: ident) { + if !features.$feature() { + // FIXME: make this translatable + #[expect(rustc::untranslatable_diagnostic)] + feature_err( + sess, + sym::$feature, + span, + format!("linking modifier `{modifier}` is unstable"), + ) + .emit(); + } + } + let assign_modifier = |dst: &mut Option| { + if dst.is_some() { + cx.emit_err(MultipleModifiers { span, modifier }); + } else { + *dst = Some(value); + } + }; + match (modifier, &mut kind) { + (sym::bundle, Some(NativeLibKind::Static { bundle, .. })) => { + assign_modifier(bundle) + } + (sym::bundle, _) => { + cx.emit_err(BundleNeedsStatic { span }); + } + + (sym::verbatim, _) => assign_modifier(&mut verbatim), + + ( + sym::whole_dash_archive, + Some(NativeLibKind::Static { whole_archive, .. }), + ) => assign_modifier(whole_archive), + (sym::whole_dash_archive, _) => { + cx.emit_err(WholeArchiveNeedsStatic { span }); + } + + (sym::as_dash_needed, Some(NativeLibKind::Dylib { as_needed })) + | (sym::as_dash_needed, Some(NativeLibKind::Framework { as_needed })) + | (sym::as_dash_needed, Some(NativeLibKind::RawDylib { as_needed })) => { + report_unstable_modifier!(native_link_modifiers_as_needed); + assign_modifier(as_needed) + } + (sym::as_dash_needed, _) => { + cx.emit_err(AsNeededCompatibility { span }); + } + + _ => { + cx.expected_specific_argument_strings( + span, + &[ + sym::bundle, + sym::verbatim, + sym::whole_dash_archive, + sym::as_dash_needed, + ], + ); + } + } + } + } + + if let Some((_, span)) = wasm_import_module { + if name.is_some() || kind.is_some() || modifiers.is_some() || cfg.is_some() { + cx.emit_err(IncompatibleWasmLink { span }); + } + } + + if wasm_import_module.is_some() { + (name, kind) = (wasm_import_module, Some(NativeLibKind::WasmImportModule)); + } + let Some((name, name_span)) = name else { + cx.emit_err(LinkRequiresName { span: cx.attr_span }); + return None; + }; + + // Do this outside of the loop so that `import_name_type` can be specified before `kind`. + if let Some((_, span)) = import_name_type { + if !matches!(kind, Some(NativeLibKind::RawDylib { .. })) { + cx.emit_err(ImportNameTypeRaw { span }); + } + } + + if let Some(NativeLibKind::RawDylib { .. }) = kind + && name.as_str().contains('\0') + { + cx.emit_err(RawDylibNoNul { span: name_span }); + } + + Some(LinkEntry { + span: cx.attr_span, + kind: kind.unwrap_or(NativeLibKind::Unspecified), + name, + cfg, + verbatim, + import_name_type, + }) + } +} + +impl LinkParser { + fn parse_link_name( + item: &MetaItemParser<'_>, + name: &mut Option<(Symbol, Span)>, + cx: &mut AcceptContext<'_, '_, S>, + ) -> bool { + if name.is_some() { + cx.duplicate_key(item.span(), sym::name); + return true; + } + let Some(nv) = item.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::name)); + return false; + }; + let Some(link_name) = nv.value_as_str() else { + cx.expected_name_value(item.span(), Some(sym::name)); + return false; + }; + + if link_name.is_empty() { + cx.emit_err(EmptyLinkName { span: nv.value_span }); + } + *name = Some((link_name, nv.value_span)); + true + } + + fn parse_link_kind( + item: &MetaItemParser<'_>, + kind: &mut Option, + cx: &mut AcceptContext<'_, '_, S>, + sess: &Session, + features: &Features, + ) -> bool { + if kind.is_some() { + cx.duplicate_key(item.span(), sym::kind); + return true; + } + let Some(nv) = item.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::kind)); + return true; + }; + let Some(link_kind) = nv.value_as_str() else { + cx.expected_name_value(item.span(), Some(sym::kind)); + return true; + }; + + let link_kind = match link_kind { + kw::Static => NativeLibKind::Static { bundle: None, whole_archive: None }, + sym::dylib => NativeLibKind::Dylib { as_needed: None }, + sym::framework => { + if !sess.target.is_like_darwin { + cx.emit_err(LinkFrameworkApple { span: nv.value_span }); + } + NativeLibKind::Framework { as_needed: None } + } + sym::raw_dash_dylib => { + if sess.target.is_like_windows { + // raw-dylib is stable and working on Windows + } else if sess.target.binary_format == BinaryFormat::Elf && features.raw_dylib_elf() + { + // raw-dylib is unstable on ELF, but the user opted in + } else if sess.target.binary_format == BinaryFormat::Elf && sess.is_nightly_build() + { + feature_err( + sess, + sym::raw_dylib_elf, + nv.value_span, + fluent_generated::attr_parsing_raw_dylib_elf_unstable, + ) + .emit(); + } else { + cx.emit_err(RawDylibOnlyWindows { span: nv.value_span }); + } + + NativeLibKind::RawDylib { as_needed: None } + } + sym::link_dash_arg => { + if !features.link_arg_attribute() { + feature_err( + sess, + sym::link_arg_attribute, + nv.value_span, + fluent_generated::attr_parsing_link_arg_unstable, + ) + .emit(); + } + NativeLibKind::LinkArg + } + _kind => { + cx.expected_specific_argument_strings( + nv.value_span, + &[ + kw::Static, + sym::dylib, + sym::framework, + sym::raw_dash_dylib, + sym::link_dash_arg, + ], + ); + return true; + } + }; + *kind = Some(link_kind); + true + } + + fn parse_link_modifiers( + item: &MetaItemParser<'_>, + modifiers: &mut Option<(Symbol, Span)>, + cx: &mut AcceptContext<'_, '_, S>, + ) -> bool { + if modifiers.is_some() { + cx.duplicate_key(item.span(), sym::modifiers); + return true; + } + let Some(nv) = item.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::modifiers)); + return true; + }; + let Some(link_modifiers) = nv.value_as_str() else { + cx.expected_name_value(item.span(), Some(sym::modifiers)); + return true; + }; + *modifiers = Some((link_modifiers, nv.value_span)); + true + } + + fn parse_link_cfg( + item: &MetaItemParser<'_>, + cfg: &mut Option, + cx: &mut AcceptContext<'_, '_, S>, + sess: &Session, + features: &Features, + ) -> bool { + if cfg.is_some() { + cx.duplicate_key(item.span(), sym::cfg); + return true; + } + let Some(link_cfg) = item.args().list() else { + cx.expected_list(item.span()); + return true; + }; + let Some(link_cfg) = link_cfg.single() else { + cx.expected_single_argument(item.span()); + return true; + }; + if !features.link_cfg() { + feature_err( + sess, + sym::link_cfg, + item.span(), + fluent_generated::attr_parsing_link_cfg_unstable, + ) + .emit(); + } + *cfg = parse_cfg_entry(cx, link_cfg); + true + } + + fn parse_link_wasm_import_module( + item: &MetaItemParser<'_>, + wasm_import_module: &mut Option<(Symbol, Span)>, + cx: &mut AcceptContext<'_, '_, S>, + ) -> bool { + if wasm_import_module.is_some() { + cx.duplicate_key(item.span(), sym::wasm_import_module); + return true; + } + let Some(nv) = item.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::wasm_import_module)); + return true; + }; + let Some(link_wasm_import_module) = nv.value_as_str() else { + cx.expected_name_value(item.span(), Some(sym::wasm_import_module)); + return true; + }; + *wasm_import_module = Some((link_wasm_import_module, item.span())); + true + } + + fn parse_link_import_name_type( + item: &MetaItemParser<'_>, + import_name_type: &mut Option<(PeImportNameType, Span)>, + cx: &mut AcceptContext<'_, '_, S>, + ) -> bool { + if import_name_type.is_some() { + cx.duplicate_key(item.span(), sym::import_name_type); + return true; + } + let Some(nv) = item.args().name_value() else { + cx.expected_name_value(item.span(), Some(sym::import_name_type)); + return true; + }; + let Some(link_import_name_type) = nv.value_as_str() else { + cx.expected_name_value(item.span(), Some(sym::import_name_type)); + return true; + }; + if cx.sess().target.arch != "x86" { + cx.emit_err(ImportNameTypeX86 { span: item.span() }); + return true; + } + + let link_import_name_type = match link_import_name_type { + sym::decorated => PeImportNameType::Decorated, + sym::noprefix => PeImportNameType::NoPrefix, + sym::undecorated => PeImportNameType::Undecorated, + _ => { + cx.expected_specific_argument_strings( + item.span(), + &[sym::decorated, sym::noprefix, sym::undecorated], + ); + return true; + } + }; + *import_name_type = Some((link_import_name_type, item.span())); + true + } +} + pub(crate) struct LinkSectionParser; impl SingleAttributeParser for LinkSectionParser { const PATH: &[Symbol] = &[sym::link_section]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(NameValueStr: "name"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Static), + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + const TEMPLATE: AttributeTemplate = template!( + NameValueStr: "name", + "https://doc.rust-lang.org/reference/abi.html#the-link_section-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(nv) = args.name_value() else { @@ -64,6 +503,7 @@ pub(crate) struct ExportStableParser; impl NoArgsAttributeParser for ExportStableParser { const PATH: &[Symbol] = &[sym::export_stable]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); //FIXME Still checked fully in `check_attr.rs` const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::ExportStable; } @@ -71,6 +511,7 @@ pub(crate) struct FfiConstParser; impl NoArgsAttributeParser for FfiConstParser { const PATH: &[Symbol] = &[sym::ffi_const]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::ForeignFn)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::FfiConst; } @@ -78,6 +519,7 @@ pub(crate) struct FfiPureParser; impl NoArgsAttributeParser for FfiPureParser { const PATH: &[Symbol] = &[sym::ffi_pure]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::ForeignFn)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::FfiPure; } @@ -85,6 +527,12 @@ pub(crate) struct StdInternalSymbolParser; impl NoArgsAttributeParser for StdInternalSymbolParser { const PATH: &[Symbol] = &[sym::rustc_std_internal_symbol]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::ForeignFn), + Allow(Target::Static), + Allow(Target::ForeignStatic), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::StdInternalSymbol; } @@ -94,7 +542,15 @@ impl SingleAttributeParser for LinkOrdinalParser { const PATH: &[Symbol] = &[sym::link_ordinal]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const TEMPLATE: AttributeTemplate = template!(List: "ordinal"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::ForeignFn), + Allow(Target::ForeignStatic), + Warn(Target::MacroCall), + ]); + const TEMPLATE: AttributeTemplate = template!( + List: &["ordinal"], + "https://doc.rust-lang.org/reference/items/external-blocks.html#the-link_ordinal-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let ordinal = parse_single_integer(cx, args)?; @@ -120,3 +576,87 @@ impl SingleAttributeParser for LinkOrdinalParser { Some(LinkOrdinal { ordinal, span: cx.attr_span }) } } + +pub(crate) struct LinkageParser; + +impl SingleAttributeParser for LinkageParser { + const PATH: &[Symbol] = &[sym::linkage]; + + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Static), + Allow(Target::ForeignStatic), + Allow(Target::ForeignFn), + ]); + + const TEMPLATE: AttributeTemplate = template!(NameValueStr: [ + "available_externally", + "common", + "extern_weak", + "external", + "internal", + "linkonce", + "linkonce_odr", + "weak", + "weak_odr", + ]); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(name_value) = args.name_value() else { + cx.expected_name_value(cx.attr_span, Some(sym::linkage)); + return None; + }; + + let Some(value) = name_value.value_as_str() else { + cx.expected_string_literal(name_value.value_span, Some(name_value.value_as_lit())); + return None; + }; + + // Use the names from src/llvm/docs/LangRef.rst here. Most types are only + // applicable to variable declarations and may not really make sense for + // Rust code in the first place but allow them anyway and trust that the + // user knows what they're doing. Who knows, unanticipated use cases may pop + // up in the future. + // + // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported + // and don't have to be, LLVM treats them as no-ops. + let linkage = match value { + sym::available_externally => Linkage::AvailableExternally, + sym::common => Linkage::Common, + sym::extern_weak => Linkage::ExternalWeak, + sym::external => Linkage::External, + sym::internal => Linkage::Internal, + sym::linkonce => Linkage::LinkOnceAny, + sym::linkonce_odr => Linkage::LinkOnceODR, + sym::weak => Linkage::WeakAny, + sym::weak_odr => Linkage::WeakODR, + + _ => { + cx.expected_specific_argument( + name_value.value_span, + &[ + sym::available_externally, + sym::common, + sym::extern_weak, + sym::external, + sym::internal, + sym::linkonce, + sym::linkonce_odr, + sym::weak, + sym::weak_odr, + ], + ); + return None; + } + }; + + Some(AttributeKind::Linkage(linkage, cx.attr_span)) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/lint_helpers.rs b/compiler/rustc_attr_parsing/src/attributes/lint_helpers.rs index 0eceff53e8b4..63b0809d0d8c 100644 --- a/compiler/rustc_attr_parsing/src/attributes/lint_helpers.rs +++ b/compiler/rustc_attr_parsing/src/attributes/lint_helpers.rs @@ -1,13 +1,16 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_span::{Span, Symbol, sym}; - -use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; -use crate::context::Stage; +use super::prelude::*; pub(crate) struct AsPtrParser; impl NoArgsAttributeParser for AsPtrParser { const PATH: &[Symbol] = &[sym::rustc_as_ptr]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::AsPtr; } @@ -15,6 +18,11 @@ pub(crate) struct PubTransparentParser; impl NoArgsAttributeParser for PubTransparentParser { const PATH: &[Symbol] = &[sym::rustc_pub_transparent]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Struct), + Allow(Target::Enum), + Allow(Target::Union), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::PubTransparent; } @@ -22,6 +30,11 @@ pub(crate) struct PassByValueParser; impl NoArgsAttributeParser for PassByValueParser { const PATH: &[Symbol] = &[sym::rustc_pass_by_value]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Struct), + Allow(Target::Enum), + Allow(Target::TyAlias), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::PassByValue; } @@ -29,5 +42,10 @@ pub(crate) struct AutomaticallyDerivedParser; impl NoArgsAttributeParser for AutomaticallyDerivedParser { const PATH: &[Symbol] = &[sym::automatically_derived]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Impl { of_trait: true }), + Error(Target::Crate), + Error(Target::WherePredicate), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::AutomaticallyDerived; } diff --git a/compiler/rustc_attr_parsing/src/attributes/loop_match.rs b/compiler/rustc_attr_parsing/src/attributes/loop_match.rs index 80808b90dc66..528090b8673d 100644 --- a/compiler/rustc_attr_parsing/src/attributes/loop_match.rs +++ b/compiler/rustc_attr_parsing/src/attributes/loop_match.rs @@ -1,13 +1,10 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_span::{Span, Symbol, sym}; - -use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; -use crate::context::Stage; +use super::prelude::*; pub(crate) struct LoopMatchParser; impl NoArgsAttributeParser for LoopMatchParser { const PATH: &[Symbol] = &[sym::loop_match]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Expression)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::LoopMatch; } @@ -15,5 +12,6 @@ pub(crate) struct ConstContinueParser; impl NoArgsAttributeParser for ConstContinueParser { const PATH: &[Symbol] = &[sym::const_continue]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Expression)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::ConstContinue; } diff --git a/compiler/rustc_attr_parsing/src/attributes/macro_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/macro_attrs.rs index eade49180ace..787003519e78 100644 --- a/compiler/rustc_attr_parsing/src/attributes/macro_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/macro_attrs.rs @@ -1,18 +1,14 @@ -use rustc_attr_data_structures::{AttributeKind, MacroUseArgs}; use rustc_errors::DiagArgValue; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; -use thin_vec::ThinVec; +use rustc_hir::attrs::MacroUseArgs; -use crate::attributes::{AcceptMapping, AttributeParser, NoArgsAttributeParser, OnDuplicate}; -use crate::context::{AcceptContext, FinalizeContext, Stage}; -use crate::parser::ArgParser; -use crate::session_diagnostics; +use super::prelude::*; +use crate::session_diagnostics::IllFormedAttributeInputLint; pub(crate) struct MacroEscapeParser; impl NoArgsAttributeParser for MacroEscapeParser { const PATH: &[Symbol] = &[sym::macro_escape]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = MACRO_USE_ALLOWED_TARGETS; const CREATE: fn(Span) -> AttributeKind = AttributeKind::MacroEscape; } @@ -31,7 +27,16 @@ pub(crate) struct MacroUseParser { first_span: Option, } -const MACRO_USE_TEMPLATE: AttributeTemplate = template!(Word, List: "name1, name2, ..."); +const MACRO_USE_TEMPLATE: AttributeTemplate = template!( + Word, List: &["name1, name2, ..."], + "https://doc.rust-lang.org/reference/macros-by-example.html#the-macro_use-attribute" +); +const MACRO_USE_ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Mod), + Allow(Target::ExternCrate), + Allow(Target::Crate), + Error(Target::WherePredicate), +]); impl AttributeParser for MacroUseParser { const ATTRIBUTES: AcceptMapping = &[( @@ -96,8 +101,8 @@ impl AttributeParser for MacroUseParser { } } ArgParser::NameValue(_) => { - let suggestions = MACRO_USE_TEMPLATE.suggestions(false, sym::macro_use); - cx.emit_err(session_diagnostics::IllFormedAttributeInputLint { + let suggestions = cx.suggestions(); + cx.emit_err(IllFormedAttributeInputLint { num_suggestions: suggestions.len(), suggestions: DiagArgValue::StrListSepByAnd( suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(), @@ -108,8 +113,79 @@ impl AttributeParser for MacroUseParser { } }, )]; + const ALLOWED_TARGETS: AllowedTargets = MACRO_USE_ALLOWED_TARGETS; fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { Some(AttributeKind::MacroUse { span: self.first_span?, arguments: self.state }) } } + +pub(crate) struct AllowInternalUnsafeParser; + +impl NoArgsAttributeParser for AllowInternalUnsafeParser { + const PATH: &[Symbol] = &[sym::allow_internal_unsafe]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Ignore; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::MacroDef), + Warn(Target::Field), + Warn(Target::Arm), + ]); + const CREATE: fn(Span) -> AttributeKind = |span| AttributeKind::AllowInternalUnsafe(span); +} + +pub(crate) struct MacroExportParser; + +impl SingleAttributeParser for MacroExportParser { + const PATH: &[Symbol] = &[sym::macro_export]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const TEMPLATE: AttributeTemplate = template!(Word, List: &["local_inner_macros"]); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::MacroDef), + Error(Target::WherePredicate), + Error(Target::Crate), + ]); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let local_inner_macros = match args { + ArgParser::NoArgs => false, + ArgParser::List(list) => { + let Some(l) = list.single() else { + let span = cx.attr_span; + let suggestions = cx.suggestions(); + cx.emit_lint( + AttributeLintKind::InvalidMacroExportArguments { suggestions }, + span, + ); + return None; + }; + match l.meta_item().and_then(|i| i.path().word_sym()) { + Some(sym::local_inner_macros) => true, + _ => { + let span = cx.attr_span; + let suggestions = cx.suggestions(); + cx.emit_lint( + AttributeLintKind::InvalidMacroExportArguments { suggestions }, + span, + ); + return None; + } + } + } + ArgParser::NameValue(_) => { + let span = cx.attr_span; + let suggestions = cx.suggestions(); + cx.emit_err(IllFormedAttributeInputLint { + num_suggestions: suggestions.len(), + suggestions: DiagArgValue::StrListSepByAnd( + suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(), + ), + span, + }); + return None; + } + }; + Some(AttributeKind::MacroExport { span: cx.attr_span, local_inner_macros }) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs index 0c10517d0440..093969586596 100644 --- a/compiler/rustc_attr_parsing/src/attributes/mod.rs +++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs @@ -7,29 +7,36 @@ //! Specifically, you might not care about managing the state of your [`AttributeParser`] //! state machine yourself. In this case you can choose to implement: //! -//! - [`SingleAttributeParser`]: makes it easy to implement an attribute which should error if it +//! - [`SingleAttributeParser`](crate::attributes::SingleAttributeParser): makes it easy to implement an attribute which should error if it //! appears more than once in a list of attributes -//! - [`CombineAttributeParser`]: makes it easy to implement an attribute which should combine the +//! - [`CombineAttributeParser`](crate::attributes::CombineAttributeParser): makes it easy to implement an attribute which should combine the //! contents of attributes, if an attribute appear multiple times in a list //! //! Attributes should be added to `crate::context::ATTRIBUTE_PARSERS` to be parsed. use std::marker::PhantomData; -use rustc_attr_data_structures::AttributeKind; use rustc_feature::{AttributeTemplate, template}; +use rustc_hir::attrs::AttributeKind; use rustc_span::{Span, Symbol}; use thin_vec::ThinVec; use crate::context::{AcceptContext, FinalizeContext, Stage}; use crate::parser::ArgParser; use crate::session_diagnostics::UnusedMultiple; +use crate::target_checking::AllowedTargets; + +/// All the parsers require roughly the same imports, so this prelude has most of the often-needed ones. +mod prelude; pub(crate) mod allow_unstable; +pub(crate) mod body; pub(crate) mod cfg; pub(crate) mod cfg_old; pub(crate) mod codegen_attrs; pub(crate) mod confusables; +pub(crate) mod crate_level; +pub(crate) mod debugger; pub(crate) mod deprecation; pub(crate) mod dummy; pub(crate) mod inline; @@ -41,7 +48,9 @@ pub(crate) mod must_use; pub(crate) mod no_implicit_prelude; pub(crate) mod non_exhaustive; pub(crate) mod path; +pub(crate) mod pin_v2; pub(crate) mod proc_macro_attrs; +pub(crate) mod prototype; pub(crate) mod repr; pub(crate) mod rustc_internal; pub(crate) mod semantics; @@ -78,6 +87,7 @@ pub(crate) trait AttributeParser: Default + 'static { /// /// If an attribute has this symbol, the `accept` function will be called on it. const ATTRIBUTES: AcceptMapping; + const ALLOWED_TARGETS: AllowedTargets; /// The parser has gotten a chance to accept the attributes on an item, /// here it can produce an attribute. @@ -115,6 +125,8 @@ pub(crate) trait SingleAttributeParser: 'static { /// and this specified whether to, for example, warn or error on the other one. const ON_DUPLICATE: OnDuplicate; + const ALLOWED_TARGETS: AllowedTargets; + /// The template this attribute parser should implement. Used for diagnostics. const TEMPLATE: AttributeTemplate; @@ -162,6 +174,7 @@ impl, S: Stage> AttributeParser for Single } }, )]; + const ALLOWED_TARGETS: AllowedTargets = T::ALLOWED_TARGETS; fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { Some(self.1?.0) @@ -246,6 +259,7 @@ pub(crate) enum AttributeOrder { pub(crate) trait NoArgsAttributeParser: 'static { const PATH: &[Symbol]; const ON_DUPLICATE: OnDuplicate; + const ALLOWED_TARGETS: AllowedTargets; /// Create the [`AttributeKind`] given attribute's [`Span`]. const CREATE: fn(Span) -> AttributeKind; @@ -263,6 +277,7 @@ impl, S: Stage> SingleAttributeParser for Without const PATH: &[Symbol] = T::PATH; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = T::ON_DUPLICATE; + const ALLOWED_TARGETS: AllowedTargets = T::ALLOWED_TARGETS; const TEMPLATE: AttributeTemplate = template!(Word); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { @@ -288,10 +303,12 @@ pub(crate) trait CombineAttributeParser: 'static { type Item; /// A function that converts individual items (of type [`Item`](Self::Item)) into the final attribute. /// - /// For example, individual representations fomr `#[repr(...)]` attributes into an `AttributeKind::Repr(x)`, + /// For example, individual representations from `#[repr(...)]` attributes into an `AttributeKind::Repr(x)`, /// where `x` is a vec of these individual reprs. const CONVERT: ConvertFn; + const ALLOWED_TARGETS: AllowedTargets; + /// The template this attribute parser should implement. Used for diagnostics. const TEMPLATE: AttributeTemplate; @@ -323,15 +340,13 @@ impl, S: Stage> Default for Combine { } impl, S: Stage> AttributeParser for Combine { - const ATTRIBUTES: AcceptMapping = &[( - T::PATH, - >::TEMPLATE, - |group: &mut Combine, cx, args| { + const ATTRIBUTES: AcceptMapping = + &[(T::PATH, T::TEMPLATE, |group: &mut Combine, cx, args| { // Keep track of the span of the first attribute, for diagnostics group.first_span.get_or_insert(cx.attr_span); group.items.extend(T::extend(cx, args)) - }, - )]; + })]; + const ALLOWED_TARGETS: AllowedTargets = T::ALLOWED_TARGETS; fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { if let Some(first_span) = self.first_span { diff --git a/compiler/rustc_attr_parsing/src/attributes/must_use.rs b/compiler/rustc_attr_parsing/src/attributes/must_use.rs index 42af3ed0bfab..51b43e96adf9 100644 --- a/compiler/rustc_attr_parsing/src/attributes/must_use.rs +++ b/compiler/rustc_attr_parsing/src/attributes/must_use.rs @@ -1,12 +1,7 @@ -use rustc_attr_data_structures::AttributeKind; use rustc_errors::DiagArgValue; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Symbol, sym}; -use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; -use crate::session_diagnostics; +use super::prelude::*; +use crate::session_diagnostics::IllFormedAttributeInputLint; pub(crate) struct MustUseParser; @@ -14,7 +9,25 @@ impl SingleAttributeParser for MustUseParser { const PATH: &[Symbol] = &[sym::must_use]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(Word, NameValueStr: "reason"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Fn), + Allow(Target::Enum), + Allow(Target::Struct), + Allow(Target::Union), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::ForeignFn), + // `impl Trait` in return position can trip + // `unused_must_use` if `Trait` is marked as + // `#[must_use]` + Allow(Target::Trait), + Error(Target::WherePredicate), + ]); + const TEMPLATE: AttributeTemplate = template!( + Word, NameValueStr: "reason", + "https://doc.rust-lang.org/reference/attributes/diagnostics.html#the-must_use-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { Some(AttributeKind::MustUse { @@ -32,9 +45,8 @@ impl SingleAttributeParser for MustUseParser { Some(value_str) } ArgParser::List(_) => { - let suggestions = - >::TEMPLATE.suggestions(false, "must_use"); - cx.emit_err(session_diagnostics::IllFormedAttributeInputLint { + let suggestions = cx.suggestions(); + cx.emit_err(IllFormedAttributeInputLint { num_suggestions: suggestions.len(), suggestions: DiagArgValue::StrListSepByAnd( suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(), diff --git a/compiler/rustc_attr_parsing/src/attributes/no_implicit_prelude.rs b/compiler/rustc_attr_parsing/src/attributes/no_implicit_prelude.rs index 47cc925f7f6b..40073ea0f461 100644 --- a/compiler/rustc_attr_parsing/src/attributes/no_implicit_prelude.rs +++ b/compiler/rustc_attr_parsing/src/attributes/no_implicit_prelude.rs @@ -1,13 +1,11 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_span::{Span, sym}; - -use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; -use crate::context::Stage; +use super::prelude::*; pub(crate) struct NoImplicitPreludeParser; impl NoArgsAttributeParser for NoImplicitPreludeParser { const PATH: &[rustc_span::Symbol] = &[sym::no_implicit_prelude]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowListWarnRest(&[Allow(Target::Mod), Allow(Target::Crate)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::NoImplicitPrelude; } diff --git a/compiler/rustc_attr_parsing/src/attributes/non_exhaustive.rs b/compiler/rustc_attr_parsing/src/attributes/non_exhaustive.rs index 94f6a65c74ec..fc41c073fd27 100644 --- a/compiler/rustc_attr_parsing/src/attributes/non_exhaustive.rs +++ b/compiler/rustc_attr_parsing/src/attributes/non_exhaustive.rs @@ -1,13 +1,25 @@ -use rustc_attr_data_structures::AttributeKind; +use rustc_hir::Target; +use rustc_hir::attrs::AttributeKind; use rustc_span::{Span, Symbol, sym}; use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; use crate::context::Stage; +use crate::target_checking::AllowedTargets; +use crate::target_checking::Policy::{Allow, Warn}; pub(crate) struct NonExhaustiveParser; impl NoArgsAttributeParser for NonExhaustiveParser { const PATH: &[Symbol] = &[sym::non_exhaustive]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Enum), + Allow(Target::Struct), + Allow(Target::Variant), + Warn(Target::Field), + Warn(Target::Arm), + Warn(Target::MacroDef), + Warn(Target::MacroCall), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::NonExhaustive; } diff --git a/compiler/rustc_attr_parsing/src/attributes/path.rs b/compiler/rustc_attr_parsing/src/attributes/path.rs index febb1b45a18f..e4cb806bb427 100644 --- a/compiler/rustc_attr_parsing/src/attributes/path.rs +++ b/compiler/rustc_attr_parsing/src/attributes/path.rs @@ -1,10 +1,4 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Symbol, sym}; - -use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; +use super::prelude::*; pub(crate) struct PathParser; @@ -12,7 +6,12 @@ impl SingleAttributeParser for PathParser { const PATH: &[Symbol] = &[sym::path]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; - const TEMPLATE: AttributeTemplate = template!(NameValueStr: "file"); + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowListWarnRest(&[Allow(Target::Mod), Error(Target::Crate)]); + const TEMPLATE: AttributeTemplate = template!( + NameValueStr: "file", + "https://doc.rust-lang.org/reference/items/modules.html#the-path-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(nv) = args.name_value() else { diff --git a/compiler/rustc_attr_parsing/src/attributes/pin_v2.rs b/compiler/rustc_attr_parsing/src/attributes/pin_v2.rs new file mode 100644 index 000000000000..597a9515b004 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/pin_v2.rs @@ -0,0 +1,21 @@ +use rustc_hir::Target; +use rustc_hir::attrs::AttributeKind; +use rustc_span::{Span, Symbol, sym}; + +use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; +use crate::context::Stage; +use crate::target_checking::AllowedTargets; +use crate::target_checking::Policy::Allow; + +pub(crate) struct PinV2Parser; + +impl NoArgsAttributeParser for PinV2Parser { + const PATH: &[Symbol] = &[sym::pin_v2]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Enum), + Allow(Target::Struct), + Allow(Target::Union), + ]); + const CREATE: fn(Span) -> AttributeKind = AttributeKind::PinV2; +} diff --git a/compiler/rustc_attr_parsing/src/attributes/prelude.rs b/compiler/rustc_attr_parsing/src/attributes/prelude.rs new file mode 100644 index 000000000000..980366b5c372 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/prelude.rs @@ -0,0 +1,29 @@ +// data structures +#[doc(hidden)] +pub(super) use rustc_feature::{AttributeTemplate, template}; +#[doc(hidden)] +pub(super) use rustc_hir::attrs::AttributeKind; +#[doc(hidden)] +pub(super) use rustc_hir::lints::AttributeLintKind; +#[doc(hidden)] +pub(super) use rustc_hir::{MethodKind, Target}; +#[doc(hidden)] +pub(super) use rustc_span::{DUMMY_SP, Ident, Span, Symbol, sym}; +#[doc(hidden)] +pub(super) use thin_vec::ThinVec; + +#[doc(hidden)] +pub(super) use crate::attributes::{ + AcceptMapping, AttributeOrder, AttributeParser, CombineAttributeParser, ConvertFn, + NoArgsAttributeParser, OnDuplicate, SingleAttributeParser, +}; +// contexts +#[doc(hidden)] +pub(super) use crate::context::{AcceptContext, FinalizeContext, Stage}; +#[doc(hidden)] +pub(super) use crate::parser::*; +// target checking +#[doc(hidden)] +pub(super) use crate::target_checking::Policy::{Allow, Error, Warn}; +#[doc(hidden)] +pub(super) use crate::target_checking::{ALL_TARGETS, AllowedTargets}; diff --git a/compiler/rustc_attr_parsing/src/attributes/proc_macro_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/proc_macro_attrs.rs index 4de77dc268ea..b9929d6f1f8e 100644 --- a/compiler/rustc_attr_parsing/src/attributes/proc_macro_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/proc_macro_attrs.rs @@ -1,18 +1,13 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; -use thin_vec::ThinVec; +use super::prelude::*; -use crate::attributes::{ - AttributeOrder, NoArgsAttributeParser, OnDuplicate, SingleAttributeParser, -}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; +const PROC_MACRO_ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Fn), Warn(Target::Crate), Warn(Target::MacroCall)]); pub(crate) struct ProcMacroParser; impl NoArgsAttributeParser for ProcMacroParser { const PATH: &[Symbol] = &[sym::proc_macro]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = PROC_MACRO_ALLOWED_TARGETS; const CREATE: fn(Span) -> AttributeKind = AttributeKind::ProcMacro; } @@ -20,6 +15,7 @@ pub(crate) struct ProcMacroAttributeParser; impl NoArgsAttributeParser for ProcMacroAttributeParser { const PATH: &[Symbol] = &[sym::proc_macro_attribute]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = PROC_MACRO_ALLOWED_TARGETS; const CREATE: fn(Span) -> AttributeKind = AttributeKind::ProcMacroAttribute; } @@ -28,8 +24,11 @@ impl SingleAttributeParser for ProcMacroDeriveParser { const PATH: &[Symbol] = &[sym::proc_macro_derive]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const TEMPLATE: AttributeTemplate = - template!(List: "TraitName, /*opt*/ attributes(name1, name2, ...)"); + const ALLOWED_TARGETS: AllowedTargets = PROC_MACRO_ALLOWED_TARGETS; + const TEMPLATE: AttributeTemplate = template!( + List: &["TraitName", "TraitName, attributes(name1, name2, ...)"], + "https://doc.rust-lang.org/reference/procedural-macros.html#derive-macros" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let (trait_name, helper_attrs) = parse_derive_like(cx, args, true)?; @@ -46,8 +45,9 @@ impl SingleAttributeParser for RustcBuiltinMacroParser { const PATH: &[Symbol] = &[sym::rustc_builtin_macro]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::MacroDef)]); const TEMPLATE: AttributeTemplate = - template!(List: "TraitName, /*opt*/ attributes(name1, name2, ...)"); + template!(List: &["TraitName", "TraitName, attributes(name1, name2, ...)"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let (builtin_name, helper_attrs) = parse_derive_like(cx, args, false)?; @@ -100,7 +100,7 @@ fn parse_derive_like( return None; }; if !attr_list.path().word_is(sym::attributes) { - cx.expected_specific_argument(attrs.span(), vec!["attributes"]); + cx.expected_specific_argument(attrs.span(), &[sym::attributes]); return None; } let Some(attr_list) = attr_list.args().list() else { diff --git a/compiler/rustc_attr_parsing/src/attributes/prototype.rs b/compiler/rustc_attr_parsing/src/attributes/prototype.rs new file mode 100644 index 000000000000..80fe82bf5429 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/prototype.rs @@ -0,0 +1,141 @@ +//! Attributes that are only used on function prototypes. + +use rustc_feature::{AttributeTemplate, template}; +use rustc_hir::Target; +use rustc_hir::attrs::{AttributeKind, MirDialect, MirPhase}; +use rustc_span::{Span, Symbol, sym}; + +use super::{AttributeOrder, OnDuplicate}; +use crate::attributes::SingleAttributeParser; +use crate::context::{AcceptContext, Stage}; +use crate::parser::ArgParser; +use crate::target_checking::AllowedTargets; +use crate::target_checking::Policy::Allow; + +pub(crate) struct CustomMirParser; + +impl SingleAttributeParser for CustomMirParser { + const PATH: &[rustc_span::Symbol] = &[sym::custom_mir]; + + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]); + + const TEMPLATE: AttributeTemplate = template!(List: &[r#"dialect = "...", phase = "...""#]); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let Some(list) = args.list() else { + cx.expected_list(cx.attr_span); + return None; + }; + + let mut dialect = None; + let mut phase = None; + let mut failed = false; + + for item in list.mixed() { + let Some(meta_item) = item.meta_item() else { + cx.expected_name_value(item.span(), None); + failed = true; + break; + }; + + if let Some(arg) = meta_item.word_is(sym::dialect) { + extract_value(cx, sym::dialect, arg, meta_item.span(), &mut dialect, &mut failed); + } else if let Some(arg) = meta_item.word_is(sym::phase) { + extract_value(cx, sym::phase, arg, meta_item.span(), &mut phase, &mut failed); + } else if let Some(word) = meta_item.path().word() { + let word = word.to_string(); + cx.unknown_key(meta_item.span(), word, &["dialect", "phase"]); + failed = true; + } else { + cx.expected_name_value(meta_item.span(), None); + failed = true; + }; + } + + let dialect = parse_dialect(cx, dialect, &mut failed); + let phase = parse_phase(cx, phase, &mut failed); + + if failed { + return None; + } + + Some(AttributeKind::CustomMir(dialect, phase, cx.attr_span)) + } +} + +fn extract_value( + cx: &mut AcceptContext<'_, '_, S>, + key: Symbol, + arg: &ArgParser<'_>, + span: Span, + out_val: &mut Option<(Symbol, Span)>, + failed: &mut bool, +) { + if out_val.is_some() { + cx.duplicate_key(span, key); + *failed = true; + return; + } + + let Some(val) = arg.name_value() else { + cx.expected_single_argument(arg.span().unwrap_or(span)); + *failed = true; + return; + }; + + let Some(value_sym) = val.value_as_str() else { + cx.expected_string_literal(val.value_span, Some(val.value_as_lit())); + *failed = true; + return; + }; + + *out_val = Some((value_sym, val.value_span)); +} + +fn parse_dialect( + cx: &mut AcceptContext<'_, '_, S>, + dialect: Option<(Symbol, Span)>, + failed: &mut bool, +) -> Option<(MirDialect, Span)> { + let (dialect, span) = dialect?; + + let dialect = match dialect { + sym::analysis => MirDialect::Analysis, + sym::built => MirDialect::Built, + sym::runtime => MirDialect::Runtime, + + _ => { + cx.expected_specific_argument(span, &[sym::analysis, sym::built, sym::runtime]); + *failed = true; + return None; + } + }; + + Some((dialect, span)) +} + +fn parse_phase( + cx: &mut AcceptContext<'_, '_, S>, + phase: Option<(Symbol, Span)>, + failed: &mut bool, +) -> Option<(MirPhase, Span)> { + let (phase, span) = phase?; + + let phase = match phase { + sym::initial => MirPhase::Initial, + sym::post_cleanup => MirPhase::PostCleanup, + sym::optimized => MirPhase::Optimized, + + _ => { + cx.expected_specific_argument(span, &[sym::initial, sym::post_cleanup, sym::optimized]); + *failed = true; + return None; + } + }; + + Some((phase, span)) +} diff --git a/compiler/rustc_attr_parsing/src/attributes/repr.rs b/compiler/rustc_attr_parsing/src/attributes/repr.rs index 521acbb607c7..0330e2515c7d 100644 --- a/compiler/rustc_attr_parsing/src/attributes/repr.rs +++ b/compiler/rustc_attr_parsing/src/attributes/repr.rs @@ -1,14 +1,9 @@ use rustc_abi::Align; use rustc_ast::{IntTy, LitIntType, LitKind, UintTy}; -use rustc_attr_data_structures::{AttributeKind, IntType, ReprAttr}; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{DUMMY_SP, Span, Symbol, sym}; +use rustc_hir::attrs::{IntType, ReprAttr}; -use super::{AcceptMapping, AttributeParser, CombineAttributeParser, ConvertFn, FinalizeContext}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::{ArgParser, MetaItemListParser, MetaItemParser}; -use crate::session_diagnostics; -use crate::session_diagnostics::IncorrectReprFormatGenericCause; +use super::prelude::*; +use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause}; /// Parse #[repr(...)] forms. /// @@ -26,8 +21,10 @@ impl CombineAttributeParser for ReprParser { const CONVERT: ConvertFn = |items, first_span| AttributeKind::Repr { reprs: items, first_span }; // FIXME(jdonszelmann): never used - const TEMPLATE: AttributeTemplate = - template!(List: "C | Rust | align(...) | packed(...) | | transparent"); + const TEMPLATE: AttributeTemplate = template!( + List: &["C", "Rust", "transparent", "align(...)", "packed(...)", ""], + "https://doc.rust-lang.org/reference/type-layout.html#representations" + ); fn extend<'c>( cx: &'c mut AcceptContext<'_, '_, S>, @@ -58,6 +55,10 @@ impl CombineAttributeParser for ReprParser { reprs } + + //FIXME Still checked fully in `check_attr.rs` + //This one is slightly more complicated because the allowed targets depend on the arguments + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); } macro_rules! int_pat { @@ -275,7 +276,7 @@ pub(crate) struct AlignParser(Option<(Align, Span)>); impl AlignParser { const PATH: &'static [Symbol] = &[sym::rustc_align]; - const TEMPLATE: AttributeTemplate = template!(List: ""); + const TEMPLATE: AttributeTemplate = template!(List: &[""]); fn parse<'c, S: Stage>( &mut self, @@ -316,9 +317,44 @@ impl AlignParser { impl AttributeParser for AlignParser { const ATTRIBUTES: AcceptMapping = &[(Self::PATH, Self::TEMPLATE, Self::parse)]; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::ForeignFn), + ]); fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { let (align, span) = self.0?; Some(AttributeKind::Align { align, span }) } } + +#[derive(Default)] +pub(crate) struct AlignStaticParser(AlignParser); + +impl AlignStaticParser { + const PATH: &'static [Symbol] = &[sym::rustc_align_static]; + const TEMPLATE: AttributeTemplate = AlignParser::TEMPLATE; + + fn parse<'c, S: Stage>( + &mut self, + cx: &'c mut AcceptContext<'_, '_, S>, + args: &'c ArgParser<'_>, + ) { + self.0.parse(cx, args) + } +} + +impl AttributeParser for AlignStaticParser { + const ATTRIBUTES: AcceptMapping = &[(Self::PATH, Self::TEMPLATE, Self::parse)]; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Static), Allow(Target::ForeignStatic)]); + + fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { + let (align, span) = self.0.0?; + Some(AttributeKind::Align { align, span }) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs index 7ca951dc0bbb..455c61097d78 100644 --- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs +++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs @@ -1,18 +1,23 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Symbol, sym}; +use super::prelude::*; +use super::util::parse_single_integer; -use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage, parse_single_integer}; -use crate::parser::ArgParser; +pub(crate) struct RustcMainParser; -pub(crate) struct RustcLayoutScalarValidRangeStart; +impl NoArgsAttributeParser for RustcMainParser { + const PATH: &'static [Symbol] = &[sym::rustc_main]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcMain; +} -impl SingleAttributeParser for RustcLayoutScalarValidRangeStart { +pub(crate) struct RustcLayoutScalarValidRangeStartParser; + +impl SingleAttributeParser for RustcLayoutScalarValidRangeStartParser { const PATH: &'static [Symbol] = &[sym::rustc_layout_scalar_valid_range_start]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const TEMPLATE: AttributeTemplate = template!(List: "start"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]); + const TEMPLATE: AttributeTemplate = template!(List: &["start"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { parse_single_integer(cx, args) @@ -20,13 +25,14 @@ impl SingleAttributeParser for RustcLayoutScalarValidRangeStart { } } -pub(crate) struct RustcLayoutScalarValidRangeEnd; +pub(crate) struct RustcLayoutScalarValidRangeEndParser; -impl SingleAttributeParser for RustcLayoutScalarValidRangeEnd { +impl SingleAttributeParser for RustcLayoutScalarValidRangeEndParser { const PATH: &'static [Symbol] = &[sym::rustc_layout_scalar_valid_range_end]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const TEMPLATE: AttributeTemplate = template!(List: "end"); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]); + const TEMPLATE: AttributeTemplate = template!(List: &["end"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { parse_single_integer(cx, args) @@ -40,6 +46,7 @@ impl SingleAttributeParser for RustcObjectLifetimeDefaultParser { const PATH: &[rustc_span::Symbol] = &[sym::rustc_object_lifetime_default]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]); const TEMPLATE: AttributeTemplate = template!(Word); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { @@ -51,3 +58,21 @@ impl SingleAttributeParser for RustcObjectLifetimeDefaultParser { Some(AttributeKind::RustcObjectLifetimeDefault) } } + +pub(crate) struct RustcSimdMonomorphizeLaneLimitParser; + +impl SingleAttributeParser for RustcSimdMonomorphizeLaneLimitParser { + const PATH: &[Symbol] = &[sym::rustc_simd_monomorphize_lane_limit]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]); + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "N"); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + let ArgParser::NameValue(nv) = args else { + cx.expected_name_value(cx.attr_span, None); + return None; + }; + Some(AttributeKind::RustcSimdMonomorphizeLaneLimit(cx.parse_limit_int(nv)?)) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/semantics.rs b/compiler/rustc_attr_parsing/src/attributes/semantics.rs index 74fdff5d2e18..d7f624832971 100644 --- a/compiler/rustc_attr_parsing/src/attributes/semantics.rs +++ b/compiler/rustc_attr_parsing/src/attributes/semantics.rs @@ -1,12 +1,9 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_span::{Span, Symbol, sym}; - -use crate::attributes::{NoArgsAttributeParser, OnDuplicate}; -use crate::context::Stage; +use super::prelude::*; pub(crate) struct MayDangleParser; impl NoArgsAttributeParser for MayDangleParser { const PATH: &[Symbol] = &[sym::may_dangle]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); //FIXME Still checked fully in `check_attr.rs` const CREATE: fn(span: Span) -> AttributeKind = AttributeKind::MayDangle; } diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs index c54fc6b41f8e..b94e23477ffe 100644 --- a/compiler/rustc_attr_parsing/src/attributes/stability.rs +++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs @@ -1,18 +1,13 @@ use std::num::NonZero; -use rustc_attr_data_structures::{ - AttributeKind, DefaultBodyStability, PartialConstStability, Stability, StabilityLevel, - StableSince, UnstableReason, VERSION_PLACEHOLDER, -}; use rustc_errors::ErrorGuaranteed; -use rustc_feature::template; -use rustc_span::{Ident, Span, Symbol, sym}; +use rustc_hir::{ + DefaultBodyStability, MethodKind, PartialConstStability, Stability, StabilityLevel, + StableSince, Target, UnstableReason, VERSION_PLACEHOLDER, +}; +use super::prelude::*; use super::util::parse_version; -use super::{AcceptMapping, AttributeParser, OnDuplicate}; -use crate::attributes::NoArgsAttributeParser; -use crate::context::{AcceptContext, FinalizeContext, Stage}; -use crate::parser::{ArgParser, MetaItemParser}; use crate::session_diagnostics::{self, UnsupportedLiteralReason}; macro_rules! reject_outside_std { @@ -25,6 +20,36 @@ macro_rules! reject_outside_std { }; } +const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Struct), + Allow(Target::Enum), + Allow(Target::Union), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::Impl { of_trait: false }), + Allow(Target::Impl { of_trait: true }), + Allow(Target::MacroDef), + Allow(Target::Crate), + Allow(Target::Mod), + Allow(Target::Use), // FIXME I don't think this does anything? + Allow(Target::Const), + Allow(Target::AssocConst), + Allow(Target::AssocTy), + Allow(Target::Trait), + Allow(Target::TraitAlias), + Allow(Target::TyAlias), + Allow(Target::Variant), + Allow(Target::Field), + Allow(Target::Param), + Allow(Target::Static), + Allow(Target::ForeignFn), + Allow(Target::ForeignStatic), + Allow(Target::ExternCrate), +]); + #[derive(Default)] pub(crate) struct StabilityParser { allowed_through_unstable_modules: Option, @@ -47,7 +72,7 @@ impl AttributeParser for StabilityParser { const ATTRIBUTES: AcceptMapping = &[ ( &[sym::stable], - template!(List: r#"feature = "name", since = "version""#), + template!(List: &[r#"feature = "name", since = "version""#]), |this, cx, args| { reject_outside_std!(cx); if !this.check_duplicate(cx) @@ -59,7 +84,7 @@ impl AttributeParser for StabilityParser { ), ( &[sym::unstable], - template!(List: r#"feature = "name", reason = "...", issue = "N""#), + template!(List: &[r#"feature = "name", reason = "...", issue = "N""#]), |this, cx, args| { reject_outside_std!(cx); if !this.check_duplicate(cx) @@ -86,6 +111,7 @@ impl AttributeParser for StabilityParser { }, ), ]; + const ALLOWED_TARGETS: AllowedTargets = ALLOWED_TARGETS; fn finalize(mut self, cx: &FinalizeContext<'_, '_, S>) -> Option { if let Some(atum) = self.allowed_through_unstable_modules { @@ -130,7 +156,7 @@ pub(crate) struct BodyStabilityParser { impl AttributeParser for BodyStabilityParser { const ATTRIBUTES: AcceptMapping = &[( &[sym::rustc_default_body_unstable], - template!(List: r#"feature = "name", reason = "...", issue = "N""#), + template!(List: &[r#"feature = "name", reason = "...", issue = "N""#]), |this, cx, args| { reject_outside_std!(cx); if this.stability.is_some() { @@ -141,6 +167,7 @@ impl AttributeParser for BodyStabilityParser { } }, )]; + const ALLOWED_TARGETS: AllowedTargets = ALLOWED_TARGETS; fn finalize(self, _cx: &FinalizeContext<'_, '_, S>) -> Option { let (stability, span) = self.stability?; @@ -153,6 +180,10 @@ pub(crate) struct ConstStabilityIndirectParser; impl NoArgsAttributeParser for ConstStabilityIndirectParser { const PATH: &[Symbol] = &[sym::rustc_const_stable_indirect]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Ignore; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + ]); const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::ConstStabilityIndirect; } @@ -176,34 +207,43 @@ impl ConstStabilityParser { impl AttributeParser for ConstStabilityParser { const ATTRIBUTES: AcceptMapping = &[ - (&[sym::rustc_const_stable], template!(List: r#"feature = "name""#), |this, cx, args| { - reject_outside_std!(cx); + ( + &[sym::rustc_const_stable], + template!(List: &[r#"feature = "name""#]), + |this, cx, args| { + reject_outside_std!(cx); - if !this.check_duplicate(cx) - && let Some((feature, level)) = parse_stability(cx, args) - { - this.stability = Some(( - PartialConstStability { level, feature, promotable: false }, - cx.attr_span, - )); - } - }), - (&[sym::rustc_const_unstable], template!(List: r#"feature = "name""#), |this, cx, args| { - reject_outside_std!(cx); - if !this.check_duplicate(cx) - && let Some((feature, level)) = parse_unstability(cx, args) - { - this.stability = Some(( - PartialConstStability { level, feature, promotable: false }, - cx.attr_span, - )); - } - }), + if !this.check_duplicate(cx) + && let Some((feature, level)) = parse_stability(cx, args) + { + this.stability = Some(( + PartialConstStability { level, feature, promotable: false }, + cx.attr_span, + )); + } + }, + ), + ( + &[sym::rustc_const_unstable], + template!(List: &[r#"feature = "name""#]), + |this, cx, args| { + reject_outside_std!(cx); + if !this.check_duplicate(cx) + && let Some((feature, level)) = parse_unstability(cx, args) + { + this.stability = Some(( + PartialConstStability { level, feature, promotable: false }, + cx.attr_span, + )); + } + }, + ), (&[sym::rustc_promotable], template!(Word), |this, cx, _| { reject_outside_std!(cx); this.promotable = true; }), ]; + const ALLOWED_TARGETS: AllowedTargets = ALLOWED_TARGETS; fn finalize(mut self, cx: &FinalizeContext<'_, '_, S>) -> Option { if self.promotable { diff --git a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs index ee81f64860f8..23ecc0bf7d29 100644 --- a/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/test_attrs.rs @@ -1,11 +1,4 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_attr_data_structures::lints::AttributeLintKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Symbol, sym}; - -use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; +use super::prelude::*; pub(crate) struct IgnoreParser; @@ -13,7 +6,12 @@ impl SingleAttributeParser for IgnoreParser { const PATH: &[Symbol] = &[sym::ignore]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; - const TEMPLATE: AttributeTemplate = template!(Word, NameValueStr: "reason"); + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowListWarnRest(&[Allow(Target::Fn), Error(Target::WherePredicate)]); + const TEMPLATE: AttributeTemplate = template!( + Word, NameValueStr: "reason", + "https://doc.rust-lang.org/reference/attributes/testing.html#the-ignore-attribute" + ); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { Some(AttributeKind::Ignore { @@ -22,8 +20,7 @@ impl SingleAttributeParser for IgnoreParser { ArgParser::NoArgs => None, ArgParser::NameValue(name_value) => { let Some(str_value) = name_value.value_as_str() else { - let suggestions = >::TEMPLATE - .suggestions(false, "ignore"); + let suggestions = cx.suggestions(); let span = cx.attr_span; cx.emit_lint( AttributeLintKind::IllFormedAttributeInput { suggestions }, @@ -34,8 +31,7 @@ impl SingleAttributeParser for IgnoreParser { Some(str_value) } ArgParser::List(_) => { - let suggestions = - >::TEMPLATE.suggestions(false, "ignore"); + let suggestions = cx.suggestions(); let span = cx.attr_span; cx.emit_lint(AttributeLintKind::IllFormedAttributeInput { suggestions }, span); return None; @@ -44,3 +40,59 @@ impl SingleAttributeParser for IgnoreParser { }) } } + +pub(crate) struct ShouldPanicParser; + +impl SingleAttributeParser for ShouldPanicParser { + const PATH: &[Symbol] = &[sym::should_panic]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowListWarnRest(&[Allow(Target::Fn), Error(Target::WherePredicate)]); + const TEMPLATE: AttributeTemplate = template!( + Word, List: &[r#"expected = "reason""#], NameValueStr: "reason", + "https://doc.rust-lang.org/reference/attributes/testing.html#the-should_panic-attribute" + ); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { + Some(AttributeKind::ShouldPanic { + span: cx.attr_span, + reason: match args { + ArgParser::NoArgs => None, + ArgParser::NameValue(name_value) => { + let Some(str_value) = name_value.value_as_str() else { + cx.expected_string_literal( + name_value.value_span, + Some(name_value.value_as_lit()), + ); + return None; + }; + Some(str_value) + } + ArgParser::List(list) => { + let Some(single) = list.single() else { + cx.expected_single_argument(list.span); + return None; + }; + let Some(single) = single.meta_item() else { + cx.expected_name_value(single.span(), Some(sym::expected)); + return None; + }; + if !single.path().word_is(sym::expected) { + cx.expected_specific_argument_strings(list.span, &[sym::expected]); + return None; + } + let Some(nv) = single.args().name_value() else { + cx.expected_name_value(single.span(), Some(sym::expected)); + return None; + }; + let Some(expected) = nv.value_as_str() else { + cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit())); + return None; + }; + Some(expected) + } + }, + }) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/traits.rs b/compiler/rustc_attr_parsing/src/attributes/traits.rs index e69a533699ba..ced3bcad2293 100644 --- a/compiler/rustc_attr_parsing/src/attributes/traits.rs +++ b/compiler/rustc_attr_parsing/src/attributes/traits.rs @@ -1,22 +1,22 @@ -use core::mem; - -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; -use rustc_span::{Span, Symbol, sym}; +use std::mem; +use super::prelude::*; use crate::attributes::{ AttributeOrder, NoArgsAttributeParser, OnDuplicate, SingleAttributeParser, }; use crate::context::{AcceptContext, Stage}; use crate::parser::ArgParser; +use crate::target_checking::Policy::{Allow, Warn}; +use crate::target_checking::{ALL_TARGETS, AllowedTargets}; pub(crate) struct SkipDuringMethodDispatchParser; impl SingleAttributeParser for SkipDuringMethodDispatchParser { const PATH: &[Symbol] = &[sym::rustc_skip_during_method_dispatch]; const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); - const TEMPLATE: AttributeTemplate = template!(List: "array, boxed_slice"); + const TEMPLATE: AttributeTemplate = template!(List: &["array, boxed_slice"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let mut array = false; @@ -42,7 +42,7 @@ impl SingleAttributeParser for SkipDuringMethodDispatchParser { Some(key @ sym::array) => (key, &mut array), Some(key @ sym::boxed_slice) => (key, &mut boxed_slice), _ => { - cx.expected_specific_argument(path.span(), vec!["array", "boxed_slice"]); + cx.expected_specific_argument(path.span(), &[sym::array, sym::boxed_slice]); continue; } }; @@ -58,6 +58,7 @@ pub(crate) struct ParenSugarParser; impl NoArgsAttributeParser for ParenSugarParser { const PATH: &[Symbol] = &[sym::rustc_paren_sugar]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::ParenSugar; } @@ -65,6 +66,7 @@ pub(crate) struct TypeConstParser; impl NoArgsAttributeParser for TypeConstParser { const PATH: &[Symbol] = &[sym::type_const]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::AssocConst)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::TypeConst; } @@ -74,6 +76,12 @@ pub(crate) struct MarkerParser; impl NoArgsAttributeParser for MarkerParser { const PATH: &[Symbol] = &[sym::marker]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Trait), + Warn(Target::Field), + Warn(Target::Arm), + Warn(Target::MacroDef), + ]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::Marker; } @@ -81,6 +89,7 @@ pub(crate) struct DenyExplicitImplParser; impl NoArgsAttributeParser for DenyExplicitImplParser { const PATH: &[Symbol] = &[sym::rustc_deny_explicit_impl]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::DenyExplicitImpl; } @@ -88,6 +97,7 @@ pub(crate) struct DoNotImplementViaObjectParser; impl NoArgsAttributeParser for DoNotImplementViaObjectParser { const PATH: &[Symbol] = &[sym::rustc_do_not_implement_via_object]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::DoNotImplementViaObject; } @@ -98,6 +108,7 @@ pub(crate) struct ConstTraitParser; impl NoArgsAttributeParser for ConstTraitParser { const PATH: &[Symbol] = &[sym::const_trait]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::ConstTrait; } @@ -107,6 +118,7 @@ pub(crate) struct SpecializationTraitParser; impl NoArgsAttributeParser for SpecializationTraitParser { const PATH: &[Symbol] = &[sym::rustc_specialization_trait]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::SpecializationTrait; } @@ -114,6 +126,7 @@ pub(crate) struct UnsafeSpecializationMarkerParser; impl NoArgsAttributeParser for UnsafeSpecializationMarkerParser { const PATH: &[Symbol] = &[sym::rustc_unsafe_specialization_marker]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::UnsafeSpecializationMarker; } @@ -123,6 +136,7 @@ pub(crate) struct CoinductiveParser; impl NoArgsAttributeParser for CoinductiveParser { const PATH: &[Symbol] = &[sym::rustc_coinductive]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::Coinductive; } @@ -130,20 +144,17 @@ pub(crate) struct AllowIncoherentImplParser; impl NoArgsAttributeParser for AllowIncoherentImplParser { const PATH: &[Symbol] = &[sym::rustc_allow_incoherent_impl]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Method(MethodKind::Inherent))]); const CREATE: fn(Span) -> AttributeKind = AttributeKind::AllowIncoherentImpl; } -pub(crate) struct CoherenceIsCoreParser; -impl NoArgsAttributeParser for CoherenceIsCoreParser { - const PATH: &[Symbol] = &[sym::rustc_coherence_is_core]; - const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; - const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::CoherenceIsCore; -} - pub(crate) struct FundamentalParser; impl NoArgsAttributeParser for FundamentalParser { const PATH: &[Symbol] = &[sym::fundamental]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Struct), Allow(Target::Trait)]); const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::Fundamental; } @@ -151,5 +162,6 @@ pub(crate) struct PointeeParser; impl NoArgsAttributeParser for PointeeParser { const PATH: &[Symbol] = &[sym::pointee]; const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(ALL_TARGETS); //FIXME Still checked fully in `check_attr.rs` const CREATE: fn(Span) -> AttributeKind = AttributeKind::Pointee; } diff --git a/compiler/rustc_attr_parsing/src/attributes/transparency.rs b/compiler/rustc_attr_parsing/src/attributes/transparency.rs index c9fdc57cc06e..ea1f5549c4ec 100644 --- a/compiler/rustc_attr_parsing/src/attributes/transparency.rs +++ b/compiler/rustc_attr_parsing/src/attributes/transparency.rs @@ -1,11 +1,6 @@ -use rustc_attr_data_structures::AttributeKind; -use rustc_feature::{AttributeTemplate, template}; use rustc_span::hygiene::Transparency; -use rustc_span::{Symbol, sym}; -use super::{AttributeOrder, OnDuplicate, SingleAttributeParser}; -use crate::context::{AcceptContext, Stage}; -use crate::parser::ArgParser; +use super::prelude::*; pub(crate) struct TransparencyParser; @@ -18,8 +13,9 @@ impl SingleAttributeParser for TransparencyParser { const ON_DUPLICATE: OnDuplicate = OnDuplicate::Custom(|cx, used, unused| { cx.dcx().span_err(vec![used, unused], "multiple macro transparency attributes"); }); + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::MacroDef)]); const TEMPLATE: AttributeTemplate = - template!(NameValueStr: "transparent|semitransparent|opaque"); + template!(NameValueStr: ["transparent", "semitransparent", "opaque"]); fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option { let Some(nv) = args.name_value() else { @@ -33,7 +29,7 @@ impl SingleAttributeParser for TransparencyParser { Some(_) => { cx.expected_specific_argument_strings( nv.value_span, - vec!["transparent", "semitransparent", "opaque"], + &[sym::transparent, sym::semitransparent, sym::opaque], ); None } diff --git a/compiler/rustc_attr_parsing/src/attributes/util.rs b/compiler/rustc_attr_parsing/src/attributes/util.rs index 503d2f1fae16..520fd9da7c2a 100644 --- a/compiler/rustc_attr_parsing/src/attributes/util.rs +++ b/compiler/rustc_attr_parsing/src/attributes/util.rs @@ -1,8 +1,16 @@ -use rustc_ast::attr::{AttributeExt, first_attr_value_str_by_name}; -use rustc_attr_data_structures::RustcVersion; +use std::num::IntErrorKind; + +use rustc_ast::LitKind; +use rustc_ast::attr::AttributeExt; use rustc_feature::is_builtin_attr_name; +use rustc_hir::RustcVersion; +use rustc_hir::limit::Limit; use rustc_span::{Symbol, sym}; +use crate::context::{AcceptContext, Stage}; +use crate::parser::{ArgParser, NameValueParser}; +use crate::session_diagnostics::LimitInvalid; + /// Parse a rustc version number written inside string literal in an attribute, /// like appears in `since = "1.0.0"`. Suffixes like "-dev" and "-nightly" are /// not accepted in this position, unlike when parsing CFG_RELEASE. @@ -20,11 +28,8 @@ pub fn parse_version(s: Symbol) -> Option { } pub fn is_builtin_attr(attr: &impl AttributeExt) -> bool { - attr.is_doc_comment() || attr.ident().is_some_and(|ident| is_builtin_attr_name(ident.name)) -} - -pub fn find_crate_name(attrs: &[impl AttributeExt]) -> Option { - first_attr_value_str_by_name(attrs, sym::crate_name) + attr.is_doc_comment().is_some() + || attr.ident().is_some_and(|ident| is_builtin_attr_name(ident.name)) } pub fn is_doc_alias_attrs_contain_symbol<'tcx, T: AttributeExt + 'tcx>( @@ -56,3 +61,63 @@ pub fn is_doc_alias_attrs_contain_symbol<'tcx, T: AttributeExt + 'tcx>( } false } + +/// Parse a single integer. +/// +/// Used by attributes that take a single integer as argument, such as +/// `#[link_ordinal]` and `#[rustc_layout_scalar_valid_range_start]`. +/// `cx` is the context given to the attribute. +/// `args` is the parser for the attribute arguments. +pub(crate) fn parse_single_integer( + cx: &mut AcceptContext<'_, '_, S>, + args: &ArgParser<'_>, +) -> Option { + let Some(list) = args.list() else { + cx.expected_list(cx.attr_span); + return None; + }; + let Some(single) = list.single() else { + cx.expected_single_argument(list.span); + return None; + }; + let Some(lit) = single.lit() else { + cx.expected_integer_literal(single.span()); + return None; + }; + let LitKind::Int(num, _ty) = lit.kind else { + cx.expected_integer_literal(single.span()); + return None; + }; + Some(num.0) +} + +impl AcceptContext<'_, '_, S> { + pub(crate) fn parse_limit_int(&self, nv: &NameValueParser) -> Option { + let Some(limit) = nv.value_as_str() else { + self.expected_string_literal(nv.value_span, Some(nv.value_as_lit())); + return None; + }; + + let error_str = match limit.as_str().parse() { + Ok(i) => return Some(Limit::new(i)), + Err(e) => match e.kind() { + IntErrorKind::PosOverflow => "`limit` is too large", + IntErrorKind::Empty => "`limit` must be a non-negative integer", + IntErrorKind::InvalidDigit => "not a valid integer", + IntErrorKind::NegOverflow => { + panic!( + "`limit` should never negatively overflow since we're parsing into a usize and we'd get Empty instead" + ) + } + IntErrorKind::Zero => { + panic!("zero is a valid `limit` so should have returned Ok() when parsing") + } + kind => panic!("unimplemented IntErrorKind variant: {:?}", kind), + }, + }; + + self.emit_err(LimitInvalid { span: self.attr_span, value_span: nv.value_span, error_str }); + + None + } +} diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs index 9b86d1018408..15904fd7d334 100644 --- a/compiler/rustc_attr_parsing/src/context.rs +++ b/compiler/rustc_attr_parsing/src/context.rs @@ -4,73 +4,94 @@ use std::ops::{Deref, DerefMut}; use std::sync::LazyLock; use private::Sealed; -use rustc_ast::{self as ast, LitKind, MetaItemLit, NodeId}; -use rustc_attr_data_structures::AttributeKind; -use rustc_attr_data_structures::lints::{AttributeLint, AttributeLintKind}; -use rustc_errors::{DiagCtxtHandle, Diagnostic}; -use rustc_feature::{AttributeTemplate, Features}; -use rustc_hir::{AttrArgs, AttrItem, AttrPath, Attribute, HashIgnoredAttrId, HirId}; +use rustc_ast::{AttrStyle, CRATE_NODE_ID, MetaItemLit, NodeId}; +use rustc_errors::{Diag, Diagnostic, Level}; +use rustc_feature::AttributeTemplate; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::lints::{AttributeLint, AttributeLintKind}; +use rustc_hir::{AttrPath, CRATE_HIR_ID, HirId}; use rustc_session::Session; -use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, Symbol, sym}; +use rustc_span::{ErrorGuaranteed, Span, Symbol}; +use crate::AttributeParser; use crate::attributes::allow_unstable::{ AllowConstFnUnstableParser, AllowInternalUnstableParser, UnstableFeatureBoundParser, }; +use crate::attributes::body::CoroutineParser; use crate::attributes::codegen_attrs::{ - ColdParser, CoverageParser, ExportNameParser, NakedParser, NoMangleParser, - OmitGdbPrettyPrinterSectionParser, OptimizeParser, TargetFeatureParser, TrackCallerParser, - UsedParser, + ColdParser, CoverageParser, ExportNameParser, ForceTargetFeatureParser, NakedParser, + NoMangleParser, ObjcClassParser, ObjcSelectorParser, OptimizeParser, SanitizeParser, + TargetFeatureParser, TrackCallerParser, UsedParser, }; use crate::attributes::confusables::ConfusablesParser; +use crate::attributes::crate_level::{ + CrateNameParser, MoveSizeLimitParser, NoCoreParser, NoStdParser, PatternComplexityLimitParser, + RecursionLimitParser, RustcCoherenceIsCoreParser, TypeLengthLimitParser, +}; +use crate::attributes::debugger::DebuggerViualizerParser; use crate::attributes::deprecation::DeprecationParser; use crate::attributes::dummy::DummyParser; use crate::attributes::inline::{InlineParser, RustcForceInlineParser}; use crate::attributes::link_attrs::{ ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser, - LinkSectionParser, StdInternalSymbolParser, + LinkParser, LinkSectionParser, LinkageParser, StdInternalSymbolParser, }; use crate::attributes::lint_helpers::{ AsPtrParser, AutomaticallyDerivedParser, PassByValueParser, PubTransparentParser, }; use crate::attributes::loop_match::{ConstContinueParser, LoopMatchParser}; -use crate::attributes::macro_attrs::{MacroEscapeParser, MacroUseParser}; +use crate::attributes::macro_attrs::{ + AllowInternalUnsafeParser, MacroEscapeParser, MacroExportParser, MacroUseParser, +}; use crate::attributes::must_use::MustUseParser; use crate::attributes::no_implicit_prelude::NoImplicitPreludeParser; use crate::attributes::non_exhaustive::NonExhaustiveParser; use crate::attributes::path::PathParser as PathAttributeParser; +use crate::attributes::pin_v2::PinV2Parser; use crate::attributes::proc_macro_attrs::{ ProcMacroAttributeParser, ProcMacroDeriveParser, ProcMacroParser, RustcBuiltinMacroParser, }; -use crate::attributes::repr::{AlignParser, ReprParser}; +use crate::attributes::prototype::CustomMirParser; +use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser}; use crate::attributes::rustc_internal::{ - RustcLayoutScalarValidRangeEnd, RustcLayoutScalarValidRangeStart, - RustcObjectLifetimeDefaultParser, + RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser, RustcMainParser, + RustcObjectLifetimeDefaultParser, RustcSimdMonomorphizeLaneLimitParser, }; use crate::attributes::semantics::MayDangleParser; use crate::attributes::stability::{ BodyStabilityParser, ConstStabilityIndirectParser, ConstStabilityParser, StabilityParser, }; -use crate::attributes::test_attrs::IgnoreParser; +use crate::attributes::test_attrs::{IgnoreParser, ShouldPanicParser}; use crate::attributes::traits::{ - AllowIncoherentImplParser, CoherenceIsCoreParser, CoinductiveParser, ConstTraitParser, - DenyExplicitImplParser, DoNotImplementViaObjectParser, FundamentalParser, MarkerParser, - ParenSugarParser, PointeeParser, SkipDuringMethodDispatchParser, SpecializationTraitParser, - TypeConstParser, UnsafeSpecializationMarkerParser, + AllowIncoherentImplParser, CoinductiveParser, ConstTraitParser, DenyExplicitImplParser, + DoNotImplementViaObjectParser, FundamentalParser, MarkerParser, ParenSugarParser, + PointeeParser, SkipDuringMethodDispatchParser, SpecializationTraitParser, TypeConstParser, + UnsafeSpecializationMarkerParser, }; use crate::attributes::transparency::TransparencyParser; use crate::attributes::{AttributeParser as _, Combine, Single, WithoutArgs}; -use crate::parser::{ArgParser, MetaItemParser, PathParser}; +use crate::parser::{ArgParser, PathParser}; use crate::session_diagnostics::{AttributeParseError, AttributeParseErrorReason, UnknownMetaItem}; +use crate::target_checking::AllowedTargets; -macro_rules! group_type { - ($stage: ty) => { - LazyLock<( - BTreeMap<&'static [Symbol], Vec<(AttributeTemplate, Box Fn(&mut AcceptContext<'_, 'sess, $stage>, &ArgParser<'a>) + Send + Sync>)>>, - Vec) -> Option>> - )> - }; +type GroupType = LazyLock>; + +pub(super) struct GroupTypeInner { + pub(super) accepters: BTreeMap<&'static [Symbol], Vec>>, + pub(super) finalizers: Vec>, } +pub(super) struct GroupTypeInnerAccept { + pub(super) template: AttributeTemplate, + pub(super) accept_fn: AcceptFn, + pub(super) allowed_targets: AllowedTargets, +} + +type AcceptFn = + Box Fn(&mut AcceptContext<'_, 'sess, S>, &ArgParser<'a>) + Send + Sync>; +type FinalizeFn = + Box) -> Option>; + macro_rules! attribute_parsers { ( pub(crate) static $name: ident = [$($names: ty),* $(,)?]; @@ -93,11 +114,11 @@ macro_rules! attribute_parsers { } }; ( - @[$ty: ty] pub(crate) static $name: ident = [$($names: ty),* $(,)?]; + @[$stage: ty] pub(crate) static $name: ident = [$($names: ty),* $(,)?]; ) => { - pub(crate) static $name: group_type!($ty) = LazyLock::new(|| { - let mut accepts = BTreeMap::<_, Vec<(AttributeTemplate, Box Fn(&mut AcceptContext<'_, 'sess, $ty>, &ArgParser<'a>) + Send + Sync>)>>::new(); - let mut finalizes = Vec::) -> Option>>::new(); + pub(crate) static $name: GroupType<$stage> = LazyLock::new(|| { + let mut accepts = BTreeMap::<_, Vec>>::new(); + let mut finalizes = Vec::>::new(); $( { thread_local! { @@ -105,11 +126,15 @@ macro_rules! attribute_parsers { }; for (path, template, accept_fn) in <$names>::ATTRIBUTES { - accepts.entry(*path).or_default().push((*template, Box::new(|cx, args| { - STATE_OBJECT.with_borrow_mut(|s| { - accept_fn(s, cx, args) - }) - }))); + accepts.entry(*path).or_default().push(GroupTypeInnerAccept { + template: *template, + accept_fn: Box::new(|cx, args| { + STATE_OBJECT.with_borrow_mut(|s| { + accept_fn(s, cx, args) + }) + }), + allowed_targets: <$names as crate::attributes::AttributeParser<$stage>>::ALLOWED_TARGETS, + }); } finalizes.push(Box::new(|cx| { @@ -119,7 +144,7 @@ macro_rules! attribute_parsers { } )* - (accepts, finalizes) + GroupTypeInner { accepters:accepts, finalizers:finalizes } }); }; } @@ -127,6 +152,7 @@ attribute_parsers!( pub(crate) static ATTRIBUTE_PARSERS = [ // tidy-alphabetical-start AlignParser, + AlignStaticParser, BodyStabilityParser, ConfusablesParser, ConstStabilityParser, @@ -139,6 +165,9 @@ attribute_parsers!( // tidy-alphabetical-start Combine, Combine, + Combine, + Combine, + Combine, Combine, Combine, Combine, @@ -146,6 +175,8 @@ attribute_parsers!( // tidy-alphabetical-start Single, + Single, + Single, Single, Single, Single, @@ -154,26 +185,38 @@ attribute_parsers!( Single, Single, Single, + Single, + Single, + Single, Single, + Single, + Single, Single, Single, + Single, Single, + Single, Single, Single, - Single, - Single, + Single, + Single, Single, + Single, + Single, + Single, Single, Single, + Single, Single>, + Single>, Single>, Single>, - Single>, Single>, Single>, Single>, Single>, Single>, + Single>, Single>, Single>, Single>, @@ -184,16 +227,20 @@ attribute_parsers!( Single>, Single>, Single>, + Single>, Single>, Single>, + Single>, Single>, - Single>, Single>, Single>, + Single>, Single>, Single>, Single>, Single>, + Single>, + Single>, Single>, Single>, Single>, @@ -213,24 +260,26 @@ mod private { #[allow(private_interfaces)] pub trait Stage: Sized + 'static + Sealed { type Id: Copy; - const SHOULD_EMIT_LINTS: bool; - fn parsers() -> &'static group_type!(Self); + fn parsers() -> &'static GroupType; fn emit_err<'sess>( &self, sess: &'sess Session, diag: impl for<'x> Diagnostic<'x>, ) -> ErrorGuaranteed; + + fn should_emit(&self) -> ShouldEmit; + + fn id_is_crate_root(id: Self::Id) -> bool; } // allow because it's a sealed trait #[allow(private_interfaces)] impl Stage for Early { type Id = NodeId; - const SHOULD_EMIT_LINTS: bool = false; - fn parsers() -> &'static group_type!(Self) { + fn parsers() -> &'static GroupType { &early::ATTRIBUTE_PARSERS } fn emit_err<'sess>( @@ -238,11 +287,15 @@ impl Stage for Early { sess: &'sess Session, diag: impl for<'x> Diagnostic<'x>, ) -> ErrorGuaranteed { - if self.emit_errors.should_emit() { - sess.dcx().emit_err(diag) - } else { - sess.dcx().create_err(diag).delay_as_bug() - } + self.should_emit().emit_err(sess.dcx().create_err(diag)) + } + + fn should_emit(&self) -> ShouldEmit { + self.emit_errors + } + + fn id_is_crate_root(id: Self::Id) -> bool { + id == CRATE_NODE_ID } } @@ -250,9 +303,8 @@ impl Stage for Early { #[allow(private_interfaces)] impl Stage for Late { type Id = HirId; - const SHOULD_EMIT_LINTS: bool = true; - fn parsers() -> &'static group_type!(Self) { + fn parsers() -> &'static GroupType { &late::ATTRIBUTE_PARSERS } fn emit_err<'sess>( @@ -262,6 +314,14 @@ impl Stage for Late { ) -> ErrorGuaranteed { tcx.dcx().emit_err(diag) } + + fn should_emit(&self) -> ShouldEmit { + ShouldEmit::ErrorsAndLints + } + + fn id_is_crate_root(id: Self::Id) -> bool { + id == CRATE_HIR_ID + } } /// used when parsing attributes for miscellaneous things *before* ast lowering @@ -279,8 +339,19 @@ pub struct Late; /// Gives [`AttributeParser`]s enough information to create errors, for example. pub struct AcceptContext<'f, 'sess, S: Stage> { pub(crate) shared: SharedContext<'f, 'sess, S>, - /// The span of the attribute currently being parsed + + /// The outer span of the attribute currently being parsed + /// #[attribute(...)] + /// ^^^^^^^^^^^^^^^^^ outer span + /// For attributes in `cfg_attr`, the outer span and inner spans are equal. pub(crate) attr_span: Span, + /// The inner span of the attribute currently being parsed + /// #[attribute(...)] + /// ^^^^^^^^^^^^^^ inner span + pub(crate) inner_span: Span, + + /// Whether it is an inner or outer attribute + pub(crate) attr_style: AttrStyle, /// The expected structure of the attribute. /// @@ -300,7 +371,10 @@ impl<'f, 'sess: 'f, S: Stage> SharedContext<'f, 'sess, S> { /// must be delayed until after HIR is built. This method will take care of the details of /// that. pub(crate) fn emit_lint(&mut self, lint: AttributeLintKind, span: Span) { - if !S::SHOULD_EMIT_LINTS { + if !matches!( + self.stage.should_emit(), + ShouldEmit::ErrorsAndLints | ShouldEmit::EarlyFatal { also_emit_lints: true } + ) { return; } let id = self.target_id; @@ -363,6 +437,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { i.kind.is_bytestr().then(|| self.sess().source_map().start_point(i.span)) }), }, + suggestions: self.suggestions(), }) } @@ -373,6 +448,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedIntegerLiteral, + suggestions: self.suggestions(), }) } @@ -383,6 +459,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedList, + suggestions: self.suggestions(), }) } @@ -393,6 +470,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedNoArgs, + suggestions: self.suggestions(), }) } @@ -404,6 +482,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedIdentifier, + suggestions: self.suggestions(), }) } @@ -416,6 +495,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedNameValue(name), + suggestions: self.suggestions(), }) } @@ -427,6 +507,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::DuplicateKey(key), + suggestions: self.suggestions(), }) } @@ -439,6 +520,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::UnexpectedLiteral, + suggestions: self.suggestions(), }) } @@ -449,6 +531,7 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedSingleArgument, + suggestions: self.suggestions(), }) } @@ -459,13 +542,15 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { template: self.template.clone(), attribute: self.attr_path.clone(), reason: AttributeParseErrorReason::ExpectedAtLeastOneArgument, + suggestions: self.suggestions(), }) } + /// produces an error along the lines of `expected one of [foo, meow]` pub(crate) fn expected_specific_argument( &self, span: Span, - possibilities: Vec<&'static str>, + possibilities: &[Symbol], ) -> ErrorGuaranteed { self.emit_err(AttributeParseError { span, @@ -477,13 +562,16 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { strings: false, list: false, }, + suggestions: self.suggestions(), }) } + /// produces an error along the lines of `expected one of [foo, meow] as an argument`. + /// i.e. slightly different wording to [`expected_specific_argument`](Self::expected_specific_argument). pub(crate) fn expected_specific_argument_and_list( &self, span: Span, - possibilities: Vec<&'static str>, + possibilities: &[Symbol], ) -> ErrorGuaranteed { self.emit_err(AttributeParseError { span, @@ -495,13 +583,15 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { strings: false, list: true, }, + suggestions: self.suggestions(), }) } + /// produces an error along the lines of `expected one of ["foo", "meow"]` pub(crate) fn expected_specific_argument_strings( &self, span: Span, - possibilities: Vec<&'static str>, + possibilities: &[Symbol], ) -> ErrorGuaranteed { self.emit_err(AttributeParseError { span, @@ -513,11 +603,24 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { strings: true, list: false, }, + suggestions: self.suggestions(), }) } pub(crate) fn warn_empty_attribute(&mut self, span: Span) { - self.emit_lint(AttributeLintKind::EmptyAttribute { first_span: span }, span); + let attr_path = self.attr_path.clone(); + let valid_without_list = self.template.word; + self.emit_lint( + AttributeLintKind::EmptyAttribute { first_span: span, attr_path, valid_without_list }, + span, + ); + } + + pub(crate) fn suggestions(&self) -> Vec { + // If the outer and inner spans are equal, we are parsing an attribute from `cfg_attr`, + // So don't display an attribute style in the suggestions + let style = (self.attr_span != self.inner_span).then_some(self.attr_style); + self.template.suggestions(style, &self.attr_path) } } @@ -548,7 +651,7 @@ pub struct SharedContext<'p, 'sess, S: Stage> { /// The id ([`NodeId`] if `S` is `Early`, [`HirId`] if `S` is `Late`) of the syntactical component this attribute was applied to pub(crate) target_id: S::Id, - emit_lint: &'p mut dyn FnMut(AttributeLint), + pub(crate) emit_lint: &'p mut dyn FnMut(AttributeLint), } /// Context given to every attribute parser during finalization. @@ -601,343 +704,28 @@ pub enum OmitDoc { Skip, } -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] pub enum ShouldEmit { + /// The operations will emit errors, and lints, and errors are fatal. + /// + /// Only relevant when early parsing, in late parsing equivalent to `ErrorsAndLints`. + /// Late parsing is never fatal, and instead tries to emit as many diagnostics as possible. + EarlyFatal { also_emit_lints: bool }, /// The operation will emit errors and lints. /// This is usually what you need. ErrorsAndLints, /// The operation will emit *not* errors and lints. - /// Use this if you are *sure* that this operation will be called at a different time with `ShouldEmit::Emit`. + /// Use this if you are *sure* that this operation will be called at a different time with `ShouldEmit::ErrorsAndLints`. Nothing, } impl ShouldEmit { - pub fn should_emit(&self) -> bool { + pub(crate) fn emit_err(&self, diag: Diag<'_>) -> ErrorGuaranteed { match self { - ShouldEmit::ErrorsAndLints => true, - ShouldEmit::Nothing => false, + ShouldEmit::EarlyFatal { .. } if diag.level() == Level::DelayedBug => diag.emit(), + ShouldEmit::EarlyFatal { .. } => diag.upgrade_to_fatal().emit(), + ShouldEmit::ErrorsAndLints => diag.emit(), + ShouldEmit::Nothing => diag.delay_as_bug(), } } } - -/// Context created once, for example as part of the ast lowering -/// context, through which all attributes can be lowered. -pub struct AttributeParser<'sess, S: Stage = Late> { - pub(crate) tools: Vec, - features: Option<&'sess Features>, - sess: &'sess Session, - stage: S, - - /// *Only* parse attributes with this symbol. - /// - /// Used in cases where we want the lowering infrastructure for parse just a single attribute. - parse_only: Option, -} - -impl<'sess> AttributeParser<'sess, Early> { - /// This method allows you to parse attributes *before* you have access to features or tools. - /// One example where this is necessary, is to parse `feature` attributes themselves for - /// example. - /// - /// Try to use this as little as possible. Attributes *should* be lowered during - /// `rustc_ast_lowering`. Some attributes require access to features to parse, which would - /// crash if you tried to do so through [`parse_limited`](Self::parse_limited). - /// - /// To make sure use is limited, supply a `Symbol` you'd like to parse. Only attributes with - /// that symbol are picked out of the list of instructions and parsed. Those are returned. - /// - /// No diagnostics will be emitted when parsing limited. Lints are not emitted at all, while - /// errors will be emitted as a delayed bugs. in other words, we *expect* attributes parsed - /// with `parse_limited` to be reparsed later during ast lowering where we *do* emit the errors - pub fn parse_limited( - sess: &'sess Session, - attrs: &[ast::Attribute], - sym: Symbol, - target_span: Span, - target_node_id: NodeId, - features: Option<&'sess Features>, - ) -> Option { - let mut p = Self { - features, - tools: Vec::new(), - parse_only: Some(sym), - sess, - stage: Early { emit_errors: ShouldEmit::Nothing }, - }; - let mut parsed = p.parse_attribute_list( - attrs, - target_span, - target_node_id, - OmitDoc::Skip, - std::convert::identity, - |_lint| { - panic!("can't emit lints here for now (nothing uses this atm)"); - }, - ); - assert!(parsed.len() <= 1); - - parsed.pop() - } - - pub fn parse_single( - sess: &'sess Session, - attr: &ast::Attribute, - target_span: Span, - target_node_id: NodeId, - features: Option<&'sess Features>, - emit_errors: ShouldEmit, - parse_fn: fn(cx: &mut AcceptContext<'_, '_, Early>, item: &ArgParser<'_>) -> T, - template: &AttributeTemplate, - ) -> T { - let mut parser = Self { - features, - tools: Vec::new(), - parse_only: None, - sess, - stage: Early { emit_errors }, - }; - let ast::AttrKind::Normal(normal_attr) = &attr.kind else { - panic!("parse_single called on a doc attr") - }; - let meta_parser = MetaItemParser::from_attr(normal_attr, parser.dcx()); - let path = meta_parser.path(); - let args = meta_parser.args(); - let mut cx: AcceptContext<'_, 'sess, Early> = AcceptContext { - shared: SharedContext { - cx: &mut parser, - target_span, - target_id: target_node_id, - emit_lint: &mut |_lint| { - panic!("can't emit lints here for now (nothing uses this atm)"); - }, - }, - attr_span: attr.span, - template, - attr_path: path.get_attribute_path(), - }; - parse_fn(&mut cx, args) - } -} - -impl<'sess, S: Stage> AttributeParser<'sess, S> { - pub fn new( - sess: &'sess Session, - features: &'sess Features, - tools: Vec, - stage: S, - ) -> Self { - Self { features: Some(features), tools, parse_only: None, sess, stage } - } - - pub(crate) fn sess(&self) -> &'sess Session { - &self.sess - } - - pub(crate) fn features(&self) -> &'sess Features { - self.features.expect("features not available at this point in the compiler") - } - - pub(crate) fn features_option(&self) -> Option<&'sess Features> { - self.features - } - - pub(crate) fn dcx(&self) -> DiagCtxtHandle<'sess> { - self.sess().dcx() - } - - /// Parse a list of attributes. - /// - /// `target_span` is the span of the thing this list of attributes is applied to, - /// and when `omit_doc` is set, doc attributes are filtered out. - pub fn parse_attribute_list( - &mut self, - attrs: &[ast::Attribute], - target_span: Span, - target_id: S::Id, - omit_doc: OmitDoc, - - lower_span: impl Copy + Fn(Span) -> Span, - mut emit_lint: impl FnMut(AttributeLint), - ) -> Vec { - let mut attributes = Vec::new(); - let mut attr_paths = Vec::new(); - - for attr in attrs { - // If we're only looking for a single attribute, skip all the ones we don't care about. - if let Some(expected) = self.parse_only { - if !attr.has_name(expected) { - continue; - } - } - - // Sometimes, for example for `#![doc = include_str!("readme.md")]`, - // doc still contains a non-literal. You might say, when we're lowering attributes - // that's expanded right? But no, sometimes, when parsing attributes on macros, - // we already use the lowering logic and these are still there. So, when `omit_doc` - // is set we *also* want to ignore these. - if omit_doc == OmitDoc::Skip && attr.has_name(sym::doc) { - continue; - } - - match &attr.kind { - ast::AttrKind::DocComment(comment_kind, symbol) => { - if omit_doc == OmitDoc::Skip { - continue; - } - - attributes.push(Attribute::Parsed(AttributeKind::DocComment { - style: attr.style, - kind: *comment_kind, - span: lower_span(attr.span), - comment: *symbol, - })) - } - // // FIXME: make doc attributes go through a proper attribute parser - // ast::AttrKind::Normal(n) if n.has_name(sym::doc) => { - // let p = GenericMetaItemParser::from_attr(&n, self.dcx()); - // - // attributes.push(Attribute::Parsed(AttributeKind::DocComment { - // style: attr.style, - // kind: CommentKind::Line, - // span: attr.span, - // comment: p.args().name_value(), - // })) - // } - ast::AttrKind::Normal(n) => { - attr_paths.push(PathParser::Ast(&n.item.path)); - - let parser = MetaItemParser::from_attr(n, self.dcx()); - let path = parser.path(); - let args = parser.args(); - let parts = path.segments().map(|i| i.name).collect::>(); - - if let Some(accepts) = S::parsers().0.get(parts.as_slice()) { - for (template, accept) in accepts { - let mut cx: AcceptContext<'_, 'sess, S> = AcceptContext { - shared: SharedContext { - cx: self, - target_span, - target_id, - emit_lint: &mut emit_lint, - }, - attr_span: lower_span(attr.span), - template, - attr_path: path.get_attribute_path(), - }; - - accept(&mut cx, args) - } - } else { - // If we're here, we must be compiling a tool attribute... Or someone - // forgot to parse their fancy new attribute. Let's warn them in any case. - // If you are that person, and you really think your attribute should - // remain unparsed, carefully read the documentation in this module and if - // you still think so you can add an exception to this assertion. - - // FIXME(jdonszelmann): convert other attributes, and check with this that - // we caught em all - // const FIXME_TEMPORARY_ATTR_ALLOWLIST: &[Symbol] = &[sym::cfg]; - // assert!( - // self.tools.contains(&parts[0]) || true, - // // || FIXME_TEMPORARY_ATTR_ALLOWLIST.contains(&parts[0]), - // "attribute {path} wasn't parsed and isn't a know tool attribute", - // ); - - attributes.push(Attribute::Unparsed(Box::new(AttrItem { - path: AttrPath::from_ast(&n.item.path), - args: self.lower_attr_args(&n.item.args, lower_span), - id: HashIgnoredAttrId { attr_id: attr.id }, - style: attr.style, - span: lower_span(attr.span), - }))); - } - } - } - } - - let mut parsed_attributes = Vec::new(); - for f in &S::parsers().1 { - if let Some(attr) = f(&mut FinalizeContext { - shared: SharedContext { - cx: self, - target_span, - target_id, - emit_lint: &mut emit_lint, - }, - all_attrs: &attr_paths, - }) { - parsed_attributes.push(Attribute::Parsed(attr)); - } - } - - attributes.extend(parsed_attributes); - - attributes - } - - /// Returns whether there is a parser for an attribute with this name - pub fn is_parsed_attribute(path: &[Symbol]) -> bool { - Late::parsers().0.contains_key(path) - } - - fn lower_attr_args(&self, args: &ast::AttrArgs, lower_span: impl Fn(Span) -> Span) -> AttrArgs { - match args { - ast::AttrArgs::Empty => AttrArgs::Empty, - ast::AttrArgs::Delimited(args) => AttrArgs::Delimited(args.clone()), - // This is an inert key-value attribute - it will never be visible to macros - // after it gets lowered to HIR. Therefore, we can extract literals to handle - // nonterminals in `#[doc]` (e.g. `#[doc = $e]`). - ast::AttrArgs::Eq { eq_span, expr } => { - // In valid code the value always ends up as a single literal. Otherwise, a dummy - // literal suffices because the error is handled elsewhere. - let lit = if let ast::ExprKind::Lit(token_lit) = expr.kind - && let Ok(lit) = - ast::MetaItemLit::from_token_lit(token_lit, lower_span(expr.span)) - { - lit - } else { - let guar = self.dcx().span_delayed_bug( - args.span().unwrap_or(DUMMY_SP), - "expr in place where literal is expected (builtin attr parsing)", - ); - ast::MetaItemLit { - symbol: sym::dummy, - suffix: None, - kind: ast::LitKind::Err(guar), - span: DUMMY_SP, - } - }; - AttrArgs::Eq { eq_span: lower_span(*eq_span), expr: lit } - } - } - } -} - -/// Parse a single integer. -/// -/// Used by attributes that take a single integer as argument, such as -/// `#[link_ordinal]` and `#[rustc_layout_scalar_valid_range_start]`. -/// `cx` is the context given to the attribute. -/// `args` is the parser for the attribute arguments. -pub(crate) fn parse_single_integer( - cx: &mut AcceptContext<'_, '_, S>, - args: &ArgParser<'_>, -) -> Option { - let Some(list) = args.list() else { - cx.expected_list(cx.attr_span); - return None; - }; - let Some(single) = list.single() else { - cx.expected_single_argument(list.span); - return None; - }; - let Some(lit) = single.lit() else { - cx.expected_integer_literal(single.span()); - return None; - }; - let LitKind::Int(num, _ty) = lit.kind else { - cx.expected_integer_literal(single.span()); - return None; - }; - Some(num.0) -} diff --git a/compiler/rustc_attr_parsing/src/interface.rs b/compiler/rustc_attr_parsing/src/interface.rs new file mode 100644 index 000000000000..953b0ebfaf04 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/interface.rs @@ -0,0 +1,406 @@ +use std::borrow::Cow; + +use rustc_ast as ast; +use rustc_ast::{AttrStyle, NodeId}; +use rustc_errors::DiagCtxtHandle; +use rustc_feature::{AttributeTemplate, Features}; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::lints::AttributeLint; +use rustc_hir::{AttrArgs, AttrItem, AttrPath, Attribute, HashIgnoredAttrId, Target}; +use rustc_session::Session; +use rustc_span::{DUMMY_SP, Span, Symbol, sym}; + +use crate::context::{AcceptContext, FinalizeContext, SharedContext, Stage}; +use crate::parser::{ArgParser, MetaItemParser, PathParser}; +use crate::{Early, Late, OmitDoc, ShouldEmit}; + +/// Context created once, for example as part of the ast lowering +/// context, through which all attributes can be lowered. +pub struct AttributeParser<'sess, S: Stage = Late> { + pub(crate) tools: Vec, + pub(crate) features: Option<&'sess Features>, + pub(crate) sess: &'sess Session, + pub(crate) stage: S, + + /// *Only* parse attributes with this symbol. + /// + /// Used in cases where we want the lowering infrastructure for parse just a single attribute. + parse_only: Option, +} + +impl<'sess> AttributeParser<'sess, Early> { + /// This method allows you to parse attributes *before* you have access to features or tools. + /// One example where this is necessary, is to parse `feature` attributes themselves for + /// example. + /// + /// Try to use this as little as possible. Attributes *should* be lowered during + /// `rustc_ast_lowering`. Some attributes require access to features to parse, which would + /// crash if you tried to do so through [`parse_limited`](Self::parse_limited). + /// + /// To make sure use is limited, supply a `Symbol` you'd like to parse. Only attributes with + /// that symbol are picked out of the list of instructions and parsed. Those are returned. + /// + /// No diagnostics will be emitted when parsing limited. Lints are not emitted at all, while + /// errors will be emitted as a delayed bugs. in other words, we *expect* attributes parsed + /// with `parse_limited` to be reparsed later during ast lowering where we *do* emit the errors + pub fn parse_limited( + sess: &'sess Session, + attrs: &[ast::Attribute], + sym: Symbol, + target_span: Span, + target_node_id: NodeId, + features: Option<&'sess Features>, + ) -> Option { + Self::parse_limited_should_emit( + sess, + attrs, + sym, + target_span, + target_node_id, + features, + ShouldEmit::Nothing, + ) + } + + /// This does the same as `parse_limited`, except it has a `should_emit` parameter which allows it to emit errors. + /// Usually you want `parse_limited`, which emits no errors. + pub fn parse_limited_should_emit( + sess: &'sess Session, + attrs: &[ast::Attribute], + sym: Symbol, + target_span: Span, + target_node_id: NodeId, + features: Option<&'sess Features>, + should_emit: ShouldEmit, + ) -> Option { + let mut parsed = Self::parse_limited_all( + sess, + attrs, + Some(sym), + Target::Crate, // Does not matter, we're not going to emit errors anyways + target_span, + target_node_id, + features, + should_emit, + ); + assert!(parsed.len() <= 1); + parsed.pop() + } + + /// This method allows you to parse a list of attributes *before* `rustc_ast_lowering`. + /// This can be used for attributes that would be removed before `rustc_ast_lowering`, such as attributes on macro calls. + /// + /// Try to use this as little as possible. Attributes *should* be lowered during + /// `rustc_ast_lowering`. Some attributes require access to features to parse, which would + /// crash if you tried to do so through [`parse_limited_all`](Self::parse_limited_all). + /// Therefore, if `parse_only` is None, then features *must* be provided. + pub fn parse_limited_all( + sess: &'sess Session, + attrs: &[ast::Attribute], + parse_only: Option, + target: Target, + target_span: Span, + target_node_id: NodeId, + features: Option<&'sess Features>, + emit_errors: ShouldEmit, + ) -> Vec { + let mut p = + Self { features, tools: Vec::new(), parse_only, sess, stage: Early { emit_errors } }; + p.parse_attribute_list( + attrs, + target_span, + target_node_id, + target, + OmitDoc::Skip, + std::convert::identity, + |lint| { + crate::lints::emit_attribute_lint(&lint, sess); + }, + ) + } + + /// This method parses a single attribute, using `parse_fn`. + /// This is useful if you already know what exact attribute this is, and want to parse it. + pub fn parse_single( + sess: &'sess Session, + attr: &ast::Attribute, + target_span: Span, + target_node_id: NodeId, + features: Option<&'sess Features>, + emit_errors: ShouldEmit, + parse_fn: fn(cx: &mut AcceptContext<'_, '_, Early>, item: &ArgParser<'_>) -> Option, + template: &AttributeTemplate, + ) -> Option { + let ast::AttrKind::Normal(normal_attr) = &attr.kind else { + panic!("parse_single called on a doc attr") + }; + let parts = + normal_attr.item.path.segments.iter().map(|seg| seg.ident.name).collect::>(); + let meta_parser = MetaItemParser::from_attr(normal_attr, &parts, &sess.psess, emit_errors)?; + let path = meta_parser.path(); + let args = meta_parser.args(); + Self::parse_single_args( + sess, + attr.span, + normal_attr.item.span(), + attr.style, + path.get_attribute_path(), + target_span, + target_node_id, + features, + emit_errors, + args, + parse_fn, + template, + ) + } + + /// This method is equivalent to `parse_single`, but parses arguments using `parse_fn` using manually created `args`. + /// This is useful when you want to parse other things than attributes using attribute parsers. + pub fn parse_single_args( + sess: &'sess Session, + attr_span: Span, + inner_span: Span, + attr_style: AttrStyle, + attr_path: AttrPath, + target_span: Span, + target_node_id: NodeId, + features: Option<&'sess Features>, + emit_errors: ShouldEmit, + args: &I, + parse_fn: fn(cx: &mut AcceptContext<'_, '_, Early>, item: &I) -> Option, + template: &AttributeTemplate, + ) -> Option { + let mut parser = Self { + features, + tools: Vec::new(), + parse_only: None, + sess, + stage: Early { emit_errors }, + }; + let mut cx: AcceptContext<'_, 'sess, Early> = AcceptContext { + shared: SharedContext { + cx: &mut parser, + target_span, + target_id: target_node_id, + emit_lint: &mut |lint| { + crate::lints::emit_attribute_lint(&lint, sess); + }, + }, + attr_span, + inner_span, + attr_style, + template, + attr_path, + }; + parse_fn(&mut cx, args) + } +} + +impl<'sess, S: Stage> AttributeParser<'sess, S> { + pub fn new( + sess: &'sess Session, + features: &'sess Features, + tools: Vec, + stage: S, + ) -> Self { + Self { features: Some(features), tools, parse_only: None, sess, stage } + } + + pub(crate) fn sess(&self) -> &'sess Session { + &self.sess + } + + pub(crate) fn features(&self) -> &'sess Features { + self.features.expect("features not available at this point in the compiler") + } + + pub(crate) fn features_option(&self) -> Option<&'sess Features> { + self.features + } + + pub(crate) fn dcx(&self) -> DiagCtxtHandle<'sess> { + self.sess().dcx() + } + + /// Parse a list of attributes. + /// + /// `target_span` is the span of the thing this list of attributes is applied to, + /// and when `omit_doc` is set, doc attributes are filtered out. + pub fn parse_attribute_list( + &mut self, + attrs: &[ast::Attribute], + target_span: Span, + target_id: S::Id, + target: Target, + omit_doc: OmitDoc, + + lower_span: impl Copy + Fn(Span) -> Span, + mut emit_lint: impl FnMut(AttributeLint), + ) -> Vec { + let mut attributes = Vec::new(); + let mut attr_paths = Vec::new(); + + for attr in attrs { + // If we're only looking for a single attribute, skip all the ones we don't care about. + if let Some(expected) = self.parse_only { + if !attr.has_name(expected) { + continue; + } + } + + // Sometimes, for example for `#![doc = include_str!("readme.md")]`, + // doc still contains a non-literal. You might say, when we're lowering attributes + // that's expanded right? But no, sometimes, when parsing attributes on macros, + // we already use the lowering logic and these are still there. So, when `omit_doc` + // is set we *also* want to ignore these. + if omit_doc == OmitDoc::Skip && attr.has_name(sym::doc) { + continue; + } + + match &attr.kind { + ast::AttrKind::DocComment(comment_kind, symbol) => { + if omit_doc == OmitDoc::Skip { + continue; + } + + attributes.push(Attribute::Parsed(AttributeKind::DocComment { + style: attr.style, + kind: *comment_kind, + span: lower_span(attr.span), + comment: *symbol, + })) + } + // // FIXME: make doc attributes go through a proper attribute parser + // ast::AttrKind::Normal(n) if n.has_name(sym::doc) => { + // let p = GenericMetaItemParser::from_attr(&n, self.dcx()); + // + // attributes.push(Attribute::Parsed(AttributeKind::DocComment { + // style: attr.style, + // kind: CommentKind::Line, + // span: attr.span, + // comment: p.args().name_value(), + // })) + // } + ast::AttrKind::Normal(n) => { + attr_paths.push(PathParser(Cow::Borrowed(&n.item.path))); + + let parts = + n.item.path.segments.iter().map(|seg| seg.ident.name).collect::>(); + + if let Some(accepts) = S::parsers().accepters.get(parts.as_slice()) { + let Some(parser) = MetaItemParser::from_attr( + n, + &parts, + &self.sess.psess, + self.stage.should_emit(), + ) else { + continue; + }; + let path = parser.path(); + let args = parser.args(); + for accept in accepts { + let mut cx: AcceptContext<'_, 'sess, S> = AcceptContext { + shared: SharedContext { + cx: self, + target_span, + target_id, + emit_lint: &mut emit_lint, + }, + attr_span: lower_span(attr.span), + inner_span: lower_span(attr.get_normal_item().span()), + attr_style: attr.style, + template: &accept.template, + attr_path: path.get_attribute_path(), + }; + + (accept.accept_fn)(&mut cx, args); + if !matches!(cx.stage.should_emit(), ShouldEmit::Nothing) { + Self::check_target(&accept.allowed_targets, target, &mut cx); + } + } + } else { + // If we're here, we must be compiling a tool attribute... Or someone + // forgot to parse their fancy new attribute. Let's warn them in any case. + // If you are that person, and you really think your attribute should + // remain unparsed, carefully read the documentation in this module and if + // you still think so you can add an exception to this assertion. + + // FIXME(jdonszelmann): convert other attributes, and check with this that + // we caught em all + // const FIXME_TEMPORARY_ATTR_ALLOWLIST: &[Symbol] = &[sym::cfg]; + // assert!( + // self.tools.contains(&parts[0]) || true, + // // || FIXME_TEMPORARY_ATTR_ALLOWLIST.contains(&parts[0]), + // "attribute {path} wasn't parsed and isn't a know tool attribute", + // ); + + attributes.push(Attribute::Unparsed(Box::new(AttrItem { + path: AttrPath::from_ast(&n.item.path), + args: self.lower_attr_args(&n.item.args, lower_span), + id: HashIgnoredAttrId { attr_id: attr.id }, + style: attr.style, + span: lower_span(attr.span), + }))); + } + } + } + } + + let mut parsed_attributes = Vec::new(); + for f in &S::parsers().finalizers { + if let Some(attr) = f(&mut FinalizeContext { + shared: SharedContext { + cx: self, + target_span, + target_id, + emit_lint: &mut emit_lint, + }, + all_attrs: &attr_paths, + }) { + parsed_attributes.push(Attribute::Parsed(attr)); + } + } + + attributes.extend(parsed_attributes); + + attributes + } + + /// Returns whether there is a parser for an attribute with this name + pub fn is_parsed_attribute(path: &[Symbol]) -> bool { + Late::parsers().accepters.contains_key(path) + } + + fn lower_attr_args(&self, args: &ast::AttrArgs, lower_span: impl Fn(Span) -> Span) -> AttrArgs { + match args { + ast::AttrArgs::Empty => AttrArgs::Empty, + ast::AttrArgs::Delimited(args) => AttrArgs::Delimited(args.clone()), + // This is an inert key-value attribute - it will never be visible to macros + // after it gets lowered to HIR. Therefore, we can extract literals to handle + // nonterminals in `#[doc]` (e.g. `#[doc = $e]`). + ast::AttrArgs::Eq { eq_span, expr } => { + // In valid code the value always ends up as a single literal. Otherwise, a dummy + // literal suffices because the error is handled elsewhere. + let lit = if let ast::ExprKind::Lit(token_lit) = expr.kind + && let Ok(lit) = + ast::MetaItemLit::from_token_lit(token_lit, lower_span(expr.span)) + { + lit + } else { + let guar = self.dcx().span_delayed_bug( + args.span().unwrap_or(DUMMY_SP), + "expr in place where literal is expected (builtin attr parsing)", + ); + ast::MetaItemLit { + symbol: sym::dummy, + suffix: None, + kind: ast::LitKind::Err(guar), + span: DUMMY_SP, + } + }; + AttrArgs::Eq { eq_span: lower_span(*eq_span), expr: lit } + } + } + } +} diff --git a/compiler/rustc_attr_parsing/src/lib.rs b/compiler/rustc_attr_parsing/src/lib.rs index dc54cb6b840c..bcd0d674c75f 100644 --- a/compiler/rustc_attr_parsing/src/lib.rs +++ b/compiler/rustc_attr_parsing/src/lib.rs @@ -1,13 +1,13 @@ //! Centralized logic for parsing and attributes. //! //! ## Architecture -//! This crate is part of a series of crates that handle attribute processing. -//! - [rustc_attr_data_structures](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_data_structures/index.html): Defines the data structures that store parsed attributes +//! This crate is part of a series of crates and modules that handle attribute processing. +//! - [rustc_hir::attrs](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/index.html): Defines the data structures that store parsed attributes //! - [rustc_attr_parsing](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_parsing/index.html): This crate, handles the parsing of attributes -//! - (planned) rustc_attr_validation: Will handle attribute validation +//! - (planned) rustc_attr_validation: Will handle attribute validation, logic currently handled in `rustc_passes` //! //! The separation between data structures and parsing follows the principle of separation of concerns. -//! Data structures (`rustc_attr_data_structures`) define what attributes look like after parsing. +//! Data structures (`rustc_hir::attrs`) define what attributes look like after parsing. //! This crate (`rustc_attr_parsing`) handles how to convert raw tokens into those structures. //! This split allows other parts of the compiler to use the data structures without needing //! the parsing logic, making the codebase more modular and maintainable. @@ -62,7 +62,7 @@ //! a "stability" of an item. So, the stability attribute has an //! [`AttributeParser`](attributes::AttributeParser) that recognizes both the `#[stable()]` //! and `#[unstable()]` syntactic attributes, and at the end produce a single -//! [`AttributeKind::Stability`](rustc_attr_data_structures::AttributeKind::Stability). +//! [`AttributeKind::Stability`](rustc_hir::attrs::AttributeKind::Stability). //! //! When multiple instances of the same attribute are allowed, they're combined into a single //! semantic attribute. For example: @@ -79,23 +79,39 @@ // tidy-alphabetical-start #![allow(internal_features)] #![doc(rust_logo)] +#![feature(decl_macro)] #![feature(rustdoc_internals)] #![recursion_limit = "256"] // tidy-alphabetical-end #[macro_use] +/// All the individual attribute parsers for each of rustc's built-in attributes. mod attributes; -pub(crate) mod context; -mod lints; -pub mod parser; -mod session_diagnostics; -pub use attributes::cfg::{CFG_TEMPLATE, EvalConfigResult, eval_config_entry, parse_cfg_attr}; -pub use attributes::cfg_old::*; -pub use attributes::util::{ - find_crate_name, is_builtin_attr, is_doc_alias_attrs_contain_symbol, parse_version, +/// All the important types given to attribute parsers when parsing +pub(crate) mod context; + +/// Code that other crates interact with, to actually parse a list (or sometimes single) +/// attribute. +mod interface; + +/// Despite this entire module called attribute parsing and the term being a little overloaded, +/// in this module the code lives that actually breaks up tokenstreams into semantic pieces of attributes, +/// like lists or name-value pairs. +pub mod parser; + +mod lints; +mod session_diagnostics; +mod target_checking; +pub mod validate_attr; + +pub use attributes::cfg::{ + CFG_TEMPLATE, EvalConfigResult, eval_config_entry, parse_cfg, parse_cfg_attr, }; -pub use context::{AttributeParser, Early, Late, OmitDoc, ShouldEmit}; +pub use attributes::cfg_old::*; +pub use attributes::util::{is_builtin_attr, is_doc_alias_attrs_contain_symbol, parse_version}; +pub use context::{Early, Late, OmitDoc, ShouldEmit}; +pub use interface::AttributeParser; pub use lints::emit_attribute_lint; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } diff --git a/compiler/rustc_attr_parsing/src/lints.rs b/compiler/rustc_attr_parsing/src/lints.rs index e648ca4fdf80..3a2a37046696 100644 --- a/compiler/rustc_attr_parsing/src/lints.rs +++ b/compiler/rustc_attr_parsing/src/lints.rs @@ -1,10 +1,13 @@ -use rustc_attr_data_structures::lints::{AttributeLint, AttributeLintKind}; +use std::borrow::Cow; + use rustc_errors::{DiagArgValue, LintEmitter}; -use rustc_hir::HirId; +use rustc_hir::Target; +use rustc_hir::lints::{AttributeLint, AttributeLintKind}; +use rustc_span::sym; use crate::session_diagnostics; -pub fn emit_attribute_lint(lint: &AttributeLint, lint_emitter: L) { +pub fn emit_attribute_lint(lint: &AttributeLint, lint_emitter: L) { let AttributeLint { id, span, kind } = lint; match kind { @@ -28,11 +31,72 @@ pub fn emit_attribute_lint(lint: &AttributeLint, lint_emi }, ); } - AttributeLintKind::EmptyAttribute { first_span } => lint_emitter.emit_node_span_lint( - rustc_session::lint::builtin::UNUSED_ATTRIBUTES, - *id, - *first_span, - session_diagnostics::EmptyAttributeList { attr_span: *first_span }, - ), + AttributeLintKind::InvalidMacroExportArguments { suggestions } => lint_emitter + .emit_node_span_lint( + rustc_session::lint::builtin::INVALID_MACRO_EXPORT_ARGUMENTS, + *id, + *span, + session_diagnostics::IllFormedAttributeInput { + num_suggestions: suggestions.len(), + suggestions: DiagArgValue::StrListSepByAnd( + suggestions.into_iter().map(|s| format!("`{s}`").into()).collect(), + ), + }, + ), + AttributeLintKind::EmptyAttribute { first_span, attr_path, valid_without_list } => { + lint_emitter.emit_node_span_lint( + rustc_session::lint::builtin::UNUSED_ATTRIBUTES, + *id, + *first_span, + session_diagnostics::EmptyAttributeList { + attr_span: *first_span, + attr_path: attr_path.clone(), + valid_without_list: *valid_without_list, + }, + ) + } + AttributeLintKind::InvalidTarget { name, target, applied, only } => lint_emitter + .emit_node_span_lint( + // This check is here because `deprecated` had its own lint group and removing this would be a breaking change + if name.segments[0].name == sym::deprecated + && ![ + Target::Closure, + Target::Expression, + Target::Statement, + Target::Arm, + Target::MacroCall, + ] + .contains(target) + { + rustc_session::lint::builtin::USELESS_DEPRECATED + } else { + rustc_session::lint::builtin::UNUSED_ATTRIBUTES + }, + *id, + *span, + session_diagnostics::InvalidTargetLint { + name: name.clone(), + target: target.plural_name(), + applied: DiagArgValue::StrListSepByAnd( + applied.into_iter().map(|i| Cow::Owned(i.to_string())).collect(), + ), + only, + attr_span: *span, + }, + ), + + &AttributeLintKind::InvalidStyle { ref name, is_used_as_inner, target, target_span } => { + lint_emitter.emit_node_span_lint( + rustc_session::lint::builtin::UNUSED_ATTRIBUTES, + *id, + *span, + session_diagnostics::InvalidAttrStyle { + name: name.clone(), + is_used_as_inner, + target_span: (!is_used_as_inner).then_some(target_span), + target, + }, + ) + } } } diff --git a/compiler/rustc_attr_parsing/src/parser.rs b/compiler/rustc_attr_parsing/src/parser.rs index aecaae947c99..7474471f2fe0 100644 --- a/compiler/rustc_attr_parsing/src/parser.rs +++ b/compiler/rustc_attr_parsing/src/parser.rs @@ -3,45 +3,30 @@ //! //! FIXME(jdonszelmann): delete `rustc_ast/attr/mod.rs` +use std::borrow::Cow; use std::fmt::{Debug, Display}; -use std::iter::Peekable; -use rustc_ast::token::{self, Delimiter, Token}; -use rustc_ast::tokenstream::{TokenStreamIter, TokenTree}; -use rustc_ast::{AttrArgs, DelimArgs, Expr, ExprKind, LitKind, MetaItemLit, NormalAttr, Path}; +use rustc_ast::token::{self, Delimiter, MetaVarKind}; +use rustc_ast::tokenstream::TokenStream; +use rustc_ast::{AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, NormalAttr, Path}; use rustc_ast_pretty::pprust; -use rustc_errors::DiagCtxtHandle; +use rustc_errors::{Diag, PResult}; use rustc_hir::{self as hir, AttrPath}; -use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, sym}; +use rustc_parse::exp; +use rustc_parse::parser::{Parser, PathStyle, token_descr}; +use rustc_session::errors::{create_lit_error, report_lit_error}; +use rustc_session::parse::ParseSess; +use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, sym}; +use thin_vec::ThinVec; -pub struct SegmentIterator<'a> { - offset: usize, - path: &'a PathParser<'a>, -} - -impl<'a> Iterator for SegmentIterator<'a> { - type Item = &'a Ident; - - fn next(&mut self) -> Option { - if self.offset >= self.path.len() { - return None; - } - - let res = match self.path { - PathParser::Ast(ast_path) => &ast_path.segments[self.offset].ident, - PathParser::Attr(attr_path) => &attr_path.segments[self.offset], - }; - - self.offset += 1; - Some(res) - } -} +use crate::ShouldEmit; +use crate::session_diagnostics::{ + InvalidMetaItem, InvalidMetaItemQuoteIdentSugg, InvalidMetaItemRemoveNegSugg, MetaBadDelim, + MetaBadDelimSugg, SuffixedLiteralInAttribute, +}; #[derive(Clone, Debug)] -pub enum PathParser<'a> { - Ast(&'a Path), - Attr(AttrPath), -} +pub struct PathParser<'a>(pub Cow<'a, Path>); impl<'a> PathParser<'a> { pub fn get_attribute_path(&self) -> hir::AttrPath { @@ -52,25 +37,19 @@ impl<'a> PathParser<'a> { } pub fn segments(&'a self) -> impl Iterator { - SegmentIterator { offset: 0, path: self } + self.0.segments.iter().map(|seg| &seg.ident) } pub fn span(&self) -> Span { - match self { - PathParser::Ast(path) => path.span, - PathParser::Attr(attr_path) => attr_path.span, - } + self.0.span } pub fn len(&self) -> usize { - match self { - PathParser::Ast(path) => path.segments.len(), - PathParser::Attr(attr_path) => attr_path.segments.len(), - } + self.0.segments.len() } pub fn segments_is(&self, segments: &[Symbol]) -> bool { - self.len() == segments.len() && self.segments().zip(segments).all(|(a, b)| a.name == *b) + self.segments().map(|segment| &segment.name).eq(segments) } pub fn word(&self) -> Option { @@ -99,10 +78,7 @@ impl<'a> PathParser<'a> { impl Display for PathParser<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PathParser::Ast(path) => write!(f, "{}", pprust::path_to_string(path)), - PathParser::Attr(attr_path) => write!(f, "{attr_path}"), - } + write!(f, "{}", pprust::path_to_string(&self.0)) } } @@ -123,21 +99,43 @@ impl<'a> ArgParser<'a> { } } - pub fn from_attr_args<'sess>(value: &'a AttrArgs, dcx: DiagCtxtHandle<'sess>) -> Self { - match value { + pub fn from_attr_args<'sess>( + value: &'a AttrArgs, + parts: &[Symbol], + psess: &'sess ParseSess, + should_emit: ShouldEmit, + ) -> Option { + Some(match value { AttrArgs::Empty => Self::NoArgs, - AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => { - Self::List(MetaItemListParser::new(args, dcx)) - } AttrArgs::Delimited(args) => { - Self::List(MetaItemListParser { sub_parsers: vec![], span: args.dspan.entire() }) + // The arguments of rustc_dummy are not validated if the arguments are delimited + if parts == &[sym::rustc_dummy] { + return Some(ArgParser::List(MetaItemListParser { + sub_parsers: ThinVec::new(), + span: args.dspan.entire(), + })); + } + + if args.delim != Delimiter::Parenthesis { + psess.dcx().emit_err(MetaBadDelim { + span: args.dspan.entire(), + sugg: MetaBadDelimSugg { open: args.dspan.open, close: args.dspan.close }, + }); + return None; + } + + Self::List( + MetaItemListParser::new(&args.tokens, args.dspan.entire(), psess, should_emit) + .map_err(|e| should_emit.emit_err(e)) + .ok()?, + ) } AttrArgs::Eq { eq_span, expr } => Self::NameValue(NameValueParser { eq_span: *eq_span, - value: expr_to_lit(dcx, &expr, *eq_span), + value: expr_to_lit(psess, &expr, expr.span, should_emit)?, value_span: expr.span, }), - } + }) } /// Asserts that this MetaItem is a list @@ -192,7 +190,15 @@ pub enum MetaItemOrLitParser<'a> { Err(Span, ErrorGuaranteed), } -impl<'a> MetaItemOrLitParser<'a> { +impl<'sess> MetaItemOrLitParser<'sess> { + pub fn parse_single( + parser: &mut Parser<'sess>, + should_emit: ShouldEmit, + ) -> PResult<'sess, MetaItemOrLitParser<'static>> { + let mut this = MetaItemListParserContext { parser, should_emit }; + this.parse_meta_item_inner() + } + pub fn span(&self) -> Span { match self { MetaItemOrLitParser::MetaItemParser(generic_meta_item_parser) => { @@ -210,7 +216,7 @@ impl<'a> MetaItemOrLitParser<'a> { } } - pub fn meta_item(&self) -> Option<&MetaItemParser<'a>> { + pub fn meta_item(&self) -> Option<&MetaItemParser<'sess>> { match self { MetaItemOrLitParser::MetaItemParser(parser) => Some(parser), _ => None, @@ -249,11 +255,16 @@ impl<'a> Debug for MetaItemParser<'a> { impl<'a> MetaItemParser<'a> { /// Create a new parser from a [`NormalAttr`], which is stored inside of any /// [`ast::Attribute`](rustc_ast::Attribute) - pub fn from_attr<'sess>(attr: &'a NormalAttr, dcx: DiagCtxtHandle<'sess>) -> Self { - Self { - path: PathParser::Ast(&attr.item.path), - args: ArgParser::from_attr_args(&attr.item.args, dcx), - } + pub fn from_attr<'sess>( + attr: &'a NormalAttr, + parts: &[Symbol], + psess: &'sess ParseSess, + should_emit: ShouldEmit, + ) -> Option { + Some(Self { + path: PathParser(Cow::Borrowed(&attr.item.path)), + args: ArgParser::from_attr_args(&attr.item.args, parts, psess, should_emit)?, + }) } } @@ -318,215 +329,238 @@ impl NameValueParser { } } -fn expr_to_lit(dcx: DiagCtxtHandle<'_>, expr: &Expr, span: Span) -> MetaItemLit { - // In valid code the value always ends up as a single literal. Otherwise, a dummy - // literal suffices because the error is handled elsewhere. - if let ExprKind::Lit(token_lit) = expr.kind - && let Ok(lit) = MetaItemLit::from_token_lit(token_lit, expr.span) - { - lit +fn expr_to_lit( + psess: &ParseSess, + expr: &Expr, + span: Span, + should_emit: ShouldEmit, +) -> Option { + if let ExprKind::Lit(token_lit) = expr.kind { + let res = MetaItemLit::from_token_lit(token_lit, expr.span); + match res { + Ok(lit) => { + if token_lit.suffix.is_some() { + should_emit.emit_err( + psess.dcx().create_err(SuffixedLiteralInAttribute { span: lit.span }), + ); + None + } else { + if !lit.kind.is_unsuffixed() { + // Emit error and continue, we can still parse the attribute as if the suffix isn't there + should_emit.emit_err( + psess.dcx().create_err(SuffixedLiteralInAttribute { span: lit.span }), + ); + } + + Some(lit) + } + } + Err(err) => { + let guar = report_lit_error(psess, err, token_lit, expr.span); + let lit = MetaItemLit { + symbol: token_lit.symbol, + suffix: token_lit.suffix, + kind: LitKind::Err(guar), + span: expr.span, + }; + Some(lit) + } + } } else { - let guar = dcx.span_delayed_bug( - span, - "expr in place where literal is expected (builtin attr parsing)", - ); - MetaItemLit { symbol: sym::dummy, suffix: None, kind: LitKind::Err(guar), span } + if matches!(should_emit, ShouldEmit::Nothing) { + return None; + } + + // Example cases: + // - `#[foo = 1+1]`: results in `ast::ExprKind::BinOp`. + // - `#[foo = include_str!("nonexistent-file.rs")]`: + // results in `ast::ExprKind::Err`. In that case we delay + // the error because an earlier error will have already + // been reported. + let msg = "attribute value must be a literal"; + let err = psess.dcx().struct_span_err(span, msg); + should_emit.emit_err(err); + None } } struct MetaItemListParserContext<'a, 'sess> { - // the tokens inside the delimiters, so `#[some::attr(a b c)]` would have `a b c` inside - inside_delimiters: Peekable>, - dcx: DiagCtxtHandle<'sess>, + parser: &'a mut Parser<'sess>, + should_emit: ShouldEmit, } impl<'a, 'sess> MetaItemListParserContext<'a, 'sess> { - fn done(&mut self) -> bool { - self.inside_delimiters.peek().is_none() + fn parse_unsuffixed_meta_item_lit(&mut self) -> PResult<'sess, MetaItemLit> { + let Some(token_lit) = self.parser.eat_token_lit() else { return Err(self.expected_lit()) }; + self.unsuffixed_meta_item_from_lit(token_lit) } - fn next_path(&mut self) -> Option { - // FIXME: Share code with `parse_path`. - let tt = self.inside_delimiters.next().map(|tt| TokenTree::uninterpolate(tt)); - - match tt.as_deref()? { - &TokenTree::Token( - Token { kind: ref kind @ (token::Ident(..) | token::PathSep), span }, - _, - ) => { - // here we have either an ident or pathsep `::`. - - let mut segments = if let &token::Ident(name, _) = kind { - // when we lookahead another pathsep, more path's coming - if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) = - self.inside_delimiters.peek() - { - self.inside_delimiters.next(); - vec![Ident::new(name, span)] - } else { - // else we have a single identifier path, that's all - return Some(AttrPath { - segments: vec![Ident::new(name, span)].into_boxed_slice(), - span, - }); - } - } else { - // if `::` is all we get, we just got a path root - vec![Ident::new(kw::PathRoot, span)] - }; - - // one segment accepted. accept n more - loop { - // another ident? - if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) = - self.inside_delimiters - .next() - .map(|tt| TokenTree::uninterpolate(tt)) - .as_deref() - { - segments.push(Ident::new(name, span)); - } else { - return None; - } - // stop unless we see another `::` - if let Some(TokenTree::Token(Token { kind: token::PathSep, .. }, _)) = - self.inside_delimiters.peek() - { - self.inside_delimiters.next(); - } else { - break; - } - } - let span = span.with_hi(segments.last().unwrap().span.hi()); - Some(AttrPath { segments: segments.into_boxed_slice(), span }) + fn unsuffixed_meta_item_from_lit( + &mut self, + token_lit: token::Lit, + ) -> PResult<'sess, MetaItemLit> { + let lit = match MetaItemLit::from_token_lit(token_lit, self.parser.prev_token.span) { + Ok(lit) => lit, + Err(err) => { + return Err(create_lit_error( + &self.parser.psess, + err, + token_lit, + self.parser.prev_token_uninterpolated_span(), + )); } - TokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => None, - _ => { - // malformed attributes can get here. We can't crash, but somewhere else should've - // already warned for this. - None - } - } - } + }; - fn value(&mut self) -> Option { - match self.inside_delimiters.next() { - Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) => { - MetaItemListParserContext { - inside_delimiters: inner_tokens.iter().peekable(), - dcx: self.dcx, - } - .value() - } - Some(TokenTree::Token(token, _)) => MetaItemLit::from_token(token), - _ => None, - } - } - - /// parses one element on the inside of a list attribute like `#[my_attr( )]` - /// - /// parses a path followed be either: - /// 1. nothing (a word attr) - /// 2. a parenthesized list - /// 3. an equals sign and a literal (name-value) - /// - /// Can also parse *just* a literal. This is for cases like as `#[my_attr("literal")]` - /// where no path is given before the literal - /// - /// Some exceptions too for interpolated attributes which are already pre-processed - fn next(&mut self) -> Option> { - // a list element is either a literal - if let Some(TokenTree::Token(token, _)) = self.inside_delimiters.peek() - && let Some(lit) = MetaItemLit::from_token(token) - { - self.inside_delimiters.next(); - return Some(MetaItemOrLitParser::Lit(lit)); - } else if let Some(TokenTree::Delimited(.., Delimiter::Invisible(_), inner_tokens)) = - self.inside_delimiters.peek() - { - self.inside_delimiters.next(); - return MetaItemListParserContext { - inside_delimiters: inner_tokens.iter().peekable(), - dcx: self.dcx, - } - .next(); + if !lit.kind.is_unsuffixed() { + // Emit error and continue, we can still parse the attribute as if the suffix isn't there + self.should_emit.emit_err( + self.parser.dcx().create_err(SuffixedLiteralInAttribute { span: lit.span }), + ); } - // or a path. - let path = self.next_path()?; - - // Paths can be followed by: - // - `(more meta items)` (another list) - // - `= lit` (a name-value) - // - nothing - Some(MetaItemOrLitParser::MetaItemParser(match self.inside_delimiters.peek() { - Some(TokenTree::Delimited(dspan, _, Delimiter::Parenthesis, inner_tokens)) => { - self.inside_delimiters.next(); - - MetaItemParser { - path: PathParser::Attr(path), - args: ArgParser::List(MetaItemListParser::new_tts( - inner_tokens.iter(), - dspan.entire(), - self.dcx, - )), - } - } - Some(TokenTree::Delimited(_, ..)) => { - self.inside_delimiters.next(); - // self.dcx.span_delayed_bug(span.entire(), "wrong delimiters"); - return None; - } - Some(TokenTree::Token(Token { kind: token::Eq, span }, _)) => { - self.inside_delimiters.next(); - let value = self.value()?; - MetaItemParser { - path: PathParser::Attr(path), - args: ArgParser::NameValue(NameValueParser { - eq_span: *span, - value_span: value.span, - value, - }), - } - } - _ => MetaItemParser { path: PathParser::Attr(path), args: ArgParser::NoArgs }, - })) + Ok(lit) } - fn parse(mut self, span: Span) -> MetaItemListParser<'a> { - let mut sub_parsers = Vec::new(); - - while !self.done() { - let Some(n) = self.next() else { - continue; + fn parse_attr_item(&mut self) -> PResult<'sess, MetaItemParser<'static>> { + if let Some(MetaVarKind::Meta { has_meta_form }) = self.parser.token.is_metavar_seq() { + return if has_meta_form { + let attr_item = self + .parser + .eat_metavar_seq(MetaVarKind::Meta { has_meta_form: true }, |this| { + MetaItemListParserContext { parser: this, should_emit: self.should_emit } + .parse_attr_item() + }) + .unwrap(); + Ok(attr_item) + } else { + self.parser.unexpected_any() }; - sub_parsers.push(n); + } - match self.inside_delimiters.peek() { - None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => { - self.inside_delimiters.next(); + let path = self.parser.parse_path(PathStyle::Mod)?; + + // Check style of arguments that this meta item has + let args = if self.parser.check(exp!(OpenParen)) { + let start = self.parser.token.span; + let (sub_parsers, _) = self.parser.parse_paren_comma_seq(|parser| { + MetaItemListParserContext { parser, should_emit: self.should_emit } + .parse_meta_item_inner() + })?; + let end = self.parser.prev_token.span; + ArgParser::List(MetaItemListParser { sub_parsers, span: start.with_hi(end.hi()) }) + } else if self.parser.eat(exp!(Eq)) { + let eq_span = self.parser.prev_token.span; + let value = self.parse_unsuffixed_meta_item_lit()?; + + ArgParser::NameValue(NameValueParser { eq_span, value, value_span: value.span }) + } else { + ArgParser::NoArgs + }; + + Ok(MetaItemParser { path: PathParser(Cow::Owned(path)), args }) + } + + fn parse_meta_item_inner(&mut self) -> PResult<'sess, MetaItemOrLitParser<'static>> { + if let Some(token_lit) = self.parser.eat_token_lit() { + // If a literal token is parsed, we commit to parsing a MetaItemLit for better errors + Ok(MetaItemOrLitParser::Lit(self.unsuffixed_meta_item_from_lit(token_lit)?)) + } else { + let prev_pros = self.parser.approx_token_stream_pos(); + match self.parse_attr_item() { + Ok(item) => Ok(MetaItemOrLitParser::MetaItemParser(item)), + Err(err) => { + // If `parse_attr_item` made any progress, it likely has a more precise error we should prefer + // If it didn't make progress we use the `expected_lit` from below + if self.parser.approx_token_stream_pos() != prev_pros { + Err(err) + } else { + err.cancel(); + Err(self.expected_lit()) + } } - Some(_) => {} + } + } + } + + fn expected_lit(&mut self) -> Diag<'sess> { + let mut err = InvalidMetaItem { + span: self.parser.token.span, + descr: token_descr(&self.parser.token), + quote_ident_sugg: None, + remove_neg_sugg: None, + }; + + // Suggest quoting idents, e.g. in `#[cfg(key = value)]`. We don't use `Token::ident` and + // don't `uninterpolate` the token to avoid suggesting anything butchered or questionable + // when macro metavariables are involved. + if self.parser.prev_token == token::Eq + && let token::Ident(..) = self.parser.token.kind + { + let before = self.parser.token.span.shrink_to_lo(); + while let token::Ident(..) = self.parser.token.kind { + self.parser.bump(); + } + err.quote_ident_sugg = Some(InvalidMetaItemQuoteIdentSugg { + before, + after: self.parser.prev_token.span.shrink_to_hi(), + }); + } + + if self.parser.token == token::Minus + && self + .parser + .look_ahead(1, |t| matches!(t.kind, rustc_ast::token::TokenKind::Literal { .. })) + { + err.remove_neg_sugg = + Some(InvalidMetaItemRemoveNegSugg { negative_sign: self.parser.token.span }); + self.parser.bump(); + self.parser.bump(); + } + + self.parser.dcx().create_err(err) + } + + fn parse( + tokens: TokenStream, + psess: &'sess ParseSess, + span: Span, + should_emit: ShouldEmit, + ) -> PResult<'sess, MetaItemListParser<'static>> { + let mut parser = Parser::new(psess, tokens, None); + let mut this = MetaItemListParserContext { parser: &mut parser, should_emit }; + + // Presumably, the majority of the time there will only be one attr. + let mut sub_parsers = ThinVec::with_capacity(1); + while this.parser.token != token::Eof { + sub_parsers.push(this.parse_meta_item_inner()?); + + if !this.parser.eat(exp!(Comma)) { + break; } } - MetaItemListParser { sub_parsers, span } + if parser.token != token::Eof { + parser.unexpected()?; + } + + Ok(MetaItemListParser { sub_parsers, span }) } } #[derive(Debug, Clone)] pub struct MetaItemListParser<'a> { - sub_parsers: Vec>, + sub_parsers: ThinVec>, pub span: Span, } impl<'a> MetaItemListParser<'a> { - fn new<'sess>(delim: &'a DelimArgs, dcx: DiagCtxtHandle<'sess>) -> Self { - MetaItemListParser::new_tts(delim.tokens.iter(), delim.dspan.entire(), dcx) - } - - fn new_tts<'sess>(tts: TokenStreamIter<'a>, span: Span, dcx: DiagCtxtHandle<'sess>) -> Self { - MetaItemListParserContext { inside_delimiters: tts.peekable(), dcx }.parse(span) + pub(crate) fn new<'sess>( + tokens: &'a TokenStream, + span: Span, + psess: &'sess ParseSess, + should_emit: ShouldEmit, + ) -> Result> { + MetaItemListParserContext::parse(tokens.clone(), psess, span, should_emit) } /// Lets you pick and choose as what you want to parse each element in the list diff --git a/compiler/rustc_attr_parsing/src/session_diagnostics.rs b/compiler/rustc_attr_parsing/src/session_diagnostics.rs index 1de25ca252b8..82bd29218313 100644 --- a/compiler/rustc_attr_parsing/src/session_diagnostics.rs +++ b/compiler/rustc_attr_parsing/src/session_diagnostics.rs @@ -1,12 +1,12 @@ use std::num::IntErrorKind; -use rustc_ast as ast; +use rustc_ast::{self as ast, Path}; use rustc_errors::codes::*; use rustc_errors::{ Applicability, Diag, DiagArgValue, DiagCtxtHandle, Diagnostic, EmissionGuarantee, Level, }; use rustc_feature::AttributeTemplate; -use rustc_hir::AttrPath; +use rustc_hir::{AttrPath, Target}; use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_span::{Span, Symbol}; @@ -459,6 +459,34 @@ pub(crate) struct NullOnLinkSection { pub span: Span, } +#[derive(Diagnostic)] +#[diag(attr_parsing_null_on_objc_class)] +pub(crate) struct NullOnObjcClass { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_null_on_objc_selector)] +pub(crate) struct NullOnObjcSelector { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_objc_class_expected_string_literal)] +pub(crate) struct ObjcClassExpectedStringLiteral { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_objc_selector_expected_string_literal)] +pub(crate) struct ObjcSelectorExpectedStringLiteral { + #[primary_span] + pub span: Span, +} + #[derive(Diagnostic)] #[diag(attr_parsing_stability_outside_std, code = E0734)] pub(crate) struct StabilityOutsideStd { @@ -475,9 +503,38 @@ pub(crate) struct EmptyConfusables { #[derive(LintDiagnostic)] #[diag(attr_parsing_empty_attribute)] +#[note] pub(crate) struct EmptyAttributeList { #[suggestion(code = "", applicability = "machine-applicable")] pub attr_span: Span, + pub attr_path: AttrPath, + pub valid_without_list: bool, +} + +#[derive(LintDiagnostic)] +#[diag(attr_parsing_invalid_target_lint)] +#[warning] +#[help] +pub(crate) struct InvalidTargetLint { + pub name: AttrPath, + pub target: &'static str, + pub applied: DiagArgValue, + pub only: &'static str, + #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")] + pub attr_span: Span, +} + +#[derive(Diagnostic)] +#[help] +#[diag(attr_parsing_invalid_target)] +pub(crate) struct InvalidTarget { + #[primary_span] + #[suggestion(code = "", applicability = "machine-applicable", style = "tool-only")] + pub span: Span, + pub name: AttrPath, + pub target: &'static str, + pub applied: DiagArgValue, + pub only: &'static str, } #[derive(Diagnostic)] @@ -498,6 +555,7 @@ pub(crate) struct ReprIdent { #[derive(Diagnostic)] #[diag(attr_parsing_unrecognized_repr_hint, code = E0552)] #[help] +#[note] pub(crate) struct UnrecognizedReprHint { #[primary_span] pub span: Span, @@ -531,7 +589,7 @@ pub(crate) struct LinkOrdinalOutOfRange { pub ordinal: u128, } -pub(crate) enum AttributeParseErrorReason { +pub(crate) enum AttributeParseErrorReason<'a> { ExpectedNoArgs, ExpectedStringLiteral { byte_string: Option, @@ -544,7 +602,7 @@ pub(crate) enum AttributeParseErrorReason { ExpectedNameValue(Option), DuplicateKey(Symbol), ExpectedSpecificArgument { - possibilities: Vec<&'static str>, + possibilities: &'a [Symbol], strings: bool, /// Should we tell the user to write a list when they didn't? list: bool, @@ -552,15 +610,16 @@ pub(crate) enum AttributeParseErrorReason { ExpectedIdentifier, } -pub(crate) struct AttributeParseError { +pub(crate) struct AttributeParseError<'a> { pub(crate) span: Span, pub(crate) attr_span: Span, pub(crate) template: AttributeTemplate, pub(crate) attribute: AttrPath, - pub(crate) reason: AttributeParseErrorReason, + pub(crate) reason: AttributeParseErrorReason<'a>, + pub(crate) suggestions: Vec, } -impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError { +impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError<'_> { fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> { let name = self.attribute.to_string(); @@ -629,7 +688,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError { list: false, } => { let quote = if strings { '"' } else { '`' }; - match possibilities.as_slice() { + match possibilities { &[] => {} &[x] => { diag.span_label( @@ -659,7 +718,7 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError { list: true, } => { let quote = if strings { '"' } else { '`' }; - match possibilities.as_slice() { + match possibilities { &[] => {} &[x] => { diag.span_label( @@ -690,18 +749,233 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError { } } - let suggestions = self.template.suggestions(false, &name); + if let Some(link) = self.template.docs { + diag.note(format!("for more information, visit <{link}>")); + } + diag.span_suggestions( self.attr_span, - if suggestions.len() == 1 { + if self.suggestions.len() == 1 { "must be of the form" } else { "try changing it to one of the following valid forms of the attribute" }, - suggestions, + self.suggestions, Applicability::HasPlaceholders, ); diag } } + +#[derive(Diagnostic)] +#[diag(attr_parsing_invalid_attr_unsafe)] +#[note] +pub(crate) struct InvalidAttrUnsafe { + #[primary_span] + #[label] + pub span: Span, + pub name: Path, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_unsafe_attr_outside_unsafe)] +pub(crate) struct UnsafeAttrOutsideUnsafe { + #[primary_span] + #[label] + pub span: Span, + #[subdiagnostic] + pub suggestion: UnsafeAttrOutsideUnsafeSuggestion, +} + +#[derive(Subdiagnostic)] +#[multipart_suggestion( + attr_parsing_unsafe_attr_outside_unsafe_suggestion, + applicability = "machine-applicable" +)] +pub(crate) struct UnsafeAttrOutsideUnsafeSuggestion { + #[suggestion_part(code = "unsafe(")] + pub left: Span, + #[suggestion_part(code = ")")] + pub right: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_meta_bad_delim)] +pub(crate) struct MetaBadDelim { + #[primary_span] + pub span: Span, + #[subdiagnostic] + pub sugg: MetaBadDelimSugg, +} + +#[derive(Subdiagnostic)] +#[multipart_suggestion( + attr_parsing_meta_bad_delim_suggestion, + applicability = "machine-applicable" +)] +pub(crate) struct MetaBadDelimSugg { + #[suggestion_part(code = "(")] + pub open: Span, + #[suggestion_part(code = ")")] + pub close: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_invalid_meta_item)] +pub(crate) struct InvalidMetaItem { + #[primary_span] + pub span: Span, + pub descr: String, + #[subdiagnostic] + pub quote_ident_sugg: Option, + #[subdiagnostic] + pub remove_neg_sugg: Option, +} + +#[derive(Subdiagnostic)] +#[multipart_suggestion(attr_parsing_quote_ident_sugg, applicability = "machine-applicable")] +pub(crate) struct InvalidMetaItemQuoteIdentSugg { + #[suggestion_part(code = "\"")] + pub before: Span, + #[suggestion_part(code = "\"")] + pub after: Span, +} + +#[derive(Subdiagnostic)] +#[multipart_suggestion(attr_parsing_remove_neg_sugg, applicability = "machine-applicable")] +pub(crate) struct InvalidMetaItemRemoveNegSugg { + #[suggestion_part(code = "")] + pub negative_sign: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_suffixed_literal_in_attribute)] +#[help] +pub(crate) struct SuffixedLiteralInAttribute { + #[primary_span] + pub span: Span, +} + +#[derive(LintDiagnostic)] +#[diag(attr_parsing_invalid_style)] +pub(crate) struct InvalidAttrStyle { + pub name: AttrPath, + pub is_used_as_inner: bool, + #[note] + pub target_span: Option, + pub target: Target, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_empty_link_name, code = E0454)] +pub(crate) struct EmptyLinkName { + #[primary_span] + #[label] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_link_framework_apple, code = E0455)] +pub(crate) struct LinkFrameworkApple { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_incompatible_wasm_link)] +pub(crate) struct IncompatibleWasmLink { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_link_requires_name, code = E0459)] +pub(crate) struct LinkRequiresName { + #[primary_span] + #[label] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_raw_dylib_no_nul)] +pub(crate) struct RawDylibNoNul { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_raw_dylib_only_windows, code = E0455)] +pub(crate) struct RawDylibOnlyWindows { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_invalid_link_modifier)] +pub(crate) struct InvalidLinkModifier { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_multiple_modifiers)] +pub(crate) struct MultipleModifiers { + #[primary_span] + pub span: Span, + pub modifier: Symbol, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_import_name_type_x86)] +pub(crate) struct ImportNameTypeX86 { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_bundle_needs_static)] +pub(crate) struct BundleNeedsStatic { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_whole_archive_needs_static)] +pub(crate) struct WholeArchiveNeedsStatic { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_as_needed_compatibility)] +pub(crate) struct AsNeededCompatibility { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_import_name_type_raw)] +pub(crate) struct ImportNameTypeRaw { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_limit_invalid)] +pub(crate) struct LimitInvalid<'a> { + #[primary_span] + pub span: Span, + #[label] + pub value_span: Span, + pub error_str: &'a str, +} + +#[derive(Diagnostic)] +#[diag(attr_parsing_cfg_attr_bad_delim)] +pub(crate) struct CfgAttrBadDelim { + #[primary_span] + pub span: Span, + #[subdiagnostic] + pub sugg: MetaBadDelimSugg, +} diff --git a/compiler/rustc_attr_parsing/src/target_checking.rs b/compiler/rustc_attr_parsing/src/target_checking.rs new file mode 100644 index 000000000000..fabd364d3d7f --- /dev/null +++ b/compiler/rustc_attr_parsing/src/target_checking.rs @@ -0,0 +1,295 @@ +use std::borrow::Cow; + +use rustc_ast::AttrStyle; +use rustc_errors::DiagArgValue; +use rustc_feature::Features; +use rustc_hir::lints::AttributeLintKind; +use rustc_hir::{MethodKind, Target}; + +use crate::AttributeParser; +use crate::context::{AcceptContext, Stage}; +use crate::session_diagnostics::InvalidTarget; + +#[derive(Debug)] +pub(crate) enum AllowedTargets { + AllowList(&'static [Policy]), + AllowListWarnRest(&'static [Policy]), + /// Special, and not the same as `AllowList(&[Allow(Target::Crate)])`. + /// For crate-level attributes we emit a specific set of lints to warn + /// people about accidentally not using them on the crate. + /// Only use this for attributes that are *exclusively* valid at the crate level. + CrateLevel, +} + +pub(crate) enum AllowedResult { + Allowed, + Warn, + Error, +} + +impl AllowedTargets { + pub(crate) fn is_allowed(&self, target: Target) -> AllowedResult { + match self { + AllowedTargets::AllowList(list) => { + if list.contains(&Policy::Allow(target)) + || list.contains(&Policy::AllowSilent(target)) + { + AllowedResult::Allowed + } else if list.contains(&Policy::Warn(target)) { + AllowedResult::Warn + } else { + AllowedResult::Error + } + } + AllowedTargets::AllowListWarnRest(list) => { + if list.contains(&Policy::Allow(target)) + || list.contains(&Policy::AllowSilent(target)) + { + AllowedResult::Allowed + } else if list.contains(&Policy::Error(target)) { + AllowedResult::Error + } else { + AllowedResult::Warn + } + } + AllowedTargets::CrateLevel => AllowedResult::Allowed, + } + } + + pub(crate) fn allowed_targets(&self) -> Vec { + match self { + AllowedTargets::AllowList(list) => list, + AllowedTargets::AllowListWarnRest(list) => list, + AllowedTargets::CrateLevel => ALL_TARGETS, + } + .iter() + .filter_map(|target| match target { + Policy::Allow(target) => Some(*target), + Policy::AllowSilent(_) => None, // Not listed in possible targets + Policy::Warn(_) => None, + Policy::Error(_) => None, + }) + .collect() + } +} + +/// This policy determines what diagnostics should be emitted based on the `Target` of the attribute. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum Policy { + /// A target that is allowed. + Allow(Target), + /// A target that is allowed and not listed in the possible targets. + /// This is useful if the target is checked elsewhere. + AllowSilent(Target), + /// Emits a FCW on this target. + /// This is useful if the target was previously allowed but should not be. + Warn(Target), + /// Emits an error on this target. + Error(Target), +} + +impl<'sess, S: Stage> AttributeParser<'sess, S> { + pub(crate) fn check_target( + allowed_targets: &AllowedTargets, + target: Target, + cx: &mut AcceptContext<'_, 'sess, S>, + ) { + Self::check_type(matches!(allowed_targets, AllowedTargets::CrateLevel), target, cx); + + match allowed_targets.is_allowed(target) { + AllowedResult::Allowed => {} + AllowedResult::Warn => { + let allowed_targets = allowed_targets.allowed_targets(); + let (applied, only) = allowed_targets_applied(allowed_targets, target, cx.features); + let name = cx.attr_path.clone(); + let attr_span = cx.attr_span; + cx.emit_lint( + AttributeLintKind::InvalidTarget { + name, + target, + only: if only { "only " } else { "" }, + applied, + }, + attr_span, + ); + } + AllowedResult::Error => { + let allowed_targets = allowed_targets.allowed_targets(); + let (applied, only) = allowed_targets_applied(allowed_targets, target, cx.features); + let name = cx.attr_path.clone(); + cx.dcx().emit_err(InvalidTarget { + span: cx.attr_span.clone(), + name, + target: target.plural_name(), + only: if only { "only " } else { "" }, + applied: DiagArgValue::StrListSepByAnd( + applied.into_iter().map(Cow::Owned).collect(), + ), + }); + } + } + } + + pub(crate) fn check_type( + crate_level: bool, + target: Target, + cx: &mut AcceptContext<'_, 'sess, S>, + ) { + let is_crate_root = S::id_is_crate_root(cx.target_id); + + if is_crate_root { + return; + } + + if !crate_level { + return; + } + + let lint = AttributeLintKind::InvalidStyle { + name: cx.attr_path.clone(), + is_used_as_inner: cx.attr_style == AttrStyle::Inner, + target, + target_span: cx.target_span, + }; + let attr_span = cx.attr_span; + + cx.emit_lint(lint, attr_span); + } +} + +/// Takes a list of `allowed_targets` for an attribute, and the `target` the attribute was applied to. +/// Does some heuristic-based filtering to remove uninteresting targets, and formats the targets into a string +pub(crate) fn allowed_targets_applied( + mut allowed_targets: Vec, + target: Target, + features: Option<&Features>, +) -> (Vec, bool) { + // Remove unstable targets from `allowed_targets` if their features are not enabled + if let Some(features) = features { + if !features.fn_delegation() { + allowed_targets.retain(|t| !matches!(t, Target::Delegation { .. })); + } + if !features.stmt_expr_attributes() { + allowed_targets.retain(|t| !matches!(t, Target::Expression | Target::Statement)); + } + if !features.extern_types() { + allowed_targets.retain(|t| !matches!(t, Target::ForeignTy)); + } + } + + // We define groups of "similar" targets. + // If at least two of the targets are allowed, and the `target` is not in the group, + // we collapse the entire group to a single entry to simplify the target list + const FUNCTION_LIKE: &[Target] = &[ + Target::Fn, + Target::Closure, + Target::ForeignFn, + Target::Method(MethodKind::Inherent), + Target::Method(MethodKind::Trait { body: false }), + Target::Method(MethodKind::Trait { body: true }), + Target::Method(MethodKind::TraitImpl), + ]; + const METHOD_LIKE: &[Target] = &[ + Target::Method(MethodKind::Inherent), + Target::Method(MethodKind::Trait { body: false }), + Target::Method(MethodKind::Trait { body: true }), + Target::Method(MethodKind::TraitImpl), + ]; + const IMPL_LIKE: &[Target] = + &[Target::Impl { of_trait: false }, Target::Impl { of_trait: true }]; + const ADT_LIKE: &[Target] = &[Target::Struct, Target::Enum]; + + let mut added_fake_targets = Vec::new(); + filter_targets( + &mut allowed_targets, + FUNCTION_LIKE, + "functions", + target, + &mut added_fake_targets, + ); + filter_targets(&mut allowed_targets, METHOD_LIKE, "methods", target, &mut added_fake_targets); + filter_targets(&mut allowed_targets, IMPL_LIKE, "impl blocks", target, &mut added_fake_targets); + filter_targets(&mut allowed_targets, ADT_LIKE, "data types", target, &mut added_fake_targets); + + let mut target_strings: Vec<_> = added_fake_targets + .iter() + .copied() + .chain(allowed_targets.iter().map(|t| t.plural_name())) + .map(|i| i.to_string()) + .collect(); + + // ensure a consistent order + target_strings.sort(); + + // If there is now only 1 target left, show that as the only possible target + let only_target = target_strings.len() == 1; + + (target_strings, only_target) +} + +fn filter_targets( + allowed_targets: &mut Vec, + target_group: &'static [Target], + target_group_name: &'static str, + target: Target, + added_fake_targets: &mut Vec<&'static str>, +) { + if target_group.contains(&target) { + return; + } + if allowed_targets.iter().filter(|at| target_group.contains(at)).count() < 2 { + return; + } + allowed_targets.retain(|t| !target_group.contains(t)); + added_fake_targets.push(target_group_name); +} + +/// This is the list of all targets to which a attribute can be applied +/// This is used for: +/// - `rustc_dummy`, which can be applied to all targets +/// - Attributes that are not parted to the new target system yet can use this list as a placeholder +pub(crate) const ALL_TARGETS: &'static [Policy] = { + use Policy::Allow; + &[ + Allow(Target::ExternCrate), + Allow(Target::Use), + Allow(Target::Static), + Allow(Target::Const), + Allow(Target::Fn), + Allow(Target::Closure), + Allow(Target::Mod), + Allow(Target::ForeignMod), + Allow(Target::GlobalAsm), + Allow(Target::TyAlias), + Allow(Target::Enum), + Allow(Target::Variant), + Allow(Target::Struct), + Allow(Target::Field), + Allow(Target::Union), + Allow(Target::Trait), + Allow(Target::TraitAlias), + Allow(Target::Impl { of_trait: false }), + Allow(Target::Impl { of_trait: true }), + Allow(Target::Expression), + Allow(Target::Statement), + Allow(Target::Arm), + Allow(Target::AssocConst), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + Allow(Target::AssocTy), + Allow(Target::ForeignFn), + Allow(Target::ForeignStatic), + Allow(Target::ForeignTy), + Allow(Target::MacroDef), + Allow(Target::Param), + Allow(Target::PatField), + Allow(Target::ExprField), + Allow(Target::WherePredicate), + Allow(Target::MacroCall), + Allow(Target::Crate), + Allow(Target::Delegation { mac: false }), + Allow(Target::Delegation { mac: true }), + ] +}; diff --git a/compiler/rustc_parse/src/validate_attr.rs b/compiler/rustc_attr_parsing/src/validate_attr.rs similarity index 87% rename from compiler/rustc_parse/src/validate_attr.rs rename to compiler/rustc_attr_parsing/src/validate_attr.rs index bc4c605afadb..927417f89f8c 100644 --- a/compiler/rustc_parse/src/validate_attr.rs +++ b/compiler/rustc_attr_parsing/src/validate_attr.rs @@ -8,16 +8,16 @@ use rustc_ast::{ self as ast, AttrArgs, Attribute, DelimArgs, MetaItem, MetaItemInner, MetaItemKind, NodeId, Path, Safety, }; -use rustc_attr_parsing::{AttributeParser, Late}; use rustc_errors::{Applicability, DiagCtxtHandle, FatalError, PResult}; use rustc_feature::{AttributeSafety, AttributeTemplate, BUILTIN_ATTRIBUTE_MAP, BuiltinAttribute}; +use rustc_parse::parse_in; use rustc_session::errors::report_lit_error; use rustc_session::lint::BuiltinLintDiag; use rustc_session::lint::builtin::{ILL_FORMED_ATTRIBUTE_INPUT, UNSAFE_ATTR_OUTSIDE_UNSAFE}; use rustc_session::parse::ParseSess; use rustc_span::{Span, Symbol, sym}; -use crate::{errors, parse_in}; +use crate::{AttributeParser, Late, session_diagnostics as errors}; pub fn check_attr(psess: &ParseSess, attr: &Attribute, id: NodeId) { if attr.is_doc_comment() || attr.has_name(sym::cfg_trace) || attr.has_name(sym::cfg_attr_trace) @@ -33,7 +33,10 @@ pub fn check_attr(psess: &ParseSess, attr: &Attribute, id: NodeId) { // Check input tokens for built-in and key-value attributes. match builtin_attr_info { // `rustc_dummy` doesn't have any restrictions specific to built-in attributes. - Some(BuiltinAttribute { name, template, .. }) if *name != sym::rustc_dummy => { + Some(BuiltinAttribute { name, template, .. }) => { + if AttributeParser::::is_parsed_attribute(slice::from_ref(&name)) { + return; + } match parse_meta(psess, attr) { // Don't check safety again, we just did that Ok(meta) => { @@ -106,7 +109,7 @@ pub fn parse_meta<'a>(psess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Met res } else { // Example cases: - // - `#[foo = 1+1]`: results in `ast::ExprKind::BinOp`. + // - `#[foo = 1+1]`: results in `ast::ExprKind::Binary`. // - `#[foo = include_str!("nonexistent-file.rs")]`: // results in `ast::ExprKind::Err`. In that case we delay // the error because an earlier error will have already @@ -133,16 +136,6 @@ fn check_meta_bad_delim(psess: &ParseSess, span: DelimSpan, delim: Delimiter) { }); } -pub(super) fn check_cfg_attr_bad_delim(psess: &ParseSess, span: DelimSpan, delim: Delimiter) { - if let Delimiter::Parenthesis = delim { - return; - } - psess.dcx().emit_err(errors::CfgAttrBadDelim { - span: span.entire(), - sugg: errors::MetaBadDelimSugg { open: span.open, close: span.close }, - }); -} - /// Checks that the given meta-item is compatible with this `AttributeTemplate`. fn is_attr_template_compatible(template: &AttributeTemplate, meta: &ast::MetaItemKind) -> bool { let is_one_allowed_subword = |items: &[MetaItemInner]| match items { @@ -214,10 +207,9 @@ pub fn check_attribute_safety( } } - // - Normal builtin attribute, or any non-builtin attribute - // - All non-builtin attributes are currently considered safe; writing `#[unsafe(..)]` is - // not permitted on non-builtin attributes or normal builtin attributes - (Some(AttributeSafety::Normal) | None, Safety::Unsafe(unsafe_span)) => { + // - Normal builtin attribute + // - Writing `#[unsafe(..)]` is not permitted on normal builtin attributes + (Some(AttributeSafety::Normal), Safety::Unsafe(unsafe_span)) => { psess.dcx().emit_err(errors::InvalidAttrUnsafe { span: unsafe_span, name: attr_item.path.clone(), @@ -231,9 +223,8 @@ pub fn check_attribute_safety( } // - Non-builtin attribute - // - No explicit `#[unsafe(..)]` written. - (None, Safety::Default) => { - // OK + (None, Safety::Unsafe(_) | Safety::Default) => { + // OK (not checked here) } ( @@ -269,9 +260,6 @@ pub fn check_builtin_meta_item( ) { if !is_attr_template_compatible(&template, &meta.kind) { // attrs with new parsers are locally validated so excluded here - if AttributeParser::::is_parsed_attribute(slice::from_ref(&name)) { - return; - } emit_malformed_attribute(psess, style, meta.span, name, template); } @@ -298,35 +286,42 @@ fn emit_malformed_attribute( suggestions.push(format!("#{inner}[{name}]")); } if let Some(descr) = template.list { - suggestions.push(format!("#{inner}[{name}({descr})]")); + for descr in descr { + suggestions.push(format!("#{inner}[{name}({descr})]")); + } } suggestions.extend(template.one_of.iter().map(|&word| format!("#{inner}[{name}({word})]"))); if let Some(descr) = template.name_value_str { - suggestions.push(format!("#{inner}[{name} = \"{descr}\"]")); + for descr in descr { + suggestions.push(format!("#{inner}[{name} = \"{descr}\"]")); + } } if should_warn(name) { psess.buffer_lint( ILL_FORMED_ATTRIBUTE_INPUT, span, ast::CRATE_NODE_ID, - BuiltinLintDiag::IllFormedAttributeInput { suggestions: suggestions.clone() }, + BuiltinLintDiag::IllFormedAttributeInput { + suggestions: suggestions.clone(), + docs: template.docs, + }, ); } else { suggestions.sort(); - psess - .dcx() - .struct_span_err(span, error_msg) - .with_span_suggestions( - span, - if suggestions.len() == 1 { - "must be of the form" - } else { - "the following are the possible correct uses" - }, - suggestions, - Applicability::HasPlaceholders, - ) - .emit(); + let mut err = psess.dcx().struct_span_err(span, error_msg).with_span_suggestions( + span, + if suggestions.len() == 1 { + "must be of the form" + } else { + "the following are the possible correct uses" + }, + suggestions, + Applicability::HasPlaceholders, + ); + if let Some(link) = template.docs { + err.note(format!("for more information, visit <{link}>")); + } + err.emit(); } } diff --git a/compiler/rustc_baked_icu_data/Cargo.toml b/compiler/rustc_baked_icu_data/Cargo.toml index cb0e145386b8..2f1ab7df3790 100644 --- a/compiler/rustc_baked_icu_data/Cargo.toml +++ b/compiler/rustc_baked_icu_data/Cargo.toml @@ -5,9 +5,8 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -icu_list = "1.2" -icu_locid = "1.2" -icu_locid_transform = "1.3.2" -icu_provider = { version = "1.2", features = ["sync"] } -zerovec = "0.10.0" +icu_list = { version = "2.0", default-features = false } +icu_locale = { version = "2.0", default-features = false, features = ["compiled_data"] } +icu_provider = { version = "2.0", features = ["baked", "sync"] } +zerovec = "0.11.0" # tidy-alphabetical-end diff --git a/compiler/rustc_baked_icu_data/src/data/any.rs b/compiler/rustc_baked_icu_data/src/data/any.rs deleted file mode 100644 index 230288766764..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/any.rs +++ /dev/null @@ -1,2 +0,0 @@ -// @generated -impl_any_provider!(BakedDataProvider); diff --git a/compiler/rustc_baked_icu_data/src/data/list_and_v1.rs.data b/compiler/rustc_baked_icu_data/src/data/list_and_v1.rs.data new file mode 100644 index 000000000000..1d60e0085fcb --- /dev/null +++ b/compiler/rustc_baked_icu_data/src/data/list_and_v1.rs.data @@ -0,0 +1,71 @@ +// @generated +/// Implement `DataProvider` on the given struct using the data +/// hardcoded in this file. This allows the struct to be used with +/// `icu`'s `_unstable` constructors. +/// +/// Using this implementation will embed the following data in the binary's data segment: +/// * 179B for the lookup data structure (33 data identifiers) +/// * 4183B[^1] for the actual data (11 unique structs) +/// +/// [^1]: these numbers can be smaller in practice due to linker deduplication +/// +/// This macro requires the following crates: +/// * `icu_list` +/// * `icu_locale/compiled_data` +/// * `icu_provider` +/// * `icu_provider/baked` +/// * `zerovec` +#[doc(hidden)] +#[macro_export] +macro_rules! __impl_list_and_v1 { + ($ provider : ty) => { + #[clippy::msrv = "1.82"] + const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO; + #[clippy::msrv = "1.82"] + impl $provider { + const DATA_LIST_AND_V1: icu_provider::baked::zerotrie::Data = { + const TRIE: icu_provider::baked::zerotrie::ZeroTrieSimpleAscii<&'static [u8]> = icu_provider::baked::zerotrie::ZeroTrieSimpleAscii { store: b"\xC8efijprtz\x18#.9DOZ\xC2ns\n\x1E\xC3NSW\x01\x02\x80\x85\x8A\x1E\xC3NSW\x01\x02\x81\x81\x81r\x1E\xC3NSW\x01\x02\x80\x86\x86t\x1E\xC3NSW\x01\x02\x82\x82\x82a\x1E\xC3NSW\x01\x02\x83\x83\x83t\x1E\xC3NSW\x01\x02\x80\x82\x82u\x1E\xC3NSW\x01\x02\x80\x87\x87r\x1E\xC3NSW\x01\x02\x80\x88\x88h\xC2\x1E-\t\xC3NSW\x01\x02\x83\x89\x89Han\xC2st\n\x1E\xC3NSW\x01\x02\x83\x89\x89\x1E\xC3NSW\x01\x02\x84\x89\x89" }; + const VALUES: &'static [::DataStruct] = &[icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" y ") }, 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x02\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\0\0\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\x01\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x06\0\0\0\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF`\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\x12\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x02\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x05\x05\x05\x06\x06\x0C\x0C\r\r\0\0\0\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\x02\0\x1B\0\0\0\0\0\x12\0\0\0\x12\0\0\x03\x06\x06\r\r\0\0\0\0\0h\0\0\0h\0\0\0\0\0\0\x0E\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\x01\n\0\0\x01\x19\0\0\0\x12\0\0\x02\x0F\x11\0\0\0\0\0D\0\0\0\0\0\0\x02\x11\x11\0\0\0\0\0\xBF\0\0\0\0\0\0\x02\x0F\x11\0\0\0\0\0\xBF\0\0\0\0\0\0\x02\x0F\x10\0\0\0\0\0\xBF\0\0\0\0\0\0\x02\x10\x11\0\0\0\0\0\xDD\0\0\0\0\0\0\x02\x0F\x11\0\0\0\0\0\xDD\0\0\0\0\0\0\x02\x0F\x0F\0\0\0\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\x03\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\0\0\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\x01\0\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x06\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\0\0\0`\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\x12\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" e ") }, 3u8) }) }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" e ") }, 3u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, 3u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, 3u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, 3u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE5\x92\x8C") }, 3u8), special_case: None }, pair: Some(icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, 3u8), special_case: None }) }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", & ") }, 4u8), special_case: None }, pair: Some(icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" & ") }, 3u8), special_case: None }) }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" et ") }, 4u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" \xD0\xB8 ") }, 4u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" ve ") }, 4u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, 3u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE3\x80\x81") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b"\xE5\x92\x8C") }, 3u8), special_case: None }, pair: None }, icu_list::provider::ListFormatterPatterns { start: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, 2u8), middle: unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", ") }, end: icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b", and ") }, 6u8), special_case: None }, pair: Some(icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(unsafe { zerovec::VarZeroCow::from_bytes_unchecked(b" and ") }, 5u8), special_case: None }) }]; + unsafe { icu_provider::baked::zerotrie::Data::from_trie_and_values_unchecked(TRIE, VALUES) } + }; + } + #[clippy::msrv = "1.82"] + impl icu_provider::DataProvider for $provider { + fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { + let mut metadata = icu_provider::DataResponseMetadata::default(); + let payload = if let Some(payload) = icu_provider::baked::DataStore::get(&Self::DATA_LIST_AND_V1, req.id, req.metadata.attributes_prefix_match) { + payload + } else { + const FALLBACKER: icu_locale::fallback::LocaleFallbackerWithConfig<'static> = icu_locale::fallback::LocaleFallbacker::new().for_config(::INFO.fallback_config); + let mut fallback_iterator = FALLBACKER.fallback_for(req.id.locale.clone()); + loop { + if let Some(payload) = icu_provider::baked::DataStore::get(&Self::DATA_LIST_AND_V1, icu_provider::DataIdentifierBorrowed::for_marker_attributes_and_locale(req.id.marker_attributes, fallback_iterator.get()), req.metadata.attributes_prefix_match) { + metadata.locale = Some(fallback_iterator.take()); + break payload; + } + if fallback_iterator.get().is_unknown() { + return Err(icu_provider::DataErrorKind::IdentifierNotFound.with_req(::INFO, req)); + } + fallback_iterator.step(); + } + }; + Ok(icu_provider::DataResponse { payload, metadata }) + } + } + }; + ($ provider : ty , ITER) => { + __impl_list_and_v1!($provider); + #[clippy::msrv = "1.82"] + impl icu_provider::IterableDataProvider for $provider { + fn iter_ids(&self) -> Result>, icu_provider::DataError> { + Ok(icu_provider::baked::DataStore::iter(&Self::DATA_LIST_AND_V1).collect()) + } + } + }; + ($ provider : ty , DRY) => {}; + ($ provider : ty , DRY , ITER) => { + __impl_list_and_v1!($provider, ITER); + }; +} +#[doc(inline)] +pub use __impl_list_and_v1 as impl_list_and_v1; diff --git a/compiler/rustc_baked_icu_data/src/data/macros.rs b/compiler/rustc_baked_icu_data/src/data/macros.rs deleted file mode 100644 index bee309f9b811..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/macros.rs +++ /dev/null @@ -1,46 +0,0 @@ -// @generated -/// Marks a type as a data provider. You can then use macros like -/// `impl_core_helloworld_v1` to add implementations. -/// -/// ```ignore -/// struct MyProvider; -/// const _: () = { -/// include!("path/to/generated/macros.rs"); -/// make_provider!(MyProvider); -/// impl_core_helloworld_v1!(MyProvider); -/// } -/// ``` -#[doc(hidden)] -#[macro_export] -macro_rules! __make_provider { - ($ name : ty) => { - #[clippy::msrv = "1.66"] - impl $name { - #[doc(hidden)] - #[allow(dead_code)] - pub const MUST_USE_MAKE_PROVIDER_MACRO: () = (); - } - }; -} -#[doc(inline)] -pub use __make_provider as make_provider; -#[macro_use] -#[path = "macros/fallback_likelysubtags_v1.data.rs"] -mod fallback_likelysubtags_v1; -#[doc(inline)] -pub use __impl_fallback_likelysubtags_v1 as impl_fallback_likelysubtags_v1; -#[macro_use] -#[path = "macros/fallback_parents_v1.data.rs"] -mod fallback_parents_v1; -#[doc(inline)] -pub use __impl_fallback_parents_v1 as impl_fallback_parents_v1; -#[macro_use] -#[path = "macros/fallback_supplement_co_v1.data.rs"] -mod fallback_supplement_co_v1; -#[doc(inline)] -pub use __impl_fallback_supplement_co_v1 as impl_fallback_supplement_co_v1; -#[macro_use] -#[path = "macros/list_and_v1.data.rs"] -mod list_and_v1; -#[doc(inline)] -pub use __impl_list_and_v1 as impl_list_and_v1; diff --git a/compiler/rustc_baked_icu_data/src/data/macros/fallback_likelysubtags_v1.data.rs b/compiler/rustc_baked_icu_data/src/data/macros/fallback_likelysubtags_v1.data.rs deleted file mode 100644 index 1adb58743f72..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/macros/fallback_likelysubtags_v1.data.rs +++ /dev/null @@ -1,40 +0,0 @@ -// @generated -/// Implement `DataProvider` on the given struct using the data -/// hardcoded in this file. This allows the struct to be used with -/// `icu`'s `_unstable` constructors. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_fallback_likelysubtags_v1 { - ($ provider : ty) => { - #[clippy::msrv = "1.66"] - const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO; - #[clippy::msrv = "1.66"] - impl $provider { - #[doc(hidden)] - pub const SINGLETON_FALLBACK_LIKELYSUBTAGS_V1: &'static ::Yokeable = &icu_locid_transform::provider::LocaleFallbackLikelySubtagsV1 { - l2s: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap::from_parts_unchecked(unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"am\0ar\0as\0be\0bg\0bgcbhobn\0brxchrcv\0doiel\0fa\0gu\0he\0hi\0hy\0ja\0ka\0kk\0km\0kn\0ko\0kokks\0ky\0lo\0maimk\0ml\0mn\0mnimr\0my\0ne\0or\0pa\0ps\0rajru\0sa\0satsd\0si\0sr\0ta\0te\0tg\0th\0ti\0tt\0uk\0ur\0yuezh\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"EthiArabBengCyrlCyrlDevaDevaBengDevaCherCyrlDevaGrekArabGujrHebrDevaArmnJpanGeorCyrlKhmrKndaKoreDevaArabCyrlLaooDevaCyrlMlymCyrlBengDevaMymrDevaOryaGuruArabDevaCyrlDevaOlckArabSinhCyrlTamlTeluCyrlThaiEthiCyrlCyrlArabHantHans") }) - }, - lr2s: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap2d::from_parts_unchecked(unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"az\0ha\0kk\0ky\0mn\0ms\0pa\0sd\0sr\0tg\0uz\0yuezh\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"\x03\0\0\0\x05\0\0\0\t\0\0\0\x0B\0\0\0\x0C\0\0\0\r\0\0\0\x0E\0\0\0\x0F\0\0\0\x13\0\0\0\x14\0\0\0\x16\0\0\0\x17\0\0\0&\0\0\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"IQ\0IR\0RU\0CM\0SD\0AF\0CN\0IR\0MN\0CN\0TR\0CN\0CC\0PK\0IN\0ME\0RO\0RU\0TR\0PK\0AF\0CN\0CN\0AU\0BN\0GB\0GF\0HK\0ID\0MO\0PA\0PF\0PH\0SR\0TH\0TW\0US\0VN\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"ArabArabCyrlArabArabArabArabArabArabArabLatnMongArabArabDevaLatnLatnLatnLatnArabArabCyrlHansHantHantHantHantHantHantHantHantHantHantHantHantHantHantHant") }) - }, - l2r: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap::from_parts_unchecked(unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"af\0am\0ar\0as\0astaz\0be\0bg\0bgcbhobn\0br\0brxbs\0ca\0cebchrcs\0cv\0cy\0da\0de\0doidsbel\0en\0es\0et\0eu\0fa\0ff\0fi\0filfo\0fr\0ga\0gd\0gl\0gu\0ha\0he\0hi\0hr\0hsbhu\0hy\0ia\0id\0ig\0is\0it\0ja\0jv\0ka\0keakgpkk\0km\0kn\0ko\0kokks\0ky\0lo\0lt\0lv\0maimi\0mk\0ml\0mn\0mnimr\0ms\0my\0ne\0nl\0nn\0no\0or\0pa\0pcmpl\0ps\0pt\0qu\0rajrm\0ro\0ru\0sa\0satsc\0sd\0si\0sk\0sl\0so\0sq\0sr\0su\0sv\0sw\0ta\0te\0tg\0th\0ti\0tk\0to\0tr\0tt\0uk\0ur\0uz\0vi\0wo\0xh\0yo\0yrlyuezh\0zu\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"ZA\0ET\0EG\0IN\0ES\0AZ\0BY\0BG\0IN\0IN\0BD\0FR\0IN\0BA\0ES\0PH\0US\0CZ\0RU\0GB\0DK\0DE\0IN\0DE\0GR\0US\0ES\0EE\0ES\0IR\0SN\0FI\0PH\0FO\0FR\0IE\0GB\0ES\0IN\0NG\0IL\0IN\0HR\0DE\0HU\0AM\x00001ID\0NG\0IS\0IT\0JP\0ID\0GE\0CV\0BR\0KZ\0KH\0IN\0KR\0IN\0IN\0KG\0LA\0LT\0LV\0IN\0NZ\0MK\0IN\0MN\0IN\0IN\0MY\0MM\0NP\0NL\0NO\0NO\0IN\0IN\0NG\0PL\0AF\0BR\0PE\0IN\0CH\0RO\0RU\0IN\0IN\0IT\0PK\0LK\0SK\0SI\0SO\0AL\0RS\0ID\0SE\0TZ\0IN\0IN\0TJ\0TH\0ET\0TM\0TO\0TR\0RU\0UA\0PK\0UZ\0VN\0SN\0ZA\0NG\0BR\0HK\0CN\0ZA\0") }) - }, - ls2r: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap2d::from_parts_unchecked(unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"az\0en\0ff\0kk\0ky\0mn\0pa\0sd\0tg\0uz\0yuezh\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"\x01\0\0\0\x02\0\0\0\x03\0\0\0\x04\0\0\0\x06\0\0\0\x07\0\0\0\x08\0\0\0\x0B\0\0\0\x0C\0\0\0\r\0\0\0\x0E\0\0\0\x11\0\0\0") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"ArabShawAdlmArabArabLatnMongArabDevaKhojSindArabArabHansBopoHanbHant") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"IR\0GB\0GN\0CN\0CN\0TR\0CN\0PK\0IN\0IN\0IN\0PK\0AF\0CN\0TW\0TW\0TW\0") }) - }, - }; - } - #[clippy::msrv = "1.66"] - impl icu_provider::DataProvider for $provider { - fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { - if req.locale.is_empty() { Ok(icu_provider::DataResponse { payload: Some(icu_provider::DataPayload::from_static_ref(Self::SINGLETON_FALLBACK_LIKELYSUBTAGS_V1)), metadata: Default::default() }) } else { Err(icu_provider::DataErrorKind::ExtraneousLocale.with_req(::KEY, req)) } - } - } - }; -} diff --git a/compiler/rustc_baked_icu_data/src/data/macros/fallback_parents_v1.data.rs b/compiler/rustc_baked_icu_data/src/data/macros/fallback_parents_v1.data.rs deleted file mode 100644 index 6f8d6590b085..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/macros/fallback_parents_v1.data.rs +++ /dev/null @@ -1,28 +0,0 @@ -// @generated -/// Implement `DataProvider` on the given struct using the data -/// hardcoded in this file. This allows the struct to be used with -/// `icu`'s `_unstable` constructors. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_fallback_parents_v1 { - ($ provider : ty) => { - #[clippy::msrv = "1.66"] - const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO; - #[clippy::msrv = "1.66"] - impl $provider { - #[doc(hidden)] - pub const SINGLETON_FALLBACK_PARENTS_V1: &'static ::Yokeable = &icu_locid_transform::provider::LocaleFallbackParentsV1 { - parents: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap::from_parts_unchecked(unsafe { zerovec::VarZeroVec::from_bytes_unchecked(b"\x84\0\0\0\0\0\x06\0\x0B\0\x10\0\x15\0\x1A\0\x1F\0$\0)\0.\x003\08\0=\0B\0G\0L\0Q\0V\0[\0`\0e\0j\0o\0t\0y\0~\0\x83\0\x88\0\x8D\0\x92\0\x97\0\x9C\0\xA1\0\xA6\0\xAB\0\xB0\0\xB5\0\xBA\0\xBF\0\xC4\0\xC9\0\xCE\0\xD3\0\xD8\0\xDD\0\xE2\0\xE7\0\xEC\0\xF1\0\xF6\0\xFB\0\0\x01\x05\x01\n\x01\x0F\x01\x14\x01\x19\x01\x1E\x01#\x01(\x01-\x012\x017\x01<\x01A\x01F\x01K\x01P\x01U\x01Z\x01_\x01d\x01i\x01n\x01s\x01x\x01}\x01\x82\x01\x87\x01\x8C\x01\x91\x01\x96\x01\x9B\x01\xA0\x01\xA5\x01\xAA\x01\xAF\x01\xB4\x01\xB9\x01\xBE\x01\xC3\x01\xC8\x01\xCD\x01\xD2\x01\xD7\x01\xDC\x01\xE1\x01\xE6\x01\xEB\x01\xF0\x01\xF5\x01\xFA\x01\xFF\x01\x04\x02\t\x02\x0E\x02\x13\x02\x18\x02\x1D\x02\"\x02'\x02,\x021\x026\x02;\x02@\x02G\x02I\x02K\x02M\x02R\x02W\x02\\\x02a\x02f\x02k\x02p\x02u\x02z\x02\x7F\x02\x84\x02\x89\x02en-150en-AGen-AIen-ATen-AUen-BBen-BEen-BMen-BSen-BWen-BZen-CCen-CHen-CKen-CMen-CXen-CYen-DEen-DGen-DKen-DMen-ERen-FIen-FJen-FKen-FMen-GBen-GDen-GGen-GHen-GIen-GMen-GYen-HKen-IEen-ILen-IMen-INen-IOen-JEen-JMen-KEen-KIen-KNen-KYen-LCen-LRen-LSen-MGen-MOen-MSen-MTen-MUen-MVen-MWen-MYen-NAen-NFen-NGen-NLen-NRen-NUen-NZen-PGen-PKen-PNen-PWen-RWen-SBen-SCen-SDen-SEen-SGen-SHen-SIen-SLen-SSen-SXen-SZen-TCen-TKen-TOen-TTen-TVen-TZen-UGen-VCen-VGen-VUen-WSen-ZAen-ZMen-ZWes-ARes-BOes-BRes-BZes-CLes-COes-CRes-CUes-DOes-ECes-GTes-HNes-MXes-NIes-PAes-PEes-PRes-PYes-SVes-USes-UYes-VEhi-Latnhtnbnnno-NOpt-AOpt-CHpt-CVpt-FRpt-GQpt-GWpt-LUpt-MOpt-MZpt-STpt-TLzh-Hant-MO") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01150en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001en\0\0\0\0\0\0\x01001es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419es\0\0\0\0\0\0\x01419en\0\0\0\0\0\0\x01IN\0fr\0\0\0\0\0\0\x01HT\0no\0\0\0\0\0\0\0\0\0\0no\0\0\0\0\0\0\0\0\0\0no\0\0\0\0\0\0\0\0\0\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0pt\0\0\0\0\0\0\x01PT\0zh\0\x01Hant\x01HK\0") }) - }, - }; - } - #[clippy::msrv = "1.66"] - impl icu_provider::DataProvider for $provider { - fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { - if req.locale.is_empty() { Ok(icu_provider::DataResponse { payload: Some(icu_provider::DataPayload::from_static_ref(Self::SINGLETON_FALLBACK_PARENTS_V1)), metadata: Default::default() }) } else { Err(icu_provider::DataErrorKind::ExtraneousLocale.with_req(::KEY, req)) } - } - } - }; -} diff --git a/compiler/rustc_baked_icu_data/src/data/macros/fallback_supplement_co_v1.data.rs b/compiler/rustc_baked_icu_data/src/data/macros/fallback_supplement_co_v1.data.rs deleted file mode 100644 index 02eec37ee09f..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/macros/fallback_supplement_co_v1.data.rs +++ /dev/null @@ -1,32 +0,0 @@ -// @generated -/// Implement `DataProvider` on the given struct using the data -/// hardcoded in this file. This allows the struct to be used with -/// `icu`'s `_unstable` constructors. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_fallback_supplement_co_v1 { - ($ provider : ty) => { - #[clippy::msrv = "1.66"] - const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO; - #[clippy::msrv = "1.66"] - impl $provider { - #[doc(hidden)] - pub const SINGLETON_FALLBACK_SUPPLEMENT_CO_V1: &'static ::Yokeable = &icu_locid_transform::provider::LocaleFallbackSupplementV1 { - parents: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap::from_parts_unchecked(unsafe { zerovec::VarZeroVec::from_bytes_unchecked(b"\x01\0\0\0\0\0yue") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"zh\0\x01Hant\0\0\0\0") }) - }, - unicode_extension_defaults: unsafe { - #[allow(unused_unsafe)] - zerovec::ZeroMap2d::from_parts_unchecked(unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"co") }, unsafe { zerovec::ZeroVec::from_bytes_unchecked(b"\x02\0\0\0") }, unsafe { zerovec::VarZeroVec::from_bytes_unchecked(b"\x02\0\0\0\0\0\x02\0zhzh-Hant") }, unsafe { zerovec::VarZeroVec::from_bytes_unchecked(b"\x02\0\0\0\0\0\x06\0pinyinstroke") }) - }, - }; - } - #[clippy::msrv = "1.66"] - impl icu_provider::DataProvider for $provider { - fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { - if req.locale.is_empty() { Ok(icu_provider::DataResponse { payload: Some(icu_provider::DataPayload::from_static_ref(Self::SINGLETON_FALLBACK_SUPPLEMENT_CO_V1)), metadata: Default::default() }) } else { Err(icu_provider::DataErrorKind::ExtraneousLocale.with_req(::KEY, req)) } - } - } - }; -} diff --git a/compiler/rustc_baked_icu_data/src/data/macros/list_and_v1.data.rs b/compiler/rustc_baked_icu_data/src/data/macros/list_and_v1.data.rs deleted file mode 100644 index 186f706cdb28..000000000000 --- a/compiler/rustc_baked_icu_data/src/data/macros/list_and_v1.data.rs +++ /dev/null @@ -1,35 +0,0 @@ -// @generated -/// Implement `DataProvider` on the given struct using the data -/// hardcoded in this file. This allows the struct to be used with -/// `icu`'s `_unstable` constructors. -#[doc(hidden)] -#[macro_export] -macro_rules! __impl_list_and_v1 { - ($ provider : ty) => { - #[clippy::msrv = "1.66"] - const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO; - #[clippy::msrv = "1.66"] - impl icu_provider::DataProvider for $provider { - fn load(&self, req: icu_provider::DataRequest) -> Result, icu_provider::DataError> { - static EN_001: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static EN_IN: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", and ", 6u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static IT: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }]); - static PT: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static FR: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" et ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" et ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" et ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" et ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static TR: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" ve ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" ve ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" ve ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" ve ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static ES: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" y ", 3u8), special_case: Some(icu_list::provider::SpecialCasePattern { condition: unsafe { icu_list::provider::SerdeDFA::from_dfa_bytes_unchecked(if cfg!(target_endian = "little") { b"rust-regex-automata-dfa-sparse\0\0\xFF\xFE\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B(\x01\0\0\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#\0\0\0" } else { b"rust-regex-automata-dfa-sparse\0\0\0\0\xFE\xFF\0\0\0\x02\0\0\0\0\0\0\0\x0E\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\x02\x02\x02\x03\x04\x04\x05\x06\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x07\x08\t\t\t\n\x0B\x0B\x0C\r\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0E\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x0F\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x10\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x12\x12\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x13\x14\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x15\x16\x17\x17\x18\x19\x19\x19\x1A\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\x1B\0\0\x01(\x01\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x01\x80\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\x05\0\x05\x05\x06\x06\x0C\x0C\r\r\0\0S\0\0\0D\0\0\0S\0\0\0D\0\0\0\0\0\0\0\0\x02\0\0\x1B\0\0\x12\0\0\0\x12\0\0\0\0\x03\0\x06\x06\r\r\0\0h\0\0\0h\0\0\0\0\0\0\0\0\x0E\0\0\0\x02\x02\x04\x07\t\t\x0B\x0E\x13\x13\x14\x14\x15\x15\x16\x16\x17\x17\x18\x18\x19\x19\x1A\x1A\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0D\0\0\0\xBF\0\0\0\xCE\0\0\0\xDD\0\0\0\xEC\0\0\0\xDD\0\0\0\xFB\0\0\0\n\x01\0\0\x19\x01\0\0\x12\0\0\0\0\x02\0\x0F\x11\0\0D\0\0\0\0\0\0\0\0\x02\0\x11\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x0F\x10\0\0\xBF\0\0\0\0\0\0\0\0\x02\0\x10\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x11\0\0\xDD\0\0\0\0\0\0\0\0\x02\0\x0F\x0F\0\0\xDD\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0#\0\0\0#\0\0\0#\0\0\0#\0\0\0\0\0\0#\0\0\0\t\0\0\0\x12\0\0\0\x12\0\0\0\0\0\0\0\0\0\0\0#\0\0\0#" }) }, pattern: icu_list::provider::ListJoinerPattern::from_parts(" e ", 3u8) }) }]); - static RU: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" и ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" и ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" и ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" и ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static UND: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static EN: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", and ", 6u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" and ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", & ", 4u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" & ", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }]); - static HI_LATN: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", aur ", 6u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" aur ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", aur ", 6u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" aur ", 5u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(", ", 2u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts(" aur ", 5u8), special_case: None }]); - static JA: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }]); - static ZH_HK: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("及", 3u8), special_case: None }]); - static ZH: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }]); - static ZH_HANT: ::Yokeable = icu_list::provider::ListFormatterPatternsV1([icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("、", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }, icu_list::provider::ConditionalListJoinerPattern { default: icu_list::provider::ListJoinerPattern::from_parts("和", 3u8), special_case: None }]); - static VALUES: [&::Yokeable; 215usize] = [&EN, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_IN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN, &EN_001, &EN_001, &EN_001, &EN_001, &EN_001, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &ES, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &FR, &HI_LATN, &IT, &IT, &IT, &IT, &JA, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &PT, &RU, &RU, &RU, &RU, &RU, &RU, &TR, &TR, &UND, &ZH, &ZH_HK, &ZH, &ZH, &ZH, &ZH_HANT, &ZH_HK, &ZH]; - static KEYS: [&str; 215usize] = ["en", "en-001", "en-150", "en-AE", "en-AG", "en-AI", "en-AS", "en-AT", "en-AU", "en-BB", "en-BE", "en-BI", "en-BM", "en-BS", "en-BW", "en-BZ", "en-CA", "en-CC", "en-CH", "en-CK", "en-CM", "en-CX", "en-CY", "en-DE", "en-DG", "en-DK", "en-DM", "en-ER", "en-FI", "en-FJ", "en-FK", "en-FM", "en-GB", "en-GD", "en-GG", "en-GH", "en-GI", "en-GM", "en-GU", "en-GY", "en-HK", "en-IE", "en-IL", "en-IM", "en-IN", "en-IO", "en-JE", "en-JM", "en-KE", "en-KI", "en-KN", "en-KY", "en-LC", "en-LR", "en-LS", "en-MG", "en-MH", "en-MO", "en-MP", "en-MS", "en-MT", "en-MU", "en-MV", "en-MW", "en-MY", "en-NA", "en-NF", "en-NG", "en-NL", "en-NR", "en-NU", "en-NZ", "en-PG", "en-PH", "en-PK", "en-PN", "en-PR", "en-PW", "en-RW", "en-SB", "en-SC", "en-SD", "en-SE", "en-SG", "en-SH", "en-SI", "en-SL", "en-SS", "en-SX", "en-SZ", "en-TC", "en-TK", "en-TO", "en-TT", "en-TV", "en-TZ", "en-UG", "en-UM", "en-VC", "en-VG", "en-VI", "en-VU", "en-WS", "en-ZA", "en-ZM", "en-ZW", "es", "es-419", "es-AR", "es-BO", "es-BR", "es-BZ", "es-CL", "es-CO", "es-CR", "es-CU", "es-DO", "es-EA", "es-EC", "es-GQ", "es-GT", "es-HN", "es-IC", "es-MX", "es-NI", "es-PA", "es-PE", "es-PH", "es-PR", "es-PY", "es-SV", "es-US", "es-UY", "es-VE", "fr", "fr-BE", "fr-BF", "fr-BI", "fr-BJ", "fr-BL", "fr-CA", "fr-CD", "fr-CF", "fr-CG", "fr-CH", "fr-CI", "fr-CM", "fr-DJ", "fr-DZ", "fr-GA", "fr-GF", "fr-GN", "fr-GP", "fr-GQ", "fr-HT", "fr-KM", "fr-LU", "fr-MA", "fr-MC", "fr-MF", "fr-MG", "fr-ML", "fr-MQ", "fr-MR", "fr-MU", "fr-NC", "fr-NE", "fr-PF", "fr-PM", "fr-RE", "fr-RW", "fr-SC", "fr-SN", "fr-SY", "fr-TD", "fr-TG", "fr-TN", "fr-VU", "fr-WF", "fr-YT", "hi-Latn", "it", "it-CH", "it-SM", "it-VA", "ja", "pt", "pt-AO", "pt-CH", "pt-CV", "pt-GQ", "pt-GW", "pt-LU", "pt-MO", "pt-MZ", "pt-PT", "pt-ST", "pt-TL", "ru", "ru-BY", "ru-KG", "ru-KZ", "ru-MD", "ru-UA", "tr", "tr-CY", "und", "zh", "zh-HK", "zh-Hans", "zh-Hans-HK", "zh-Hans-MO", "zh-Hant", "zh-MO", "zh-SG"]; - if let Ok(payload) = KEYS.binary_search_by(|k| req.locale.strict_cmp(k.as_bytes()).reverse()).map(|i| *unsafe { VALUES.get_unchecked(i) }) { Ok(icu_provider::DataResponse { payload: Some(icu_provider::DataPayload::from_static_ref(payload)), metadata: Default::default() }) } else { Err(icu_provider::DataErrorKind::MissingLocale.with_req(::KEY, req)) } - } - } - }; -} diff --git a/compiler/rustc_baked_icu_data/src/data/mod.rs b/compiler/rustc_baked_icu_data/src/data/mod.rs index 465689f0cb8d..3146188a8e7a 100644 --- a/compiler/rustc_baked_icu_data/src/data/mod.rs +++ b/compiler/rustc_baked_icu_data/src/data/mod.rs @@ -1,31 +1,40 @@ // @generated -include!("macros.rs"); +include!("list_and_v1.rs.data"); +/// Marks a type as a data provider. You can then use macros like +/// `impl_core_helloworld_v1` to add implementations. +/// +/// ```ignore +/// struct MyProvider; +/// const _: () = { +/// include!("path/to/generated/macros.rs"); +/// make_provider!(MyProvider); +/// impl_core_helloworld_v1!(MyProvider); +/// } +/// ``` +#[doc(hidden)] +#[macro_export] +macro_rules! __make_provider { + ($ name : ty) => { + #[clippy::msrv = "1.82"] + impl $name { + #[allow(dead_code)] + pub(crate) const MUST_USE_MAKE_PROVIDER_MACRO: () = (); + } + icu_provider::marker::impl_data_provider_never_marker!($name); + }; +} +#[doc(inline)] +pub use __make_provider as make_provider; +/// This macro requires the following crates: +/// * `icu_list` +/// * `icu_locale/compiled_data` +/// * `icu_provider` +/// * `icu_provider/baked` +/// * `zerovec` +#[allow(unused_macros)] macro_rules! impl_data_provider { ($ provider : ty) => { make_provider!($provider); - impl_fallback_likelysubtags_v1!($provider); - impl_fallback_parents_v1!($provider); - impl_fallback_supplement_co_v1!($provider); impl_list_and_v1!($provider); }; } -#[allow(unused_macros)] -macro_rules! impl_any_provider { - ($ provider : ty) => { - #[clippy::msrv = "1.66"] - impl icu_provider::AnyProvider for $provider { - fn load_any(&self, key: icu_provider::DataKey, req: icu_provider::DataRequest) -> Result { - match key.hashed() { - h if h == ::KEY.hashed() => icu_provider::DataProvider::::load(self, req).map(icu_provider::DataResponse::wrap_into_any_response), - h if h == ::KEY.hashed() => icu_provider::DataProvider::::load(self, req).map(icu_provider::DataResponse::wrap_into_any_response), - h if h == ::KEY.hashed() => icu_provider::DataProvider::::load(self, req).map(icu_provider::DataResponse::wrap_into_any_response), - h if h == ::KEY.hashed() => icu_provider::DataProvider::::load(self, req).map(icu_provider::DataResponse::wrap_into_any_response), - _ => Err(icu_provider::DataErrorKind::MissingDataKey.with_req(key, req)), - } - } - } - }; -} -#[clippy::msrv = "1.66"] -pub struct BakedDataProvider; -impl_data_provider!(BakedDataProvider); diff --git a/compiler/rustc_baked_icu_data/src/lib.rs b/compiler/rustc_baked_icu_data/src/lib.rs index f3f6522f5758..ea4c8242c629 100644 --- a/compiler/rustc_baked_icu_data/src/lib.rs +++ b/compiler/rustc_baked_icu_data/src/lib.rs @@ -14,10 +14,9 @@ //! To regenerate the data, run this command: //! //! ```text -//! icu4x-datagen -W --pretty --fingerprint --use-separate-crates \ -//! --format mod -l en es fr it ja pt ru tr zh zh-Hans zh-Hant \ -//! -k list/and@1 fallback/likelysubtags@1 fallback/parents@1 fallback/supplement/co@1 \ -//! --cldr-tag latest --icuexport-tag latest -o src/data +//! icu4x-datagen -W --pretty --use-separate-crates \ +//! --format baked --locales @en @es @fr @it @ja @pt @ru @tr @zh @zh-Hans @zh-Hant \ +//! -m ListAndV1 -o src/data //! ``` // tidy-alphabetical-start @@ -29,26 +28,26 @@ // #![warn(unreachable_pub)] // don't use because this crate is mostly generated code // tidy-alphabetical-end -mod data { - include!("data/mod.rs"); - include!("data/any.rs"); -} +pub struct BakedDataProvider; -pub use data::BakedDataProvider; +include!("data/mod.rs"); +const _: () = { + impl_data_provider!(BakedDataProvider); +}; pub const fn baked_data_provider() -> BakedDataProvider { - data::BakedDataProvider + BakedDataProvider } pub mod supported_locales { - pub const EN: icu_locid::Locale = icu_locid::locale!("en"); - pub const ES: icu_locid::Locale = icu_locid::locale!("es"); - pub const FR: icu_locid::Locale = icu_locid::locale!("fr"); - pub const IT: icu_locid::Locale = icu_locid::locale!("it"); - pub const JA: icu_locid::Locale = icu_locid::locale!("ja"); - pub const PT: icu_locid::Locale = icu_locid::locale!("pt"); - pub const RU: icu_locid::Locale = icu_locid::locale!("ru"); - pub const TR: icu_locid::Locale = icu_locid::locale!("tr"); - pub const ZH_HANS: icu_locid::Locale = icu_locid::locale!("zh-Hans"); - pub const ZH_HANT: icu_locid::Locale = icu_locid::locale!("zh-Hant"); + pub const EN: icu_locale::Locale = icu_locale::locale!("en"); + pub const ES: icu_locale::Locale = icu_locale::locale!("es"); + pub const FR: icu_locale::Locale = icu_locale::locale!("fr"); + pub const IT: icu_locale::Locale = icu_locale::locale!("it"); + pub const JA: icu_locale::Locale = icu_locale::locale!("ja"); + pub const PT: icu_locale::Locale = icu_locale::locale!("pt"); + pub const RU: icu_locale::Locale = icu_locale::locale!("ru"); + pub const TR: icu_locale::Locale = icu_locale::locale!("tr"); + pub const ZH_HANS: icu_locale::Locale = icu_locale::locale!("zh-Hans"); + pub const ZH_HANT: icu_locale::Locale = icu_locale::locale!("zh-Hant"); } diff --git a/compiler/rustc_borrowck/messages.ftl b/compiler/rustc_borrowck/messages.ftl index 33b80c4b03d6..f59e106c7ac3 100644 --- a/compiler/rustc_borrowck/messages.ftl +++ b/compiler/rustc_borrowck/messages.ftl @@ -90,7 +90,7 @@ borrowck_lifetime_constraints_error = lifetime may not live long enough borrowck_limitations_implies_static = - due to current limitations in the borrow checker, this implies a `'static` lifetime + due to a current limitation of the type system, this implies a `'static` lifetime borrowck_move_closure_suggestion = consider adding 'move' keyword before the nested closure diff --git a/compiler/rustc_borrowck/src/borrowck_errors.rs b/compiler/rustc_borrowck/src/borrowck_errors.rs index c9be5575da5c..7c9011505d64 100644 --- a/compiler/rustc_borrowck/src/borrowck_errors.rs +++ b/compiler/rustc_borrowck/src/borrowck_errors.rs @@ -426,7 +426,7 @@ impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> { } pub(crate) fn path_does_not_live_long_enough(&self, span: Span, path: &str) -> Diag<'infcx> { - struct_span_code_err!(self.dcx(), span, E0597, "{} does not live long enough", path,) + struct_span_code_err!(self.dcx(), span, E0597, "{} does not live long enough", path) } pub(crate) fn cannot_return_reference_to_local( @@ -480,7 +480,7 @@ impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> { } pub(crate) fn temporary_value_borrowed_for_too_long(&self, span: Span) -> Diag<'infcx> { - struct_span_code_err!(self.dcx(), span, E0716, "temporary value dropped while borrowed",) + struct_span_code_err!(self.dcx(), span, E0716, "temporary value dropped while borrowed") } } diff --git a/compiler/rustc_borrowck/src/consumers.rs b/compiler/rustc_borrowck/src/consumers.rs index 1c4ff5a67790..548973714105 100644 --- a/compiler/rustc_borrowck/src/consumers.rs +++ b/compiler/rustc_borrowck/src/consumers.rs @@ -17,7 +17,7 @@ pub use super::polonius::legacy::{ RichLocation, RustcFacts, }; pub use super::region_infer::RegionInferenceContext; -use crate::{BorrowCheckRootCtxt, do_mir_borrowck}; +use crate::BorrowCheckRootCtxt; /// Struct used during mir borrowck to collect bodies with facts for a typeck root and all /// its nested bodies. @@ -127,13 +127,6 @@ pub fn get_bodies_with_borrowck_facts( ) -> FxHashMap> { let mut root_cx = BorrowCheckRootCtxt::new(tcx, root_def_id, Some(BorrowckConsumer::new(options))); - - // See comment in `rustc_borrowck::mir_borrowck` - let nested_bodies = tcx.nested_bodies_within(root_def_id); - for def_id in nested_bodies { - root_cx.get_or_insert_nested(def_id); - } - - do_mir_borrowck(&mut root_cx, root_def_id); + root_cx.do_mir_borrowck(); root_cx.consumer.unwrap().bodies } diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs index 57db2e9fb574..813ceaeb8da9 100644 --- a/compiler/rustc_borrowck/src/dataflow.rs +++ b/compiler/rustc_borrowck/src/dataflow.rs @@ -1,7 +1,6 @@ use std::fmt; use rustc_data_structures::fx::FxIndexMap; -use rustc_data_structures::graph; use rustc_index::bit_set::{DenseBitSet, MixedBitSet}; use rustc_middle::mir::{ self, BasicBlock, Body, CallReturnPlaces, Location, Place, TerminatorEdges, @@ -45,7 +44,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> { } fn apply_early_statement_effect( - &mut self, + &self, state: &mut Self::Domain, stmt: &mir::Statement<'tcx>, loc: Location, @@ -56,7 +55,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> { } fn apply_primary_statement_effect( - &mut self, + &self, state: &mut Self::Domain, stmt: &mir::Statement<'tcx>, loc: Location, @@ -67,7 +66,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> { } fn apply_early_terminator_effect( - &mut self, + &self, state: &mut Self::Domain, term: &mir::Terminator<'tcx>, loc: Location, @@ -78,7 +77,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> { } fn apply_primary_terminator_effect<'mir>( - &mut self, + &self, state: &mut Self::Domain, term: &'mir mir::Terminator<'tcx>, loc: Location, @@ -93,7 +92,7 @@ impl<'a, 'tcx> Analysis<'tcx> for Borrowck<'a, 'tcx> { } fn apply_call_return_effect( - &mut self, + &self, _state: &mut Self::Domain, _block: BasicBlock, _return_places: CallReturnPlaces<'_, 'tcx>, @@ -317,9 +316,8 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { loans_out_of_scope_at_location: FxIndexMap::default(), }; for (loan_idx, loan_data) in borrow_set.iter_enumerated() { - let issuing_region = loan_data.region; let loan_issued_at = loan_data.reserve_location; - prec.precompute_loans_out_of_scope(loan_idx, issuing_region, loan_issued_at); + prec.precompute_loans_out_of_scope(loan_idx, loan_issued_at); } prec.loans_out_of_scope_at_location @@ -328,45 +326,7 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> { /// Loans are in scope while they are live: whether they are contained within any live region. /// In the location-insensitive analysis, a loan will be contained in a region if the issuing /// region can reach it in the subset graph. So this is a reachability problem. - fn precompute_loans_out_of_scope( - &mut self, - loan_idx: BorrowIndex, - issuing_region: RegionVid, - loan_issued_at: Location, - ) { - let sccs = self.regioncx.constraint_sccs(); - let universal_regions = self.regioncx.universal_regions(); - - // The loop below was useful for the location-insensitive analysis but shouldn't be - // impactful in the location-sensitive case. It seems that it does, however, as without it a - // handful of tests fail. That likely means some liveness or outlives data related to choice - // regions is missing - // FIXME: investigate the impact of loans traversing applied member constraints and why some - // tests fail otherwise. - // - // We first handle the cases where the loan doesn't go out of scope, depending on the - // issuing region's successors. - for successor in graph::depth_first_search(&self.regioncx.region_graph(), issuing_region) { - // Via applied member constraints - // - // The issuing region can flow into the choice regions, and they are either: - // - placeholders or free regions themselves, - // - or also transitively outlive a free region. - // - // That is to say, if there are applied member constraints here, the loan escapes the - // function and cannot go out of scope. We could early return here. - // - // For additional insurance via fuzzing and crater, we verify that the constraint's min - // choice indeed escapes the function. In the future, we could e.g. turn this check into - // a debug assert and early return as an optimization. - let scc = sccs.scc(successor); - for constraint in self.regioncx.applied_member_constraints(scc) { - if universal_regions.is_universal_region(constraint.min_choice) { - return; - } - } - } - + fn precompute_loans_out_of_scope(&mut self, loan_idx: BorrowIndex, loan_issued_at: Location) { let first_block = loan_issued_at.block; let first_bb_data = &self.body.basic_blocks[first_block]; @@ -573,7 +533,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { } fn apply_early_statement_effect( - &mut self, + &self, state: &mut Self::Domain, _statement: &mir::Statement<'tcx>, location: Location, @@ -582,7 +542,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { } fn apply_primary_statement_effect( - &mut self, + &self, state: &mut Self::Domain, stmt: &mir::Statement<'tcx>, location: Location, @@ -617,7 +577,6 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { mir::StatementKind::FakeRead(..) | mir::StatementKind::SetDiscriminant { .. } - | mir::StatementKind::Deinit(..) | mir::StatementKind::StorageLive(..) | mir::StatementKind::Retag { .. } | mir::StatementKind::PlaceMention(..) @@ -631,7 +590,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { } fn apply_early_terminator_effect( - &mut self, + &self, state: &mut Self::Domain, _terminator: &mir::Terminator<'tcx>, location: Location, @@ -640,7 +599,7 @@ impl<'tcx> rustc_mir_dataflow::Analysis<'tcx> for Borrows<'_, 'tcx> { } fn apply_primary_terminator_effect<'mir>( - &mut self, + &self, state: &mut Self::Domain, terminator: &'mir mir::Terminator<'tcx>, _location: Location, diff --git a/compiler/rustc_borrowck/src/def_use.rs b/compiler/rustc_borrowck/src/def_use.rs index b9ced81c46c1..502265a83523 100644 --- a/compiler/rustc_borrowck/src/def_use.rs +++ b/compiler/rustc_borrowck/src/def_use.rs @@ -80,7 +80,7 @@ pub(crate) fn categorize(context: PlaceContext) -> Option { // Backwards incompatible drop hint is not a use, just a marker for linting. PlaceContext::NonUse(NonUseContext::BackwardIncompatibleDropHint) => None, - PlaceContext::MutatingUse(MutatingUseContext::Deinit | MutatingUseContext::SetDiscriminant) => { + PlaceContext::MutatingUse(MutatingUseContext::SetDiscriminant) => { bug!("These statements are not allowed in this MIR phase") } } diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs index 0de4bd67f0ce..15b2a5ef2e21 100644 --- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs @@ -3,7 +3,7 @@ use std::rc::Rc; use rustc_errors::Diag; use rustc_hir::def_id::LocalDefId; -use rustc_infer::infer::region_constraints::{Constraint, RegionConstraintData}; +use rustc_infer::infer::region_constraints::{Constraint, ConstraintKind, RegionConstraintData}; use rustc_infer::infer::{ InferCtxt, RegionResolutionError, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt as _, }; @@ -277,7 +277,7 @@ where // `QueryNormalizeExt::query_normalize` used in the query and `normalize` called below: // the former fails to normalize the `nll/relate_tys/impl-fn-ignore-binder-via-bottom.rs` // test. Check after #85499 lands to see if its fixes have erased this difference. - let (param_env, value) = key.into_parts(); + let ty::ParamEnvAnd { param_env, value } = key; let _ = ocx.normalize(&cause, param_env, value.value); let diag = try_extract_error_from_fulfill_cx( @@ -324,7 +324,7 @@ where mbcx.infcx.tcx.infer_ctxt().build_with_canonical(cause.span, &self.canonical_query); let ocx = ObligationCtxt::new(&infcx); - let (param_env, value) = key.into_parts(); + let ty::ParamEnvAnd { param_env, value } = key; let _ = ocx.deeply_normalize(&cause, param_env, value.value); let diag = try_extract_error_from_fulfill_cx( @@ -420,7 +420,7 @@ fn try_extract_error_from_fulfill_cx<'a, 'tcx>( // We generally shouldn't have errors here because the query was // already run, but there's no point using `span_delayed_bug` // when we're going to emit an error here anyway. - let _errors = ocx.select_all_or_error(); + let _errors = ocx.evaluate_obligations_error_on_ambiguity(); let region_constraints = ocx.infcx.with_region_constraints(|r| r.clone()); try_extract_error_from_region_constraints( ocx.infcx, @@ -454,25 +454,24 @@ fn try_extract_error_from_region_constraints<'a, 'tcx>( (RePlaceholder(a_p), RePlaceholder(b_p)) => a_p.bound == b_p.bound, _ => a_region == b_region, }; - let mut check = - |constraint: &Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| match *constraint { - Constraint::RegSubReg(sub, sup) - if ((exact && sup == placeholder_region) - || (!exact && regions_the_same(sup, placeholder_region))) - && sup != sub => - { - Some((sub, cause.clone())) - } - Constraint::VarSubReg(vid, sup) - if (exact - && sup == placeholder_region - && !universe_of_region(vid).can_name(placeholder_universe)) - || (!exact && regions_the_same(sup, placeholder_region)) => - { - Some((ty::Region::new_var(infcx.tcx, vid), cause.clone())) - } - _ => None, - }; + let mut check = |c: &Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| match c.kind { + ConstraintKind::RegSubReg + if ((exact && c.sup == placeholder_region) + || (!exact && regions_the_same(c.sup, placeholder_region))) + && c.sup != c.sub => + { + Some((c.sub, cause.clone())) + } + ConstraintKind::VarSubReg + if (exact + && c.sup == placeholder_region + && !universe_of_region(c.sub.as_var()).can_name(placeholder_universe)) + || (!exact && regions_the_same(c.sup, placeholder_region)) => + { + Some((c.sub, cause.clone())) + } + _ => None, + }; let mut find_culprit = |exact_match: bool| { region_constraints diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs index be8b3f0bc1e3..f84ed64e036e 100644 --- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs @@ -410,18 +410,18 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } let typeck = self.infcx.tcx.typeck(self.mir_def_id()); let parent = self.infcx.tcx.parent_hir_node(expr.hir_id); - let (def_id, call_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent + let (def_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent && let hir::ExprKind::MethodCall(_, _, args, _) = parent_expr.kind { let def_id = typeck.type_dependent_def_id(parent_expr.hir_id); - (def_id, Some(parent_expr.hir_id), args, 1) + (def_id, args, 1) } else if let hir::Node::Expr(parent_expr) = parent && let hir::ExprKind::Call(call, args) = parent_expr.kind && let ty::FnDef(def_id, _) = typeck.node_type(call.hir_id).kind() { - (Some(*def_id), Some(call.hir_id), args, 0) + (Some(*def_id), args, 0) } else { - (None, None, &[][..], 0) + (None, &[][..], 0) }; let ty = place.ty(self.body, self.infcx.tcx).ty; @@ -459,11 +459,12 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { // If the moved place is used generically by the callee and a reference to it // would still satisfy any bounds on its type, suggest borrowing. if let Some(¶m) = arg_param - && let Some(generic_args) = call_id.and_then(|id| typeck.node_args_opt(id)) + && let hir::Node::Expr(call_expr) = parent && let Some(ref_mutability) = self.suggest_borrow_generic_arg( err, + typeck, + call_expr, def_id, - generic_args, param, moved_place, pos + offset, @@ -502,8 +503,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } if let hir::Node::Expr(parent_expr) = parent && let hir::ExprKind::Call(call_expr, _) = parent_expr.kind - && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IntoIterIntoIter, _)) = - call_expr.kind + && let hir::ExprKind::Path(qpath) = call_expr.kind + && tcx.qpath_is_lang_item(qpath, LangItem::IntoIterIntoIter) { // Do not suggest `.clone()` in a `for` loop, we already suggest borrowing. } else if let UseSpans::FnSelfUse { kind: CallKind::Normal { .. }, .. } = move_spans @@ -627,8 +628,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { fn suggest_borrow_generic_arg( &self, err: &mut Diag<'_>, + typeck: &ty::TypeckResults<'tcx>, + call_expr: &hir::Expr<'tcx>, callee_did: DefId, - generic_args: ty::GenericArgsRef<'tcx>, param: ty::ParamTy, moved_place: PlaceRef<'tcx>, moved_arg_pos: usize, @@ -639,6 +641,19 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let sig = tcx.fn_sig(callee_did).instantiate_identity().skip_binder(); let clauses = tcx.predicates_of(callee_did); + let generic_args = match call_expr.kind { + // For method calls, generic arguments are attached to the call node. + hir::ExprKind::MethodCall(..) => typeck.node_args_opt(call_expr.hir_id)?, + // For normal calls, generic arguments are in the callee's type. + // This diagnostic is only run for `FnDef` callees. + hir::ExprKind::Call(callee, _) + if let &ty::FnDef(_, args) = typeck.node_type(callee.hir_id).kind() => + { + args + } + _ => return None, + }; + // First, is there at least one method on one of `param`'s trait bounds? // This keeps us from suggesting borrowing the argument to `mem::drop`, e.g. if !clauses.instantiate_identity(tcx).predicates.iter().any(|clause| { @@ -1298,7 +1313,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let ocx = ObligationCtxt::new_with_diagnostics(self.infcx); let cause = ObligationCause::misc(expr.span, self.mir_def_id()); ocx.register_bound(cause, self.infcx.param_env, ty, clone_trait); - let errors = ocx.select_all_or_error(); + let errors = ocx.evaluate_obligations_error_on_ambiguity(); if errors.iter().all(|error| { match error.obligation.predicate.as_clause().and_then(|c| c.as_trait_clause()) { Some(clause) => match clause.self_ty().skip_binder().kind() { @@ -1482,7 +1497,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let cause = ObligationCause::misc(span, self.mir_def_id()); ocx.register_bound(cause, self.infcx.param_env, ty, def_id); - let errors = ocx.select_all_or_error(); + let errors = ocx.evaluate_obligations_error_on_ambiguity(); // Only emit suggestion if all required predicates are on generic let predicates: Result, _> = errors @@ -2297,6 +2312,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let typeck_results = tcx.typeck(self.mir_def_id()); struct ExprFinder<'hir> { + tcx: TyCtxt<'hir>, issue_span: Span, expr_span: Span, body_expr: Option<&'hir hir::Expr<'hir>>, @@ -2321,9 +2337,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { // }; // corresponding to the desugaring of a for loop `for in { }`. if let hir::ExprKind::Call(path, [arg]) = ex.kind - && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IntoIterIntoIter, _)) = - path.kind + && let hir::ExprKind::Path(qpath) = path.kind + && self.tcx.qpath_is_lang_item(qpath, LangItem::IntoIterIntoIter) && arg.span.contains(self.issue_span) + && ex.span.desugaring_kind() == Some(DesugaringKind::ForLoop) { // Find `IntoIterator::into_iter()` self.head = Some(arg); @@ -2340,10 +2357,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { .. }) = stmt.kind && let hir::ExprKind::Call(path, _args) = call.kind - && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IteratorNext, _)) = - path.kind - && let hir::PatKind::Struct(path, [field, ..], _) = bind.pat.kind - && let hir::QPath::LangItem(LangItem::OptionSome, pat_span) = path + && let hir::ExprKind::Path(qpath) = path.kind + && self.tcx.qpath_is_lang_item(qpath, LangItem::IteratorNext) + && let hir::PatKind::Struct(qpath, [field, ..], _) = bind.pat.kind + && self.tcx.qpath_is_lang_item(qpath, LangItem::OptionSome) && call.span.contains(self.issue_span) { // Find `` and the span for the whole `for` loop. @@ -2355,7 +2372,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { self.loop_bind = Some(ident); } self.head_span = Some(*head_span); - self.pat_span = Some(pat_span); + self.pat_span = Some(bind.pat.span); self.loop_span = Some(stmt.span); } @@ -2370,6 +2387,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } let mut finder = ExprFinder { + tcx, expr_span: span, issue_span, loop_bind: None, @@ -2977,6 +2995,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { self.buffer_error(err); } + #[tracing::instrument(level = "debug", skip(self, explanation))] fn report_local_value_does_not_live_long_enough( &self, location: Location, @@ -2986,13 +3005,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { borrow_spans: UseSpans<'tcx>, explanation: BorrowExplanation<'tcx>, ) -> Diag<'infcx> { - debug!( - "report_local_value_does_not_live_long_enough(\ - {:?}, {:?}, {:?}, {:?}, {:?}\ - )", - location, name, borrow, drop_span, borrow_spans - ); - let borrow_span = borrow_spans.var_or_use_path_span(); if let BorrowExplanation::MustBeValidFor { category, @@ -3959,7 +3971,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } ProjectionElem::ConstantIndex { .. } | ProjectionElem::Subslice { .. } - | ProjectionElem::Subtype(_) | ProjectionElem::Index(_) | ProjectionElem::UnwrapUnsafeBinder(_) => kind, }, diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs index fdca6b565409..6bcfa9d8bf9a 100644 --- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs +++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs @@ -416,6 +416,26 @@ impl<'tcx> BorrowExplanation<'tcx> { { self.add_object_lifetime_default_note(tcx, err, unsize_ty); } + + let mut preds = path + .iter() + .filter_map(|constraint| match constraint.category { + ConstraintCategory::Predicate(pred) if !pred.is_dummy() => Some(pred), + _ => None, + }) + .collect::>(); + preds.sort(); + preds.dedup(); + if !preds.is_empty() { + let s = if preds.len() == 1 { "" } else { "s" }; + err.span_note( + preds, + format!( + "requirement{s} that the value outlives `{region_name}` introduced here" + ), + ); + } + self.add_lifetime_bound_suggestion_to_diagnostic(err, &category, span, region_name); } _ => {} @@ -438,7 +458,7 @@ impl<'tcx> BorrowExplanation<'tcx> { let elaborated_args = std::iter::zip(*args, &generics.own_params).map(|(arg, param)| { - if let Some(ty::Dynamic(obj, _, ty::Dyn)) = arg.as_type().map(Ty::kind) { + if let Some(ty::Dynamic(obj, _)) = arg.as_type().map(Ty::kind) { let default = tcx.object_lifetime_default(param.def_id); let re_static = tcx.lifetimes.re_static; @@ -464,7 +484,7 @@ impl<'tcx> BorrowExplanation<'tcx> { has_dyn = true; - Ty::new_dynamic(tcx, obj, implied_region, ty::Dyn).into() + Ty::new_dynamic(tcx, obj, implied_region).into() } else { arg } @@ -565,7 +585,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { let (blame_constraint, path) = self.regioncx.best_blame_constraint( borrow_region, NllRegionVariableOrigin::FreeRegion, - |r| self.regioncx.provides_universal_region(r, borrow_region, outlived_region), + outlived_region, ); let BlameConstraint { category, from_closure, cause, .. } = blame_constraint; @@ -667,7 +687,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { } } - Some(Cause::DropVar(local, location)) => { + Some(Cause::DropVar(local, location)) if !is_local_boring(local) => { let mut should_note_order = false; if self.local_name(local).is_some() && let Some((WriteKind::StorageDeadOrDrop, place)) = kind_place @@ -685,7 +705,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { } } - Some(Cause::LiveVar(..)) | None => { + Some(Cause::LiveVar(..) | Cause::DropVar(..)) | None => { // Here, under NLL: no cause was found. Under polonius: no cause was found, or a // boring local was found, which we ignore like NLLs do to match its diagnostics. if let Some(region) = self.to_error_region_vid(borrow_region_vid) { diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs index 56fdaf1c724a..e13c1c712d8d 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mod.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs @@ -6,7 +6,9 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxIndexMap; use rustc_errors::{Applicability, Diag, EmissionGuarantee, MultiSpan, listify}; use rustc_hir::def::{CtorKind, Namespace}; -use rustc_hir::{self as hir, CoroutineKind, LangItem}; +use rustc_hir::{ + self as hir, CoroutineKind, GenericBound, LangItem, WhereBoundPredicate, WherePredicateKind, +}; use rustc_index::{IndexSlice, IndexVec}; use rustc_infer::infer::{BoundRegionConversionTime, NllRegionVariableOrigin}; use rustc_infer::traits::SelectionError; @@ -51,7 +53,7 @@ mod conflict_errors; mod explain_borrow; mod move_errors; mod mutability_errors; -mod opaque_suggestions; +mod opaque_types; mod region_errors; pub(crate) use bound_region_errors::{ToUniverseInfo, UniverseInfo}; @@ -400,7 +402,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { ProjectionElem::Downcast(..) if opt.including_downcast => return None, ProjectionElem::Downcast(..) => (), ProjectionElem::OpaqueCast(..) => (), - ProjectionElem::Subtype(..) => (), ProjectionElem::UnwrapUnsafeBinder(_) => (), ProjectionElem::Field(field, _ty) => { // FIXME(project-rfc_2229#36): print capture precisely here. @@ -482,9 +483,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { PlaceRef { local, projection: proj_base }.ty(self.body, self.infcx.tcx) } ProjectionElem::Downcast(..) => place.ty(self.body, self.infcx.tcx), - ProjectionElem::Subtype(ty) - | ProjectionElem::OpaqueCast(ty) - | ProjectionElem::UnwrapUnsafeBinder(ty) => PlaceTy::from_ty(*ty), + ProjectionElem::OpaqueCast(ty) | ProjectionElem::UnwrapUnsafeBinder(ty) => { + PlaceTy::from_ty(*ty) + } ProjectionElem::Field(_, field_type) => PlaceTy::from_ty(*field_type), }, }; @@ -613,7 +614,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { /// Return the name of the provided `Ty` (that must be a reference) with a synthesized lifetime /// name where required. pub(super) fn get_name_for_ty(&self, ty: Ty<'tcx>, counter: usize) -> String { - let mut printer = ty::print::FmtPrinter::new(self.infcx.tcx, Namespace::TypeNS); + let mut p = ty::print::FmtPrinter::new(self.infcx.tcx, Namespace::TypeNS); // We need to add synthesized lifetimes where appropriate. We do // this by hooking into the pretty printer and telling it to label the @@ -624,19 +625,19 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { | ty::RePlaceholder(ty::PlaceholderRegion { bound: ty::BoundRegion { kind: br, .. }, .. - }) => printer.region_highlight_mode.highlighting_bound_region(br, counter), + }) => p.region_highlight_mode.highlighting_bound_region(br, counter), _ => {} } } - ty.print(&mut printer).unwrap(); - printer.into_buffer() + ty.print(&mut p).unwrap(); + p.into_buffer() } /// Returns the name of the provided `Ty` (that must be a reference)'s region with a /// synthesized lifetime name where required. pub(super) fn get_region_name_for_ty(&self, ty: Ty<'tcx>, counter: usize) -> String { - let mut printer = ty::print::FmtPrinter::new(self.infcx.tcx, Namespace::TypeNS); + let mut p = ty::print::FmtPrinter::new(self.infcx.tcx, Namespace::TypeNS); let region = if let ty::Ref(region, ..) = ty.kind() { match region.kind() { @@ -644,7 +645,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { | ty::RePlaceholder(ty::PlaceholderRegion { bound: ty::BoundRegion { kind: br, .. }, .. - }) => printer.region_highlight_mode.highlighting_bound_region(br, counter), + }) => p.region_highlight_mode.highlighting_bound_region(br, counter), _ => {} } region @@ -652,31 +653,72 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { bug!("ty for annotation of borrow region is not a reference"); }; - region.print(&mut printer).unwrap(); - printer.into_buffer() + region.print(&mut p).unwrap(); + p.into_buffer() } /// Add a note to region errors and borrow explanations when higher-ranked regions in predicates /// implicitly introduce an "outlives `'static`" constraint. + /// + /// This is very similar to `fn suggest_static_lifetime_for_gat_from_hrtb` which handles this + /// note for failed type tests instead of outlives errors. fn add_placeholder_from_predicate_note( &self, - err: &mut Diag<'_, G>, + diag: &mut Diag<'_, G>, path: &[OutlivesConstraint<'tcx>], ) { - let predicate_span = path.iter().find_map(|constraint| { + let tcx = self.infcx.tcx; + let Some((gat_hir_id, generics)) = path.iter().find_map(|constraint| { let outlived = constraint.sub; if let Some(origin) = self.regioncx.definitions.get(outlived) - && let NllRegionVariableOrigin::Placeholder(_) = origin.origin - && let ConstraintCategory::Predicate(span) = constraint.category + && let NllRegionVariableOrigin::Placeholder(placeholder) = origin.origin + && let Some(id) = placeholder.bound.kind.get_id() + && let Some(placeholder_id) = id.as_local() + && let gat_hir_id = tcx.local_def_id_to_hir_id(placeholder_id) + && let Some(generics_impl) = + tcx.parent_hir_node(tcx.parent_hir_id(gat_hir_id)).generics() { - Some(span) + Some((gat_hir_id, generics_impl)) } else { None } - }); + }) else { + return; + }; - if let Some(span) = predicate_span { - err.span_note(span, "due to current limitations in the borrow checker, this implies a `'static` lifetime"); + // Look for the where-bound which introduces the placeholder. + // As we're using the HIR, we need to handle both `for<'a> T: Trait<'a>` + // and `T: for<'a> Trait`<'a>. + for pred in generics.predicates { + let WherePredicateKind::BoundPredicate(WhereBoundPredicate { + bound_generic_params, + bounds, + .. + }) = pred.kind + else { + continue; + }; + if bound_generic_params + .iter() + .rfind(|bgp| tcx.local_def_id_to_hir_id(bgp.def_id) == gat_hir_id) + .is_some() + { + diag.span_note(pred.span, fluent::borrowck_limitations_implies_static); + return; + } + for bound in bounds.iter() { + if let GenericBound::Trait(bound) = bound { + if bound + .bound_generic_params + .iter() + .rfind(|bgp| tcx.local_def_id_to_hir_id(bgp.def_id) == gat_hir_id) + .is_some() + { + diag.span_note(bound.span, fluent::borrowck_limitations_implies_static); + return; + } + } + } } } diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs index 1067f1e40ef6..be2a5b1ebe22 100644 --- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs @@ -9,7 +9,8 @@ use rustc_middle::bug; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::move_paths::{LookupResult, MovePathIndex}; -use rustc_span::{BytePos, ExpnKind, MacroKind, Span}; +use rustc_span::def_id::DefId; +use rustc_span::{BytePos, DUMMY_SP, ExpnKind, MacroKind, Span}; use rustc_trait_selection::error_reporting::traits::FindExprBySpan; use rustc_trait_selection::infer::InferCtxtExt; use tracing::debug; @@ -138,9 +139,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { // whether or not the right-hand side is a place expression if let LocalInfo::User(BindingForm::Var(VarBindingForm { opt_match_place: Some((opt_match_place, match_span)), - binding_mode: _, - opt_ty_info: _, - pat_span: _, + .. })) = *local_decl.local_info() { let stmt_source_info = self.body.source_info(location); @@ -507,12 +506,18 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { ); let closure_span = tcx.def_span(def_id); + self.cannot_move_out_of(span, &place_description) .with_span_label(upvar_span, "captured outer variable") .with_span_label( closure_span, format!("captured by this `{closure_kind}` closure"), ) + .with_span_help( + self.get_closure_bound_clause_span(*def_id), + "`Fn` and `FnMut` closures require captured values to be able to be \ + consumed multiple times, but `FnOnce` closures may consume them only once", + ) } _ => { let source = self.borrowed_content_source(deref_base); @@ -561,6 +566,47 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { err } + fn get_closure_bound_clause_span(&self, def_id: DefId) -> Span { + let tcx = self.infcx.tcx; + let typeck_result = tcx.typeck(self.mir_def_id()); + // Check whether the closure is an argument to a call, if so, + // get the instantiated where-bounds of that call. + let closure_hir_id = tcx.local_def_id_to_hir_id(def_id.expect_local()); + let hir::Node::Expr(parent) = tcx.parent_hir_node(closure_hir_id) else { return DUMMY_SP }; + + let predicates = match parent.kind { + hir::ExprKind::Call(callee, _) => { + let Some(ty) = typeck_result.node_type_opt(callee.hir_id) else { return DUMMY_SP }; + let ty::FnDef(fn_def_id, args) = ty.kind() else { return DUMMY_SP }; + tcx.predicates_of(fn_def_id).instantiate(tcx, args) + } + hir::ExprKind::MethodCall(..) => { + let Some((_, method)) = typeck_result.type_dependent_def(parent.hir_id) else { + return DUMMY_SP; + }; + let args = typeck_result.node_args(parent.hir_id); + tcx.predicates_of(method).instantiate(tcx, args) + } + _ => return DUMMY_SP, + }; + + // Check whether one of the where-bounds requires the closure to impl `Fn[Mut]`. + for (pred, span) in predicates.predicates.iter().zip(predicates.spans.iter()) { + if let Some(clause) = pred.as_trait_clause() + && let ty::Closure(clause_closure_def_id, _) = clause.self_ty().skip_binder().kind() + && *clause_closure_def_id == def_id + && (tcx.lang_items().fn_mut_trait() == Some(clause.def_id()) + || tcx.lang_items().fn_trait() == Some(clause.def_id())) + { + // Found `` + // We point at the `Fn()` or `FnMut()` bound that coerced the closure, which + // could be changed to `FnOnce()` to avoid the move error. + return *span; + } + } + DUMMY_SP + } + fn add_move_hints(&self, error: GroupedMoveError<'tcx>, err: &mut Diag<'_>, span: Span) { match error { GroupedMoveError::MovesFromPlace { mut binds_to, move_from, .. } => { diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs index 5d9416b59fce..e23c7eebeca1 100644 --- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs @@ -3,6 +3,7 @@ use core::ops::ControlFlow; +use either::Either; use hir::{ExprKind, Param}; use rustc_abi::FieldIdx; use rustc_errors::{Applicability, Diag}; @@ -12,15 +13,16 @@ use rustc_middle::bug; use rustc_middle::hir::place::PlaceBase; use rustc_middle::mir::visit::PlaceContext; use rustc_middle::mir::{ - self, BindingForm, Local, LocalDecl, LocalInfo, LocalKind, Location, Mutability, Place, - PlaceRef, ProjectionElem, + self, BindingForm, Body, BorrowKind, Local, LocalDecl, LocalInfo, LocalKind, Location, + Mutability, Operand, Place, PlaceRef, ProjectionElem, RawPtrKind, Rvalue, Statement, + StatementKind, TerminatorKind, }; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, Upcast}; use rustc_span::{BytePos, DesugaringKind, Span, Symbol, kw, sym}; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::InferCtxtExt; use rustc_trait_selection::traits; -use tracing::debug; +use tracing::{debug, trace}; use crate::diagnostics::BorrowedContentSource; use crate::{MirBorrowckCtxt, session_diagnostics}; @@ -31,6 +33,33 @@ pub(crate) enum AccessKind { Mutate, } +/// Finds all statements that assign directly to local (i.e., X = ...) and returns their +/// locations. +fn find_assignments(body: &Body<'_>, local: Local) -> Vec { + use rustc_middle::mir::visit::Visitor; + + struct FindLocalAssignmentVisitor { + needle: Local, + locations: Vec, + } + + impl<'tcx> Visitor<'tcx> for FindLocalAssignmentVisitor { + fn visit_local(&mut self, local: Local, place_context: PlaceContext, location: Location) { + if self.needle != local { + return; + } + + if place_context.is_place_assignment() { + self.locations.push(location); + } + } + } + + let mut visitor = FindLocalAssignmentVisitor { needle: local, locations: vec![] }; + visitor.visit_body(body); + visitor.locations +} + impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { pub(crate) fn report_mutability_error( &mut self, @@ -163,7 +192,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { [ .., ProjectionElem::Index(_) - | ProjectionElem::Subtype(_) | ProjectionElem::ConstantIndex { .. } | ProjectionElem::OpaqueCast { .. } | ProjectionElem::Subslice { .. } @@ -307,8 +335,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { LocalInfo::User(BindingForm::Var(mir::VarBindingForm { binding_mode: BindingMode(ByRef::No, Mutability::Not), opt_ty_info: Some(sp), - opt_match_place: _, - pat_span: _, + .. })) => { if suggest { err.span_note(sp, "the binding is already a mutable borrow"); @@ -384,7 +411,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } - // Also suggest adding mut for upvars + // Also suggest adding mut for upvars. PlaceRef { local, projection: [proj_base @ .., ProjectionElem::Field(upvar_index, _)], @@ -438,9 +465,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } - // complete hack to approximate old AST-borrowck - // diagnostic: if the span starts with a mutable borrow of - // a local variable, then just suggest the user remove it. + // Complete hack to approximate old AST-borrowck diagnostic: if the span starts + // with a mutable borrow of a local variable, then just suggest the user remove it. PlaceRef { local: _, projection: [] } if self .infcx @@ -677,17 +703,17 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { /// - is the trait from the local crate? If not, we can't suggest changing signatures /// - `Span` of the argument in the trait definition fn is_error_in_trait(&self, local: Local) -> (bool, bool, Option) { + let tcx = self.infcx.tcx; if self.body.local_kind(local) != LocalKind::Arg { return (false, false, None); } let my_def = self.body.source.def_id(); - let Some(td) = - self.infcx.tcx.impl_of_assoc(my_def).and_then(|x| self.infcx.tcx.trait_id_of_impl(x)) + let Some(td) = tcx.trait_impl_of_assoc(my_def).map(|id| self.infcx.tcx.impl_trait_id(id)) else { return (false, false, None); }; - let implemented_trait_item = self.infcx.tcx.associated_item(my_def).trait_item_def_id; + let implemented_trait_item = self.infcx.tcx.trait_item_of(my_def); ( true, @@ -722,6 +748,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { opt_ty_info: _, opt_match_place: _, pat_span, + introductions: _, })) => pat_span, _ => local_decl.source_info.span, }; @@ -768,7 +795,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { ); } - // point to span of upvar making closure call require mutable borrow + // Point to span of upvar making closure call that requires a mutable borrow fn show_mutating_upvar( &self, tcx: TyCtxt<'_>, @@ -824,7 +851,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } else { bug!("not an upvar") }; - // sometimes we deliberately don't store the name of a place when coming from a macro in + // Sometimes we deliberately don't store the name of a place when coming from a macro in // another crate. We generally want to limit those diagnostics a little, to hide // implementation details (such as those from pin!() or format!()). In that case show a // slightly different error message, or none at all if something else happened. In other @@ -935,8 +962,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let def_id = tcx.hir_enclosing_body_owner(fn_call_id); let mut look_at_return = true; - // If the HIR node is a function or method call gets the def ID - // of the called function or method and the span and args of the call expr + // If the HIR node is a function or method call, get the DefId + // of the callee function or method, the span, and args of the call expr let get_call_details = || { let hir::Node::Expr(hir::Expr { hir_id, kind, .. }) = node else { return None; @@ -1050,7 +1077,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let mut cur_expr = expr; while let ExprKind::MethodCall(path_segment, recv, _, _) = cur_expr.kind { if path_segment.ident.name == sym::iter { - // check `_ty` has `iter_mut` method + // Check that the type has an `iter_mut` method. let res = self .infcx .tcx @@ -1080,38 +1107,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } - /// Finds all statements that assign directly to local (i.e., X = ...) and returns their - /// locations. - fn find_assignments(&self, local: Local) -> Vec { - use rustc_middle::mir::visit::Visitor; - - struct FindLocalAssignmentVisitor { - needle: Local, - locations: Vec, - } - - impl<'tcx> Visitor<'tcx> for FindLocalAssignmentVisitor { - fn visit_local( - &mut self, - local: Local, - place_context: PlaceContext, - location: Location, - ) { - if self.needle != local { - return; - } - - if place_context.is_place_assignment() { - self.locations.push(location); - } - } - } - - let mut visitor = FindLocalAssignmentVisitor { needle: local, locations: vec![] }; - visitor.visit_body(self.body); - visitor.locations - } - fn suggest_make_local_mut(&self, err: &mut Diag<'_>, local: Local, name: Symbol) { let local_decl = &self.body.local_decls[local]; @@ -1121,7 +1116,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let (is_trait_sig, is_local, local_trait) = self.is_error_in_trait(local); if is_trait_sig && !is_local { - // Do not suggest to change the signature when the trait comes from another crate. + // Do not suggest changing the signature when the trait comes from another crate. err.span_label( local_decl.source_info.span, format!("this is an immutable {pointer_desc}"), @@ -1130,11 +1125,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } let decl_span = local_decl.source_info.span; - let amp_mut_sugg = match *local_decl.local_info() { + let (amp_mut_sugg, local_var_ty_info) = match *local_decl.local_info() { LocalInfo::User(mir::BindingForm::ImplicitSelf(_)) => { let (span, suggestion) = suggest_ampmut_self(self.infcx.tcx, decl_span); let additional = local_trait.map(|span| suggest_ampmut_self(self.infcx.tcx, span)); - Some(AmpMutSugg { has_sugg: true, span, suggestion, additional }) + (AmpMutSugg::Type { span, suggestion, additional }, None) } LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm { @@ -1142,261 +1137,293 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { opt_ty_info, .. })) => { - // check if the RHS is from desugaring + // Check if the RHS is from desugaring. + let first_assignment = find_assignments(&self.body, local).first().copied(); + let first_assignment_stmt = first_assignment + .and_then(|loc| self.body[loc.block].statements.get(loc.statement_index)); + trace!(?first_assignment_stmt); let opt_assignment_rhs_span = - self.find_assignments(local).first().map(|&location| { - if let Some(mir::Statement { - source_info: _, - kind: - mir::StatementKind::Assign(box ( - _, - mir::Rvalue::Use(mir::Operand::Copy(place)), - )), - .. - }) = self.body[location.block].statements.get(location.statement_index) - { - self.body.local_decls[place.local].source_info.span - } else { - self.body.source_info(location).span - } - }); - match opt_assignment_rhs_span.and_then(|s| s.desugaring_kind()) { - // on for loops, RHS points to the iterator part - Some(DesugaringKind::ForLoop) => { - let span = opt_assignment_rhs_span.unwrap(); - self.suggest_similar_mut_method_for_for_loop(err, span); + first_assignment.map(|loc| self.body.source_info(loc).span); + let mut source_span = opt_assignment_rhs_span; + if let Some(mir::Statement { + source_info: _, + kind: + mir::StatementKind::Assign(box (_, mir::Rvalue::Use(mir::Operand::Copy(place)))), + .. + }) = first_assignment_stmt + { + let local_span = self.body.local_decls[place.local].source_info.span; + // `&self` in async functions have a `desugaring_kind`, but the local we assign + // it with does not, so use the local_span for our checks later. + source_span = Some(local_span); + if let Some(DesugaringKind::ForLoop) = local_span.desugaring_kind() { + // On for loops, RHS points to the iterator part. + self.suggest_similar_mut_method_for_for_loop(err, local_span); err.span_label( - span, + local_span, format!("this iterator yields `{pointer_sigil}` {pointer_desc}s",), ); - None - } - // don't create labels for compiler-generated spans - Some(_) => None, - // don't create labels for the span not from user's code - None if opt_assignment_rhs_span - .is_some_and(|span| self.infcx.tcx.sess.source_map().is_imported(span)) => - { - None - } - None => { - if name != kw::SelfLower { - suggest_ampmut( - self.infcx.tcx, - local_decl.ty, - decl_span, - opt_assignment_rhs_span, - opt_ty_info, - ) - } else { - match local_decl.local_info() { - LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm { - opt_ty_info: None, - .. - })) => { - let (span, sugg) = - suggest_ampmut_self(self.infcx.tcx, decl_span); - Some(AmpMutSugg { - has_sugg: true, - span, - suggestion: sugg, - additional: None, - }) - } - // explicit self (eg `self: &'a Self`) - _ => suggest_ampmut( - self.infcx.tcx, - local_decl.ty, - decl_span, - opt_assignment_rhs_span, - opt_ty_info, - ), - } - } + return; } } + + // Don't create labels for compiler-generated spans or spans not from users' code. + if source_span.is_some_and(|s| { + s.desugaring_kind().is_some() || self.infcx.tcx.sess.source_map().is_imported(s) + }) { + return; + } + + // This could be because we're in an `async fn`. + if name == kw::SelfLower && opt_ty_info.is_none() { + let (span, suggestion) = suggest_ampmut_self(self.infcx.tcx, decl_span); + (AmpMutSugg::Type { span, suggestion, additional: None }, None) + } else if let Some(sugg) = + suggest_ampmut(self.infcx, self.body(), first_assignment_stmt) + { + (sugg, opt_ty_info) + } else { + return; + } } LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm { - binding_mode: BindingMode(ByRef::Yes(_), _), + binding_mode: BindingMode(ByRef::Yes(..), _), .. })) => { let pattern_span: Span = local_decl.source_info.span; - suggest_ref_mut(self.infcx.tcx, pattern_span).map(|span| AmpMutSugg { - has_sugg: true, - span, - suggestion: "mut ".to_owned(), - additional: None, - }) + let Some(span) = suggest_ref_mut(self.infcx.tcx, pattern_span) else { + return; + }; + (AmpMutSugg::Type { span, suggestion: "mut ".to_owned(), additional: None }, None) } _ => unreachable!(), }; - match amp_mut_sugg { - Some(AmpMutSugg { - has_sugg: true, - span: err_help_span, - suggestion: suggested_code, - additional, - }) => { - let mut sugg = vec![(err_help_span, suggested_code)]; - if let Some(s) = additional { - sugg.push(s); - } + let mut suggest = |suggs: Vec<_>, applicability, extra| { + if suggs.iter().any(|(span, _)| self.infcx.tcx.sess.source_map().is_imported(*span)) { + return; + } - if sugg.iter().all(|(span, _)| !self.infcx.tcx.sess.source_map().is_imported(*span)) - { - err.multipart_suggestion_verbose( - format!( - "consider changing this to be a mutable {pointer_desc}{}", - if is_trait_sig { - " in the `impl` method and the `trait` definition" - } else { - "" - } - ), - sugg, - Applicability::MachineApplicable, + err.multipart_suggestion_verbose( + format!( + "consider changing this to be a mutable {pointer_desc}{}{extra}", + if is_trait_sig { + " in the `impl` method and the `trait` definition" + } else { + "" + } + ), + suggs, + applicability, + ); + }; + + let (mut sugg, add_type_annotation_if_not_exists) = match amp_mut_sugg { + AmpMutSugg::Type { span, suggestion, additional } => { + let mut sugg = vec![(span, suggestion)]; + sugg.extend(additional); + suggest(sugg, Applicability::MachineApplicable, ""); + return; + } + AmpMutSugg::MapGetMut { span, suggestion } => { + if self.infcx.tcx.sess.source_map().is_imported(span) { + return; + } + err.multipart_suggestion_verbose( + "consider using `get_mut`", + vec![(span, suggestion)], + Applicability::MaybeIncorrect, + ); + return; + } + AmpMutSugg::Expr { span, suggestion } => { + // `Expr` suggestions should change type annotations if they already exist (probably immut), + // but do not add new type annotations. + (vec![(span, suggestion)], false) + } + AmpMutSugg::ChangeBinding => (vec![], true), + }; + + // Find a binding's type to make mutable. + let (binding_exists, span) = match local_var_ty_info { + // If this is a variable binding with an explicit type, + // then we will suggest changing it to be mutable. + // This is `Applicability::MachineApplicable`. + Some(ty_span) => (true, ty_span), + + // Otherwise, we'll suggest *adding* an annotated type, we'll suggest + // the RHS's type for that. + // This is `Applicability::HasPlaceholders`. + None => (false, decl_span), + }; + + if !binding_exists && !add_type_annotation_if_not_exists { + suggest(sugg, Applicability::MachineApplicable, ""); + return; + } + + // If the binding already exists and is a reference with an explicit + // lifetime, then we can suggest adding ` mut`. This is special-cased from + // the path without an explicit lifetime. + let (sugg_span, sugg_str, suggest_now) = if let Ok(src) = self.infcx.tcx.sess.source_map().span_to_snippet(span) + && src.starts_with("&'") + // Note that `&' a T` is invalid so this is correct. + && let Some(ws_pos) = src.find(char::is_whitespace) + { + let span = span.with_lo(span.lo() + BytePos(ws_pos as u32)).shrink_to_lo(); + (span, " mut".to_owned(), true) + // If there is already a binding, we modify it to be `mut`. + } else if binding_exists { + // Shrink the span to just after the `&` in `&variable`. + let span = span.with_lo(span.lo() + BytePos(1)).shrink_to_lo(); + (span, "mut ".to_owned(), true) + } else { + // Otherwise, suggest that the user annotates the binding; We provide the + // type of the local. + let ty = local_decl.ty.builtin_deref(true).unwrap(); + + (span, format!("{}mut {}", if local_decl.ty.is_ref() { "&" } else { "*" }, ty), false) + }; + + if suggest_now { + // Suggest changing `&x` to `&mut x` and changing `&T` to `&mut T` at the same time. + let has_change = !sugg.is_empty(); + sugg.push((sugg_span, sugg_str)); + suggest( + sugg, + Applicability::MachineApplicable, + // FIXME(fee1-dead) this somehow doesn't fire + if has_change { " and changing the binding's type" } else { "" }, + ); + return; + } else if !sugg.is_empty() { + suggest(sugg, Applicability::MachineApplicable, ""); + return; + } + + let def_id = self.body.source.def_id(); + let hir_id = if let Some(local_def_id) = def_id.as_local() + && let Some(body) = self.infcx.tcx.hir_maybe_body_owned_by(local_def_id) + { + BindingFinder { span: sugg_span }.visit_body(&body).break_value() + } else { + None + }; + let node = hir_id.map(|hir_id| self.infcx.tcx.hir_node(hir_id)); + + let Some(hir::Node::LetStmt(local)) = node else { + err.span_label( + sugg_span, + format!("consider changing this binding's type to be: `{sugg_str}`"), + ); + return; + }; + + let tables = self.infcx.tcx.typeck(def_id.as_local().unwrap()); + if let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait() + && let Some(expr) = local.init + && let ty = tables.node_type_opt(expr.hir_id) + && let Some(ty) = ty + && let ty::Ref(..) = ty.kind() + { + match self + .infcx + .type_implements_trait_shallow(clone_trait, ty.peel_refs(), self.infcx.param_env) + .as_deref() + { + Some([]) => { + // FIXME: This error message isn't useful, since we're just + // vaguely suggesting to clone a value that already + // implements `Clone`. + // + // A correct suggestion here would take into account the fact + // that inference may be affected by missing types on bindings, + // etc., to improve "tests/ui/borrowck/issue-91206.stderr", for + // example. + } + None => { + if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) = expr.kind + && segment.ident.name == sym::clone + { + err.span_help( + span, + format!( + "`{}` doesn't implement `Clone`, so this call clones \ + the reference `{ty}`", + ty.peel_refs(), + ), + ); + } + // The type doesn't implement Clone. + let trait_ref = ty::Binder::dummy(ty::TraitRef::new( + self.infcx.tcx, + clone_trait, + [ty.peel_refs()], + )); + let obligation = traits::Obligation::new( + self.infcx.tcx, + traits::ObligationCause::dummy(), + self.infcx.param_env, + trait_ref, + ); + self.infcx.err_ctxt().suggest_derive( + &obligation, + err, + trait_ref.upcast(self.infcx.tcx), ); } - } - Some(AmpMutSugg { - has_sugg: false, span: err_label_span, suggestion: message, .. - }) => { - let def_id = self.body.source.def_id(); - let hir_id = if let Some(local_def_id) = def_id.as_local() - && let Some(body) = self.infcx.tcx.hir_maybe_body_owned_by(local_def_id) - { - BindingFinder { span: err_label_span }.visit_body(&body).break_value() - } else { - None - }; - - if let Some(hir_id) = hir_id - && let hir::Node::LetStmt(local) = self.infcx.tcx.hir_node(hir_id) - { - let tables = self.infcx.tcx.typeck(def_id.as_local().unwrap()); - if let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait() - && let Some(expr) = local.init - && let ty = tables.node_type_opt(expr.hir_id) - && let Some(ty) = ty - && let ty::Ref(..) = ty.kind() + Some(errors) => { + if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) = expr.kind + && segment.ident.name == sym::clone { - match self - .infcx - .type_implements_trait_shallow( - clone_trait, - ty.peel_refs(), - self.infcx.param_env, - ) - .as_deref() - { - Some([]) => { - // FIXME: This error message isn't useful, since we're just - // vaguely suggesting to clone a value that already - // implements `Clone`. - // - // A correct suggestion here would take into account the fact - // that inference may be affected by missing types on bindings, - // etc., to improve "tests/ui/borrowck/issue-91206.stderr", for - // example. - } - None => { - if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) = - expr.kind - && segment.ident.name == sym::clone - { - err.span_help( - span, - format!( - "`{}` doesn't implement `Clone`, so this call clones \ - the reference `{ty}`", - ty.peel_refs(), - ), - ); - } - // The type doesn't implement Clone. - let trait_ref = ty::Binder::dummy(ty::TraitRef::new( - self.infcx.tcx, - clone_trait, - [ty.peel_refs()], - )); - let obligation = traits::Obligation::new( - self.infcx.tcx, - traits::ObligationCause::dummy(), - self.infcx.param_env, - trait_ref, - ); - self.infcx.err_ctxt().suggest_derive( - &obligation, - err, - trait_ref.upcast(self.infcx.tcx), - ); - } - Some(errors) => { - if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) = - expr.kind - && segment.ident.name == sym::clone - { - err.span_help( - span, - format!( - "`{}` doesn't implement `Clone` because its \ + err.span_help( + span, + format!( + "`{}` doesn't implement `Clone` because its \ implementations trait bounds could not be met, so \ this call clones the reference `{ty}`", - ty.peel_refs(), - ), - ); - err.note(format!( - "the following trait bounds weren't met: {}", - errors - .iter() - .map(|e| e.obligation.predicate.to_string()) - .collect::>() - .join("\n"), - )); - } - // The type doesn't implement Clone because of unmet obligations. - for error in errors { - if let traits::FulfillmentErrorCode::Select( - traits::SelectionError::Unimplemented, - ) = error.code - && let ty::PredicateKind::Clause(ty::ClauseKind::Trait( - pred, - )) = error.obligation.predicate.kind().skip_binder() - { - self.infcx.err_ctxt().suggest_derive( - &error.obligation, - err, - error.obligation.predicate.kind().rebind(pred), - ); - } - } - } + ty.peel_refs(), + ), + ); + err.note(format!( + "the following trait bounds weren't met: {}", + errors + .iter() + .map(|e| e.obligation.predicate.to_string()) + .collect::>() + .join("\n"), + )); + } + // The type doesn't implement Clone because of unmet obligations. + for error in errors { + if let traits::FulfillmentErrorCode::Select( + traits::SelectionError::Unimplemented, + ) = error.code + && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(pred)) = + error.obligation.predicate.kind().skip_binder() + { + self.infcx.err_ctxt().suggest_derive( + &error.obligation, + err, + error.obligation.predicate.kind().rebind(pred), + ); } } - let (changing, span, sugg) = match local.ty { - Some(ty) => ("changing", ty.span, message), - None => { - ("specifying", local.pat.span.shrink_to_hi(), format!(": {message}")) - } - }; - err.span_suggestion_verbose( - span, - format!("consider {changing} this binding's type"), - sugg, - Applicability::HasPlaceholders, - ); - } else { - err.span_label( - err_label_span, - format!("consider changing this binding's type to be: `{message}`"), - ); } } - None => {} } + let (changing, span, sugg) = match local.ty { + Some(ty) => ("changing", ty.span, sugg_str), + None => ("specifying", local.pat.span.shrink_to_hi(), format!(": {sugg_str}")), + }; + err.span_suggestion_verbose( + span, + format!("consider {changing} this binding's type"), + sugg, + Applicability::HasPlaceholders, + ); } } @@ -1463,11 +1490,25 @@ fn suggest_ampmut_self(tcx: TyCtxt<'_>, span: Span) -> (Span, String) { } } -struct AmpMutSugg { - has_sugg: bool, - span: Span, - suggestion: String, - additional: Option<(Span, String)>, +enum AmpMutSugg { + /// Type suggestion. Changes `&self` to `&mut self`, `x: &T` to `x: &mut T`, + /// `ref x` to `ref mut x`, etc. + Type { + span: Span, + suggestion: String, + additional: Option<(Span, String)>, + }, + /// Suggestion for expressions, `&x` to `&mut x`, `&x[i]` to `&mut x[i]`, etc. + Expr { + span: Span, + suggestion: String, + }, + /// Suggests `.get_mut` in the case of `&map[&key]` for Hash/BTreeMap. + MapGetMut { + span: Span, + suggestion: String, + }, + ChangeBinding, } // When we want to suggest a user change a local variable to be a `&mut`, there @@ -1486,110 +1527,111 @@ struct AmpMutSugg { // This implementation attempts to emulate AST-borrowck prioritization // by trying (3.), then (2.) and finally falling back on (1.). fn suggest_ampmut<'tcx>( - tcx: TyCtxt<'tcx>, - decl_ty: Ty<'tcx>, - decl_span: Span, - opt_assignment_rhs_span: Option, - opt_ty_info: Option, + infcx: &crate::BorrowckInferCtxt<'tcx>, + body: &Body<'tcx>, + opt_assignment_rhs_stmt: Option<&Statement<'tcx>>, ) -> Option { - // if there is a RHS and it starts with a `&` from it, then check if it is + let tcx = infcx.tcx; + // If there is a RHS and it starts with a `&` from it, then check if it is // mutable, and if not, put suggest putting `mut ` to make it mutable. - // we don't have to worry about lifetime annotations here because they are + // We don't have to worry about lifetime annotations here because they are // not valid when taking a reference. For example, the following is not valid Rust: // // let x: &i32 = &'a 5; // ^^ lifetime annotation not allowed // - if let Some(rhs_span) = opt_assignment_rhs_span - && let Ok(rhs_str) = tcx.sess.source_map().span_to_snippet(rhs_span) - && let Some(rhs_str_no_amp) = rhs_str.strip_prefix('&') + if let Some(rhs_stmt) = opt_assignment_rhs_stmt + && let StatementKind::Assign(box (lhs, rvalue)) = &rhs_stmt.kind + && let mut rhs_span = rhs_stmt.source_info.span + && let Ok(mut rhs_str) = tcx.sess.source_map().span_to_snippet(rhs_span) { - // Suggest changing `&raw const` to `&raw mut` if applicable. - if rhs_str_no_amp.trim_start().strip_prefix("raw const").is_some() { - let const_idx = rhs_str.find("const").unwrap() as u32; - let const_span = rhs_span - .with_lo(rhs_span.lo() + BytePos(const_idx)) - .with_hi(rhs_span.lo() + BytePos(const_idx + "const".len() as u32)); + let mut rvalue = rvalue; - return Some(AmpMutSugg { - has_sugg: true, - span: const_span, - suggestion: "mut".to_owned(), - additional: None, - }); - } - - // Figure out if rhs already is `&mut`. - let is_mut = if let Some(rest) = rhs_str_no_amp.trim_start().strip_prefix("mut") { - match rest.chars().next() { - // e.g. `&mut x` - Some(c) if c.is_whitespace() => true, - // e.g. `&mut(x)` - Some('(') => true, - // e.g. `&mut{x}` - Some('{') => true, - // e.g. `&mutablevar` - _ => false, + // Take some special care when handling `let _x = &*_y`: + // We want to know if this is part of an overloaded index, so `let x = &a[0]`, + // or whether this is a usertype ascription (`let _x: &T = y`). + if let Rvalue::Ref(_, BorrowKind::Shared, place) = rvalue + && place.projection.len() == 1 + && place.projection[0] == ProjectionElem::Deref + && let Some(assign) = find_assignments(&body, place.local).first() + { + // If this is a usertype ascription (`let _x: &T = _y`) then pierce through it as either we want + // to suggest `&mut` on the expression (handled here) or we return `None` and let the caller + // suggest `&mut` on the type if the expression seems fine (e.g. `let _x: &T = &mut _y`). + if let Some(user_ty_projs) = body.local_decls[lhs.local].user_ty.as_ref() + && let [user_ty_proj] = user_ty_projs.contents.as_slice() + && user_ty_proj.projs.is_empty() + && let Either::Left(rhs_stmt_new) = body.stmt_at(*assign) + && let StatementKind::Assign(box (_, rvalue_new)) = &rhs_stmt_new.kind + && let rhs_span_new = rhs_stmt_new.source_info.span + && let Ok(rhs_str_new) = tcx.sess.source_map().span_to_snippet(rhs_span) + { + (rvalue, rhs_span, rhs_str) = (rvalue_new, rhs_span_new, rhs_str_new); } - } else { - false - }; - // if the reference is already mutable then there is nothing we can do - // here. - if !is_mut { - // shrink the span to just after the `&` in `&variable` - let span = rhs_span.with_lo(rhs_span.lo() + BytePos(1)).shrink_to_lo(); - // FIXME(Ezrashaw): returning is bad because we still might want to - // update the annotated type, see #106857. - return Some(AmpMutSugg { - has_sugg: true, - span, - suggestion: "mut ".to_owned(), - additional: None, - }); + if let Either::Right(call) = body.stmt_at(*assign) + && let TerminatorKind::Call { + func: Operand::Constant(box const_operand), args, .. + } = &call.kind + && let ty::FnDef(method_def_id, method_args) = *const_operand.ty().kind() + && let Some(trait_) = tcx.trait_of_assoc(method_def_id) + && tcx.is_lang_item(trait_, hir::LangItem::Index) + { + let trait_ref = ty::TraitRef::from_assoc( + tcx, + tcx.require_lang_item(hir::LangItem::IndexMut, rhs_span), + method_args, + ); + // The type only implements `Index` but not `IndexMut`, we must not suggest `&mut`. + if !infcx + .type_implements_trait(trait_ref.def_id, trait_ref.args, infcx.param_env) + .must_apply_considering_regions() + { + // Suggest `get_mut` if type is a `BTreeMap` or `HashMap`. + if let ty::Adt(def, _) = trait_ref.self_ty().kind() + && [sym::BTreeMap, sym::HashMap] + .into_iter() + .any(|s| tcx.is_diagnostic_item(s, def.did())) + && let [map, key] = &**args + && let Ok(map) = tcx.sess.source_map().span_to_snippet(map.span) + && let Ok(key) = tcx.sess.source_map().span_to_snippet(key.span) + { + let span = rhs_span; + let suggestion = format!("{map}.get_mut({key}).unwrap()"); + return Some(AmpMutSugg::MapGetMut { span, suggestion }); + } + return None; + } + } + } + + let sugg = match rvalue { + Rvalue::Ref(_, BorrowKind::Shared, _) if let Some(ref_idx) = rhs_str.find('&') => { + // Shrink the span to just after the `&` in `&variable`. + Some(( + rhs_span.with_lo(rhs_span.lo() + BytePos(ref_idx as u32 + 1)).shrink_to_lo(), + "mut ".to_owned(), + )) + } + Rvalue::RawPtr(RawPtrKind::Const, _) if let Some(const_idx) = rhs_str.find("const") => { + // Suggest changing `&raw const` to `&raw mut` if applicable. + let const_idx = const_idx as u32; + Some(( + rhs_span + .with_lo(rhs_span.lo() + BytePos(const_idx)) + .with_hi(rhs_span.lo() + BytePos(const_idx + "const".len() as u32)), + "mut".to_owned(), + )) + } + _ => None, + }; + + if let Some((span, suggestion)) = sugg { + return Some(AmpMutSugg::Expr { span, suggestion }); } } - let (binding_exists, span) = match opt_ty_info { - // if this is a variable binding with an explicit type, - // then we will suggest changing it to be mutable. - // this is `Applicability::MachineApplicable`. - Some(ty_span) => (true, ty_span), - - // otherwise, we'll suggest *adding* an annotated type, we'll suggest - // the RHS's type for that. - // this is `Applicability::HasPlaceholders`. - None => (false, decl_span), - }; - - // if the binding already exists and is a reference with an explicit - // lifetime, then we can suggest adding ` mut`. this is special-cased from - // the path without an explicit lifetime. - if let Ok(src) = tcx.sess.source_map().span_to_snippet(span) - && src.starts_with("&'") - // note that `& 'a T` is invalid so this is correct. - && let Some(ws_pos) = src.find(char::is_whitespace) - { - let span = span.with_lo(span.lo() + BytePos(ws_pos as u32)).shrink_to_lo(); - Some(AmpMutSugg { has_sugg: true, span, suggestion: " mut".to_owned(), additional: None }) - // if there is already a binding, we modify it to be `mut` - } else if binding_exists { - // shrink the span to just after the `&` in `&variable` - let span = span.with_lo(span.lo() + BytePos(1)).shrink_to_lo(); - Some(AmpMutSugg { has_sugg: true, span, suggestion: "mut ".to_owned(), additional: None }) - } else { - // otherwise, suggest that the user annotates the binding; we provide the - // type of the local. - let ty = decl_ty.builtin_deref(true).unwrap(); - - Some(AmpMutSugg { - has_sugg: false, - span, - suggestion: format!("{}mut {}", if decl_ty.is_ref() { "&" } else { "*" }, ty), - additional: None, - }) - } + Some(AmpMutSugg::ChangeBinding) } /// If the type is a `Coroutine`, `Closure`, or `CoroutineClosure` diff --git a/compiler/rustc_borrowck/src/diagnostics/opaque_suggestions.rs b/compiler/rustc_borrowck/src/diagnostics/opaque_suggestions.rs deleted file mode 100644 index 7192a889adcb..000000000000 --- a/compiler/rustc_borrowck/src/diagnostics/opaque_suggestions.rs +++ /dev/null @@ -1,243 +0,0 @@ -#![allow(rustc::diagnostic_outside_of_impl)] -#![allow(rustc::untranslatable_diagnostic)] - -use std::ops::ControlFlow; - -use either::Either; -use itertools::Itertools as _; -use rustc_data_structures::fx::FxIndexSet; -use rustc_errors::{Diag, Subdiagnostic}; -use rustc_hir as hir; -use rustc_hir::def_id::DefId; -use rustc_middle::mir::{self, ConstraintCategory, Location}; -use rustc_middle::ty::{ - self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, -}; -use rustc_trait_selection::errors::impl_trait_overcapture_suggestion; - -use crate::MirBorrowckCtxt; -use crate::borrow_set::BorrowData; -use crate::consumers::RegionInferenceContext; -use crate::type_check::Locations; - -impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { - /// Try to note when an opaque is involved in a borrowck error and that - /// opaque captures lifetimes due to edition 2024. - // FIXME: This code is otherwise somewhat general, and could easily be adapted - // to explain why other things overcapture... like async fn and RPITITs. - pub(crate) fn note_due_to_edition_2024_opaque_capture_rules( - &self, - borrow: &BorrowData<'tcx>, - diag: &mut Diag<'_>, - ) { - // We look at all the locals. Why locals? Because it's the best thing - // I could think of that's correlated with the *instantiated* higher-ranked - // binder for calls, since we don't really store those anywhere else. - for ty in self.body.local_decls.iter().map(|local| local.ty) { - if !ty.has_opaque_types() { - continue; - } - - let tcx = self.infcx.tcx; - let ControlFlow::Break((opaque_def_id, offending_region_idx, location)) = ty - .visit_with(&mut FindOpaqueRegion { - regioncx: &self.regioncx, - tcx, - borrow_region: borrow.region, - }) - else { - continue; - }; - - // If an opaque explicitly captures a lifetime, then no need to point it out. - // FIXME: We should be using a better heuristic for `use<>`. - if tcx.rendered_precise_capturing_args(opaque_def_id).is_some() { - continue; - } - - // If one of the opaque's bounds mentions the region, then no need to - // point it out, since it would've been captured on edition 2021 as well. - // - // Also, while we're at it, collect all the lifetimes that the opaque - // *does* mention. We'll use that for the `+ use<'a>` suggestion below. - let mut visitor = CheckExplicitRegionMentionAndCollectGenerics { - tcx, - generics: tcx.generics_of(opaque_def_id), - offending_region_idx, - seen_opaques: [opaque_def_id].into_iter().collect(), - seen_lifetimes: Default::default(), - }; - if tcx - .explicit_item_bounds(opaque_def_id) - .skip_binder() - .visit_with(&mut visitor) - .is_break() - { - continue; - } - - // If we successfully located a terminator, then point it out - // and provide a suggestion if it's local. - match self.body.stmt_at(location) { - Either::Right(mir::Terminator { source_info, .. }) => { - diag.span_note( - source_info.span, - "this call may capture more lifetimes than intended, \ - because Rust 2024 has adjusted the `impl Trait` lifetime capture rules", - ); - let mut captured_args = visitor.seen_lifetimes; - // Add in all of the type and const params, too. - // Ordering here is kinda strange b/c we're walking backwards, - // but we're trying to provide *a* suggestion, not a nice one. - let mut next_generics = Some(visitor.generics); - let mut any_synthetic = false; - while let Some(generics) = next_generics { - for param in &generics.own_params { - if param.kind.is_ty_or_const() { - captured_args.insert(param.def_id); - } - if param.kind.is_synthetic() { - any_synthetic = true; - } - } - next_generics = generics.parent.map(|def_id| tcx.generics_of(def_id)); - } - - if let Some(opaque_def_id) = opaque_def_id.as_local() - && let hir::OpaqueTyOrigin::FnReturn { parent, .. } = - tcx.hir_expect_opaque_ty(opaque_def_id).origin - { - if let Some(sugg) = impl_trait_overcapture_suggestion( - tcx, - opaque_def_id, - parent, - captured_args, - ) { - sugg.add_to_diag(diag); - } - } else { - diag.span_help( - tcx.def_span(opaque_def_id), - format!( - "if you can modify this crate, add a precise \ - capturing bound to avoid overcapturing: `+ use<{}>`", - if any_synthetic { - "/* Args */".to_string() - } else { - captured_args - .into_iter() - .map(|def_id| tcx.item_name(def_id)) - .join(", ") - } - ), - ); - } - return; - } - Either::Left(_) => {} - } - } - } -} - -/// This visitor contains the bulk of the logic for this lint. -struct FindOpaqueRegion<'a, 'tcx> { - tcx: TyCtxt<'tcx>, - regioncx: &'a RegionInferenceContext<'tcx>, - borrow_region: ty::RegionVid, -} - -impl<'tcx> TypeVisitor> for FindOpaqueRegion<'_, 'tcx> { - type Result = ControlFlow<(DefId, usize, Location), ()>; - - fn visit_ty(&mut self, ty: Ty<'tcx>) -> Self::Result { - // If we find an opaque in a local ty, then for each of its captured regions, - // try to find a path between that captured regions and our borrow region... - if let ty::Alias(ty::Opaque, opaque) = *ty.kind() - && let hir::OpaqueTyOrigin::FnReturn { parent, in_trait_or_impl: None } = - self.tcx.opaque_ty_origin(opaque.def_id) - { - let variances = self.tcx.variances_of(opaque.def_id); - for (idx, (arg, variance)) in std::iter::zip(opaque.args, variances).enumerate() { - // Skip uncaptured args. - if *variance == ty::Bivariant { - continue; - } - // We only care about regions. - let Some(opaque_region) = arg.as_region() else { - continue; - }; - // Don't try to convert a late-bound region, which shouldn't exist anyways (yet). - if opaque_region.is_bound() { - continue; - } - let opaque_region_vid = self.regioncx.to_region_vid(opaque_region); - - // Find a path between the borrow region and our opaque capture. - if let Some((path, _)) = - self.regioncx.find_constraint_paths_between_regions(self.borrow_region, |r| { - r == opaque_region_vid - }) - { - for constraint in path { - // If we find a call in this path, then check if it defines the opaque. - if let ConstraintCategory::CallArgument(Some(call_ty)) = constraint.category - && let ty::FnDef(call_def_id, _) = *call_ty.kind() - // This function defines the opaque :D - && call_def_id == parent - && let Locations::Single(location) = constraint.locations - { - return ControlFlow::Break((opaque.def_id, idx, location)); - } - } - } - } - } - - ty.super_visit_with(self) - } -} - -struct CheckExplicitRegionMentionAndCollectGenerics<'tcx> { - tcx: TyCtxt<'tcx>, - generics: &'tcx ty::Generics, - offending_region_idx: usize, - seen_opaques: FxIndexSet, - seen_lifetimes: FxIndexSet, -} - -impl<'tcx> TypeVisitor> for CheckExplicitRegionMentionAndCollectGenerics<'tcx> { - type Result = ControlFlow<(), ()>; - - fn visit_ty(&mut self, ty: Ty<'tcx>) -> Self::Result { - match *ty.kind() { - ty::Alias(ty::Opaque, opaque) => { - if self.seen_opaques.insert(opaque.def_id) { - for (bound, _) in self - .tcx - .explicit_item_bounds(opaque.def_id) - .iter_instantiated_copied(self.tcx, opaque.args) - { - bound.visit_with(self)?; - } - } - ControlFlow::Continue(()) - } - _ => ty.super_visit_with(self), - } - } - - fn visit_region(&mut self, r: ty::Region<'tcx>) -> Self::Result { - match r.kind() { - ty::ReEarlyParam(param) => { - if param.index as usize == self.offending_region_idx { - ControlFlow::Break(()) - } else { - self.seen_lifetimes.insert(self.generics.region_param(param, self.tcx).def_id); - ControlFlow::Continue(()) - } - } - _ => ControlFlow::Continue(()), - } - } -} diff --git a/compiler/rustc_borrowck/src/diagnostics/opaque_types.rs b/compiler/rustc_borrowck/src/diagnostics/opaque_types.rs new file mode 100644 index 000000000000..999136264184 --- /dev/null +++ b/compiler/rustc_borrowck/src/diagnostics/opaque_types.rs @@ -0,0 +1,311 @@ +#![allow(rustc::diagnostic_outside_of_impl)] +#![allow(rustc::untranslatable_diagnostic)] + +use std::ops::ControlFlow; + +use either::Either; +use itertools::Itertools as _; +use rustc_data_structures::fx::FxIndexSet; +use rustc_errors::{Diag, Subdiagnostic}; +use rustc_hir as hir; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::{self, ConstraintCategory, Location}; +use rustc_middle::ty::{ + self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor, +}; +use rustc_span::Span; +use rustc_trait_selection::error_reporting::infer::region::unexpected_hidden_region_diagnostic; +use rustc_trait_selection::errors::impl_trait_overcapture_suggestion; + +use crate::MirBorrowckCtxt; +use crate::borrow_set::BorrowData; +use crate::consumers::RegionInferenceContext; +use crate::region_infer::opaque_types::DeferredOpaqueTypeError; +use crate::type_check::Locations; + +impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { + pub(crate) fn report_opaque_type_errors(&mut self, errors: Vec>) { + if errors.is_empty() { + return; + } + + let infcx = self.infcx; + let mut guar = None; + let mut last_unexpected_hidden_region: Option<(Span, Ty<'_>, ty::OpaqueTypeKey<'tcx>)> = + None; + for error in errors { + guar = Some(match error { + DeferredOpaqueTypeError::InvalidOpaqueTypeArgs(err) => err.report(infcx), + DeferredOpaqueTypeError::LifetimeMismatchOpaqueParam(err) => { + infcx.dcx().emit_err(err) + } + DeferredOpaqueTypeError::UnexpectedHiddenRegion { + opaque_type_key, + hidden_type, + member_region, + } => { + let named_ty = + self.regioncx.name_regions_for_member_constraint(infcx.tcx, hidden_type.ty); + let named_key = self + .regioncx + .name_regions_for_member_constraint(infcx.tcx, opaque_type_key); + let named_region = + self.regioncx.name_regions_for_member_constraint(infcx.tcx, member_region); + let diag = unexpected_hidden_region_diagnostic( + infcx, + self.mir_def_id(), + hidden_type.span, + named_ty, + named_region, + named_key, + ); + if last_unexpected_hidden_region + != Some((hidden_type.span, named_ty, named_key)) + { + last_unexpected_hidden_region = + Some((hidden_type.span, named_ty, named_key)); + diag.emit() + } else { + diag.delay_as_bug() + } + } + DeferredOpaqueTypeError::NonDefiningUseInDefiningScope { + span, + opaque_type_key, + } => infcx.dcx().span_err( + span, + format!( + "non-defining use of `{}` in the defining scope", + Ty::new_opaque( + infcx.tcx, + opaque_type_key.def_id.to_def_id(), + opaque_type_key.args + ) + ), + ), + }); + } + let guar = guar.unwrap(); + self.root_cx.set_tainted_by_errors(guar); + self.infcx.set_tainted_by_errors(guar); + } + + /// Try to note when an opaque is involved in a borrowck error and that + /// opaque captures lifetimes due to edition 2024. + // FIXME: This code is otherwise somewhat general, and could easily be adapted + // to explain why other things overcapture... like async fn and RPITITs. + pub(crate) fn note_due_to_edition_2024_opaque_capture_rules( + &self, + borrow: &BorrowData<'tcx>, + diag: &mut Diag<'_>, + ) { + // We look at all the locals. Why locals? Because it's the best thing + // I could think of that's correlated with the *instantiated* higher-ranked + // binder for calls, since we don't really store those anywhere else. + for ty in self.body.local_decls.iter().map(|local| local.ty) { + if !ty.has_opaque_types() { + continue; + } + + let tcx = self.infcx.tcx; + let ControlFlow::Break((opaque_def_id, offending_region_idx, location)) = ty + .visit_with(&mut FindOpaqueRegion { + regioncx: &self.regioncx, + tcx, + borrow_region: borrow.region, + }) + else { + continue; + }; + + // If an opaque explicitly captures a lifetime, then no need to point it out. + // FIXME: We should be using a better heuristic for `use<>`. + if tcx.rendered_precise_capturing_args(opaque_def_id).is_some() { + continue; + } + + // If one of the opaque's bounds mentions the region, then no need to + // point it out, since it would've been captured on edition 2021 as well. + // + // Also, while we're at it, collect all the lifetimes that the opaque + // *does* mention. We'll use that for the `+ use<'a>` suggestion below. + let mut visitor = CheckExplicitRegionMentionAndCollectGenerics { + tcx, + generics: tcx.generics_of(opaque_def_id), + offending_region_idx, + seen_opaques: [opaque_def_id].into_iter().collect(), + seen_lifetimes: Default::default(), + }; + if tcx + .explicit_item_bounds(opaque_def_id) + .skip_binder() + .visit_with(&mut visitor) + .is_break() + { + continue; + } + + // If we successfully located a terminator, then point it out + // and provide a suggestion if it's local. + match self.body.stmt_at(location) { + Either::Right(mir::Terminator { source_info, .. }) => { + diag.span_note( + source_info.span, + "this call may capture more lifetimes than intended, \ + because Rust 2024 has adjusted the `impl Trait` lifetime capture rules", + ); + let mut captured_args = visitor.seen_lifetimes; + // Add in all of the type and const params, too. + // Ordering here is kinda strange b/c we're walking backwards, + // but we're trying to provide *a* suggestion, not a nice one. + let mut next_generics = Some(visitor.generics); + let mut any_synthetic = false; + while let Some(generics) = next_generics { + for param in &generics.own_params { + if param.kind.is_ty_or_const() { + captured_args.insert(param.def_id); + } + if param.kind.is_synthetic() { + any_synthetic = true; + } + } + next_generics = generics.parent.map(|def_id| tcx.generics_of(def_id)); + } + + if let Some(opaque_def_id) = opaque_def_id.as_local() + && let hir::OpaqueTyOrigin::FnReturn { parent, .. } = + tcx.hir_expect_opaque_ty(opaque_def_id).origin + { + if let Some(sugg) = impl_trait_overcapture_suggestion( + tcx, + opaque_def_id, + parent, + captured_args, + ) { + sugg.add_to_diag(diag); + } + } else { + diag.span_help( + tcx.def_span(opaque_def_id), + format!( + "if you can modify this crate, add a precise \ + capturing bound to avoid overcapturing: `+ use<{}>`", + if any_synthetic { + "/* Args */".to_string() + } else { + captured_args + .into_iter() + .map(|def_id| tcx.item_name(def_id)) + .join(", ") + } + ), + ); + } + return; + } + Either::Left(_) => {} + } + } + } +} + +/// This visitor contains the bulk of the logic for this lint. +struct FindOpaqueRegion<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + regioncx: &'a RegionInferenceContext<'tcx>, + borrow_region: ty::RegionVid, +} + +impl<'tcx> TypeVisitor> for FindOpaqueRegion<'_, 'tcx> { + type Result = ControlFlow<(DefId, usize, Location), ()>; + + fn visit_ty(&mut self, ty: Ty<'tcx>) -> Self::Result { + // If we find an opaque in a local ty, then for each of its captured regions, + // try to find a path between that captured regions and our borrow region... + if let ty::Alias(ty::Opaque, opaque) = *ty.kind() + && let hir::OpaqueTyOrigin::FnReturn { parent, in_trait_or_impl: None } = + self.tcx.opaque_ty_origin(opaque.def_id) + { + let variances = self.tcx.variances_of(opaque.def_id); + for (idx, (arg, variance)) in std::iter::zip(opaque.args, variances).enumerate() { + // Skip uncaptured args. + if *variance == ty::Bivariant { + continue; + } + // We only care about regions. + let Some(opaque_region) = arg.as_region() else { + continue; + }; + // Don't try to convert a late-bound region, which shouldn't exist anyways (yet). + if opaque_region.is_bound() { + continue; + } + let opaque_region_vid = self.regioncx.to_region_vid(opaque_region); + + // Find a path between the borrow region and our opaque capture. + if let Some(path) = self + .regioncx + .constraint_path_between_regions(self.borrow_region, opaque_region_vid) + { + for constraint in path { + // If we find a call in this path, then check if it defines the opaque. + if let ConstraintCategory::CallArgument(Some(call_ty)) = constraint.category + && let ty::FnDef(call_def_id, _) = *call_ty.kind() + // This function defines the opaque :D + && call_def_id == parent + && let Locations::Single(location) = constraint.locations + { + return ControlFlow::Break((opaque.def_id, idx, location)); + } + } + } + } + } + + ty.super_visit_with(self) + } +} + +struct CheckExplicitRegionMentionAndCollectGenerics<'tcx> { + tcx: TyCtxt<'tcx>, + generics: &'tcx ty::Generics, + offending_region_idx: usize, + seen_opaques: FxIndexSet, + seen_lifetimes: FxIndexSet, +} + +impl<'tcx> TypeVisitor> for CheckExplicitRegionMentionAndCollectGenerics<'tcx> { + type Result = ControlFlow<(), ()>; + + fn visit_ty(&mut self, ty: Ty<'tcx>) -> Self::Result { + match *ty.kind() { + ty::Alias(ty::Opaque, opaque) => { + if self.seen_opaques.insert(opaque.def_id) { + for (bound, _) in self + .tcx + .explicit_item_bounds(opaque.def_id) + .iter_instantiated_copied(self.tcx, opaque.args) + { + bound.visit_with(self)?; + } + } + ControlFlow::Continue(()) + } + _ => ty.super_visit_with(self), + } + } + + fn visit_region(&mut self, r: ty::Region<'tcx>) -> Self::Result { + match r.kind() { + ty::ReEarlyParam(param) => { + if param.index as usize == self.offending_region_idx { + ControlFlow::Break(()) + } else { + self.seen_lifetimes.insert(self.generics.region_param(param, self.tcx).def_id); + ControlFlow::Continue(()) + } + } + _ => ControlFlow::Continue(()), + } + } +} diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs index b130cf8ed276..bad5f03e4127 100644 --- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs @@ -23,7 +23,6 @@ use rustc_trait_selection::error_reporting::infer::nice_region_error::{ self, HirTraitObjectVisitor, NiceRegionError, TraitObjectVisitor, find_anon_type, find_param_with_region, suggest_adding_lifetime_params, }; -use rustc_trait_selection::error_reporting::infer::region::unexpected_hidden_region_diagnostic; use rustc_trait_selection::infer::InferCtxtExt; use rustc_trait_selection::traits::{Obligation, ObligationCtxt}; use tracing::{debug, instrument, trace}; @@ -62,7 +61,7 @@ impl<'tcx> ConstraintDescription for ConstraintCategory<'tcx> { | ConstraintCategory::Boring | ConstraintCategory::BoringNoLocation | ConstraintCategory::Internal - | ConstraintCategory::IllegalUniverse => "", + | ConstraintCategory::OutlivesUnnameablePlaceholder(..) => "", } } } @@ -92,9 +91,6 @@ impl<'tcx> RegionErrors<'tcx> { ) -> impl Iterator, ErrorGuaranteed)> { self.0.into_iter() } - pub(crate) fn has_errors(&self) -> Option { - self.0.get(0).map(|x| x.1) - } } impl std::fmt::Debug for RegionErrors<'_> { @@ -108,18 +104,6 @@ pub(crate) enum RegionErrorKind<'tcx> { /// A generic bound failure for a type test (`T: 'a`). TypeTestError { type_test: TypeTest<'tcx> }, - /// An unexpected hidden region for an opaque type. - UnexpectedHiddenRegion { - /// The span for the member constraint. - span: Span, - /// The hidden type. - hidden_ty: Ty<'tcx>, - /// The opaque type. - key: ty::OpaqueTypeKey<'tcx>, - /// The unexpected region. - member_region: ty::Region<'tcx>, - }, - /// Higher-ranked subtyping error. BoundUniversalRegionError { /// The placeholder free region. @@ -218,7 +202,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { diag: &mut Diag<'_>, lower_bound: RegionVid, ) { - let mut suggestions = vec![]; let tcx = self.infcx.tcx; // find generic associated types in the given region 'lower_bound' @@ -240,9 +223,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { .collect::>(); debug!(?gat_id_and_generics); - // find higher-ranked trait bounds bounded to the generic associated types + // Look for the where-bound which introduces the placeholder. + // As we're using the HIR, we need to handle both `for<'a> T: Trait<'a>` + // and `T: for<'a> Trait`<'a>. let mut hrtb_bounds = vec![]; - gat_id_and_generics.iter().flatten().for_each(|(gat_hir_id, generics)| { + gat_id_and_generics.iter().flatten().for_each(|&(gat_hir_id, generics)| { for pred in generics.predicates { let BoundPredicate(WhereBoundPredicate { bound_generic_params, bounds, .. }) = pred.kind @@ -251,17 +236,32 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { }; if bound_generic_params .iter() - .rfind(|bgp| tcx.local_def_id_to_hir_id(bgp.def_id) == *gat_hir_id) + .rfind(|bgp| tcx.local_def_id_to_hir_id(bgp.def_id) == gat_hir_id) .is_some() { for bound in *bounds { hrtb_bounds.push(bound); } + } else { + for bound in *bounds { + if let Trait(trait_bound) = bound { + if trait_bound + .bound_generic_params + .iter() + .rfind(|bgp| tcx.local_def_id_to_hir_id(bgp.def_id) == gat_hir_id) + .is_some() + { + hrtb_bounds.push(bound); + return; + } + } + } } } }); debug!(?hrtb_bounds); + let mut suggestions = vec![]; hrtb_bounds.iter().for_each(|bound| { let Trait(PolyTraitRef { trait_ref, span: trait_span, .. }) = bound else { return; @@ -310,11 +310,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { pub(crate) fn report_region_errors(&mut self, nll_errors: RegionErrors<'tcx>) { // Iterate through all the errors, producing a diagnostic for each one. The diagnostics are // buffered in the `MirBorrowckCtxt`. - let mut outlives_suggestion = OutlivesSuggestionBuilder::default(); - let mut last_unexpected_hidden_region: Option<(Span, Ty<'_>, ty::OpaqueTypeKey<'tcx>)> = - None; - for (nll_error, _) in nll_errors.into_iter() { match nll_error { RegionErrorKind::TypeTestError { type_test } => { @@ -365,30 +361,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { } } - RegionErrorKind::UnexpectedHiddenRegion { span, hidden_ty, key, member_region } => { - let named_ty = - self.regioncx.name_regions_for_member_constraint(self.infcx.tcx, hidden_ty); - let named_key = - self.regioncx.name_regions_for_member_constraint(self.infcx.tcx, key); - let named_region = self - .regioncx - .name_regions_for_member_constraint(self.infcx.tcx, member_region); - let diag = unexpected_hidden_region_diagnostic( - self.infcx, - self.mir_def_id(), - span, - named_ty, - named_region, - named_key, - ); - if last_unexpected_hidden_region != Some((span, named_ty, named_key)) { - self.buffer_error(diag); - last_unexpected_hidden_region = Some((span, named_ty, named_key)); - } else { - diag.delay_as_bug(); - } - } - RegionErrorKind::BoundUniversalRegionError { longer_fr, placeholder, @@ -397,11 +369,15 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { let error_vid = self.regioncx.region_from_element(longer_fr, &error_element); // Find the code to blame for the fact that `longer_fr` outlives `error_fr`. - let (_, cause) = self.regioncx.find_outlives_blame_span( - longer_fr, - NllRegionVariableOrigin::Placeholder(placeholder), - error_vid, - ); + let cause = self + .regioncx + .best_blame_constraint( + longer_fr, + NllRegionVariableOrigin::Placeholder(placeholder), + error_vid, + ) + .0 + .cause; let universe = placeholder.universe; let universe_info = self.regioncx.universe_info(universe); @@ -457,9 +433,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { ) { debug!("report_region_error(fr={:?}, outlived_fr={:?})", fr, outlived_fr); - let (blame_constraint, path) = self.regioncx.best_blame_constraint(fr, fr_origin, |r| { - self.regioncx.provides_universal_region(r, fr, outlived_fr) - }); + let (blame_constraint, path) = + self.regioncx.best_blame_constraint(fr, fr_origin, outlived_fr); let BlameConstraint { category, cause, variance_info, .. } = blame_constraint; debug!("report_region_error: category={:?} {:?} {:?}", category, cause, variance_info); @@ -1159,7 +1134,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { Obligation::misc(tcx, span, self.mir_def_id(), self.infcx.param_env, pred) })); - if ocx.select_all_or_error().is_empty() && count > 0 { + if ocx.evaluate_obligations_error_on_ambiguity().is_empty() && count > 0 { diag.span_suggestion_verbose( tcx.hir_body(*body).value.peel_blocks().span.shrink_to_lo(), fluent::borrowck_dereference_suggestion, diff --git a/compiler/rustc_borrowck/src/handle_placeholders.rs b/compiler/rustc_borrowck/src/handle_placeholders.rs index aaaf2f45c869..d23ecf6c7079 100644 --- a/compiler/rustc_borrowck/src/handle_placeholders.rs +++ b/compiler/rustc_borrowck/src/handle_placeholders.rs @@ -1,7 +1,6 @@ //! Logic for lowering higher-kinded outlives constraints //! (with placeholders and universes) and turn them into regular //! outlives constraints. - use rustc_data_structures::frozen::Frozen; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::graph::scc; @@ -10,12 +9,11 @@ use rustc_index::IndexVec; use rustc_infer::infer::RegionVariableOrigin; use rustc_middle::mir::ConstraintCategory; use rustc_middle::ty::{RegionVid, UniverseIndex}; -use tracing::debug; +use tracing::{debug, trace}; use crate::constraints::{ConstraintSccIndex, OutlivesConstraintSet}; use crate::consumers::OutlivesConstraint; use crate::diagnostics::UniverseInfo; -use crate::member_constraints::MemberConstraintSet; use crate::region_infer::values::{LivenessValues, PlaceholderIndices}; use crate::region_infer::{ConstraintSccs, RegionDefinition, Representative, TypeTest}; use crate::ty::VarianceDiagInfo; @@ -30,7 +28,6 @@ pub(crate) struct LoweredConstraints<'tcx> { pub(crate) constraint_sccs: Sccs, pub(crate) definitions: Frozen>>, pub(crate) scc_annotations: IndexVec, - pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>, pub(crate) outlives_constraints: Frozen>, pub(crate) type_tests: Vec>, pub(crate) liveness_constraints: LivenessValues, @@ -64,18 +61,71 @@ impl scc::Annotations for SccAnnotations<'_, '_, RegionTracker> { type SccIdx = ConstraintSccIndex; } +#[derive(Copy, Debug, Clone, PartialEq, Eq)] +enum PlaceholderReachability { + /// This SCC reaches no placeholders. + NoPlaceholders, + /// This SCC reaches at least one placeholder. + Placeholders { + /// The largest-universed placeholder we can reach + max_universe: (UniverseIndex, RegionVid), + + /// The placeholder with the smallest ID + min_placeholder: RegionVid, + + /// The placeholder with the largest ID + max_placeholder: RegionVid, + }, +} + +impl PlaceholderReachability { + /// Merge the reachable placeholders of two graph components. + fn merge(self, other: PlaceholderReachability) -> PlaceholderReachability { + use PlaceholderReachability::*; + match (self, other) { + (NoPlaceholders, NoPlaceholders) => NoPlaceholders, + (NoPlaceholders, p @ Placeholders { .. }) + | (p @ Placeholders { .. }, NoPlaceholders) => p, + ( + Placeholders { + min_placeholder: min_pl, + max_placeholder: max_pl, + max_universe: max_u, + }, + Placeholders { min_placeholder, max_placeholder, max_universe }, + ) => Placeholders { + min_placeholder: min_pl.min(min_placeholder), + max_placeholder: max_pl.max(max_placeholder), + max_universe: max_u.max(max_universe), + }, + } + } + + fn max_universe(&self) -> Option<(UniverseIndex, RegionVid)> { + match self { + Self::NoPlaceholders => None, + Self::Placeholders { max_universe, .. } => Some(*max_universe), + } + } + + /// If we have reached placeholders, determine if they can + /// be named from this universe. + fn can_be_named_by(&self, from: UniverseIndex) -> bool { + self.max_universe() + .is_none_or(|(max_placeholder_universe, _)| from.can_name(max_placeholder_universe)) + } +} + /// An annotation for region graph SCCs that tracks /// the values of its elements. This annotates a single SCC. #[derive(Copy, Debug, Clone)] pub(crate) struct RegionTracker { - /// The largest universe of a placeholder reached from this SCC. - /// This includes placeholders within this SCC. - max_placeholder_universe_reached: UniverseIndex, + reachable_placeholders: PlaceholderReachability, /// The largest universe nameable from this SCC. /// It is the smallest nameable universes of all - /// existential regions reachable from it. - max_nameable_universe: UniverseIndex, + /// existential regions reachable from it. Small Rvids are preferred. + max_nameable_universe: (UniverseIndex, RegionVid), /// The representative Region Variable Id for this SCC. pub(crate) representative: Representative, @@ -83,69 +133,78 @@ pub(crate) struct RegionTracker { impl RegionTracker { pub(crate) fn new(rvid: RegionVid, definition: &RegionDefinition<'_>) -> Self { - let placeholder_universe = + let reachable_placeholders = if matches!(definition.origin, NllRegionVariableOrigin::Placeholder(_)) { - definition.universe + PlaceholderReachability::Placeholders { + max_universe: (definition.universe, rvid), + min_placeholder: rvid, + max_placeholder: rvid, + } } else { - UniverseIndex::ROOT + PlaceholderReachability::NoPlaceholders }; Self { - max_placeholder_universe_reached: placeholder_universe, - max_nameable_universe: definition.universe, + reachable_placeholders, + max_nameable_universe: (definition.universe, rvid), representative: Representative::new(rvid, definition), } } /// The largest universe this SCC can name. It's the smallest - /// largest nameable uninverse of any reachable region. + /// largest nameable universe of any reachable region, or + /// `max_nameable(r) = min (max_nameable(r') for r' reachable from r)` pub(crate) fn max_nameable_universe(self) -> UniverseIndex { - self.max_nameable_universe + self.max_nameable_universe.0 } - fn merge_min_max_seen(&mut self, other: &Self) { - self.max_placeholder_universe_reached = std::cmp::max( - self.max_placeholder_universe_reached, - other.max_placeholder_universe_reached, - ); - - self.max_nameable_universe = - std::cmp::min(self.max_nameable_universe, other.max_nameable_universe); + pub(crate) fn max_placeholder_universe_reached(self) -> UniverseIndex { + if let Some((universe, _)) = self.reachable_placeholders.max_universe() { + universe + } else { + UniverseIndex::ROOT + } } - /// Returns `true` if during the annotated SCC reaches a placeholder - /// with a universe larger than the smallest nameable universe of any - /// reachable existential region. - pub(crate) fn has_incompatible_universes(&self) -> bool { - self.max_nameable_universe().cannot_name(self.max_placeholder_universe_reached) + /// Determine if we can name all the placeholders in `other`. + pub(crate) fn can_name_all_placeholders(&self, other: Self) -> bool { + other.reachable_placeholders.can_be_named_by(self.max_nameable_universe.0) } - /// Determine if the tracked universes of the two SCCs are compatible. - pub(crate) fn universe_compatible_with(&self, other: Self) -> bool { - self.max_nameable_universe().can_name(other.max_nameable_universe()) - || self.max_nameable_universe().can_name(other.max_placeholder_universe_reached) + /// If this SCC reaches a placeholder it can't name, return it. + fn unnameable_placeholder(&self) -> Option<(UniverseIndex, RegionVid)> { + self.reachable_placeholders.max_universe().filter(|&(placeholder_universe, _)| { + !self.max_nameable_universe().can_name(placeholder_universe) + }) } } impl scc::Annotation for RegionTracker { - fn merge_scc(mut self, other: Self) -> Self { - self.representative = self.representative.merge_scc(other.representative); - self.merge_min_max_seen(&other); - self + fn merge_scc(self, other: Self) -> Self { + trace!("{:?} << {:?}", self.representative, other.representative); + + Self { + representative: self.representative.min(other.representative), + max_nameable_universe: self.max_nameable_universe.min(other.max_nameable_universe), + reachable_placeholders: self.reachable_placeholders.merge(other.reachable_placeholders), + } } - fn merge_reached(mut self, other: Self) -> Self { - // No update to in-component values, only add seen values. - self.merge_min_max_seen(&other); - self + fn merge_reached(self, other: Self) -> Self { + Self { + max_nameable_universe: self.max_nameable_universe.min(other.max_nameable_universe), + reachable_placeholders: self.reachable_placeholders.merge(other.reachable_placeholders), + representative: self.representative, + } } } /// Determines if the region variable definitions contain /// placeholders, and compute them for later use. -fn region_definitions<'tcx>( - universal_regions: &UniversalRegions<'tcx>, +// FIXME: This is also used by opaque type handling. Move it to a separate file. +pub(super) fn region_definitions<'tcx>( infcx: &BorrowckInferCtxt<'tcx>, + universal_regions: &UniversalRegions<'tcx>, ) -> (Frozen>>, bool) { let var_infos = infcx.get_region_var_infos(); // Create a RegionDefinition for each inference variable. This happens here because @@ -157,7 +216,7 @@ fn region_definitions<'tcx>( for info in var_infos.iter() { let origin = match info.origin { RegionVariableOrigin::Nll(origin) => origin, - _ => NllRegionVariableOrigin::Existential { from_forall: false }, + _ => NllRegionVariableOrigin::Existential { name: None }, }; let definition = RegionDefinition { origin, universe: info.universe, external_name: None }; @@ -209,29 +268,17 @@ pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>( infcx: &BorrowckInferCtxt<'tcx>, ) -> LoweredConstraints<'tcx> { let universal_regions = &universal_region_relations.universal_regions; - let (definitions, has_placeholders) = region_definitions(universal_regions, infcx); + let (definitions, has_placeholders) = region_definitions(infcx, universal_regions); let MirTypeckRegionConstraints { placeholder_indices, placeholder_index_to_region: _, liveness_constraints, mut outlives_constraints, - mut member_constraints, universe_causes, type_tests, } = constraints; - if let Some(guar) = universal_regions.tainted_by_errors() { - debug!("Universal regions tainted by errors; removing constraints!"); - // Suppress unhelpful extra errors in `infer_opaque_types` by clearing out all - // outlives bounds that we may end up checking. - outlives_constraints = Default::default(); - member_constraints = Default::default(); - - // Also taint the entire scope. - infcx.set_tainted_by_errors(guar); - } - let fr_static = universal_regions.fr_static; let compute_sccs = |constraints: &OutlivesConstraintSet<'tcx>, @@ -253,7 +300,6 @@ pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>( return LoweredConstraints { type_tests, - member_constraints, constraint_sccs, scc_annotations: scc_annotations.scc_to_annotation, definitions, @@ -290,7 +336,6 @@ pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>( constraint_sccs, definitions, scc_annotations, - member_constraints, outlives_constraints: Frozen::freeze(outlives_constraints), type_tests, liveness_constraints, @@ -299,7 +344,7 @@ pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>( } } -fn rewrite_placeholder_outlives<'tcx>( +pub(crate) fn rewrite_placeholder_outlives<'tcx>( sccs: &Sccs, annotations: &SccAnnotations<'_, '_, RegionTracker>, fr_static: RegionVid, @@ -321,28 +366,52 @@ fn rewrite_placeholder_outlives<'tcx>( let annotation = annotations[scc]; - // If this SCC participates in a universe violation, - // e.g. if it reaches a region with a universe smaller than - // the largest region reached, add a requirement that it must - // outlive `'static`. - if annotation.has_incompatible_universes() { - // Optimisation opportunity: this will add more constraints than - // needed for correctness, since an SCC upstream of another with - // a universe violation will "infect" its downstream SCCs to also - // outlive static. - let scc_representative_outlives_static = OutlivesConstraint { - sup: annotation.representative.rvid(), - sub: fr_static, - category: ConstraintCategory::IllegalUniverse, - locations: Locations::All(rustc_span::DUMMY_SP), - span: rustc_span::DUMMY_SP, - variance_info: VarianceDiagInfo::None, - from_closure: false, - }; - outlives_constraints.push(scc_representative_outlives_static); - added_constraints = true; - debug!("Added {:?}: 'static!", annotation.representative.rvid()); - } + let Some((max_u, max_u_rvid)) = annotation.unnameable_placeholder() else { + continue; + }; + + debug!( + "Placeholder universe {max_u:?} is too large for its SCC, represented by {:?}", + annotation.representative + ); + + // We only add one `r: 'static` constraint per SCC, where `r` is the SCC representative. + // That constraint is annotated with some placeholder `unnameable` where + // `unnameable` is unnameable from `r` and there is a path in the constraint graph + // between them. + // + // There is one exception; if some other region in this SCC can't name `'r`, then + // we pick the region with the smallest universe in the SCC, so that a path can + // always start in `'r` to find a motivation that isn't cyclic. + let blame_to = if annotation.representative.rvid() == max_u_rvid { + // Assertion: the region that lowered our universe is an existential one and we are a placeholder! + + // The SCC's representative is not nameable from some region + // that ends up in the SCC. + let small_universed_rvid = annotation.max_nameable_universe.1; + debug!( + "{small_universed_rvid:?} lowered our universe to {:?}", + annotation.max_nameable_universe() + ); + small_universed_rvid + } else { + // `max_u_rvid` is not nameable by the SCC's representative. + max_u_rvid + }; + + // FIXME: if we can extract a useful blame span here, future error + // reporting and constraint search can be simplified. + + added_constraints = true; + outlives_constraints.push(OutlivesConstraint { + sup: annotation.representative.rvid(), + sub: fr_static, + category: ConstraintCategory::OutlivesUnnameablePlaceholder(blame_to), + locations: Locations::All(rustc_span::DUMMY_SP), + span: rustc_span::DUMMY_SP, + variance_info: VarianceDiagInfo::None, + from_closure: false, + }); } added_constraints } diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 321b18c9b78b..db9df8a84f4e 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -19,10 +19,13 @@ use std::borrow::Cow; use std::cell::{OnceCell, RefCell}; use std::marker::PhantomData; use std::ops::{ControlFlow, Deref}; +use std::rc::Rc; use borrow_set::LocalsStateAtExit; +use polonius_engine::AllFacts; use root_cx::BorrowCheckRootCtxt; use rustc_abi::FieldIdx; +use rustc_data_structures::frozen::Frozen; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::graph::dominators::Dominators; use rustc_errors::LintDiagnostic; @@ -31,6 +34,7 @@ use rustc_hir::CRATE_HIR_ID; use rustc_hir::def_id::LocalDefId; use rustc_index::bit_set::MixedBitSet; use rustc_index::{IndexSlice, IndexVec}; +use rustc_infer::infer::outlives::env::RegionBoundPairs; use rustc_infer::infer::{ InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt, }; @@ -44,14 +48,15 @@ use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces} use rustc_mir_dataflow::move_paths::{ InitIndex, InitLocation, LookupResult, MoveData, MovePathIndex, }; -use rustc_mir_dataflow::{Analysis, Results, ResultsVisitor, visit_results}; +use rustc_mir_dataflow::points::DenseLocationMap; +use rustc_mir_dataflow::{Analysis, EntryStates, Results, ResultsVisitor, visit_results}; use rustc_session::lint::builtin::{TAIL_EXPR_DROP_ORDER, UNUSED_MUT}; use rustc_span::{ErrorGuaranteed, Span, Symbol}; use smallvec::SmallVec; use tracing::{debug, instrument}; use crate::borrow_set::{BorrowData, BorrowSet}; -use crate::consumers::BodyWithBorrowckFacts; +use crate::consumers::{BodyWithBorrowckFacts, RustcFacts}; use crate::dataflow::{BorrowIndex, Borrowck, BorrowckDomain, Borrows}; use crate::diagnostics::{ AccessKind, BorrowckDiagnosticsBuffer, IllegalMoveOriginKind, MoveError, RegionName, @@ -59,12 +64,17 @@ use crate::diagnostics::{ use crate::path_utils::*; use crate::place_ext::PlaceExt; use crate::places_conflict::{PlaceConflictBias, places_conflict}; -use crate::polonius::PoloniusDiagnosticsContext; -use crate::polonius::legacy::{PoloniusLocationTable, PoloniusOutput}; +use crate::polonius::legacy::{ + PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput, +}; +use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext}; use crate::prefixes::PrefixSet; use crate::region_infer::RegionInferenceContext; +use crate::region_infer::opaque_types::DeferredOpaqueTypeError; use crate::renumber::RegionCtxt; use crate::session_diagnostics::VarNeedNotMut; +use crate::type_check::free_region_relations::UniversalRegionRelations; +use crate::type_check::{Locations, MirTypeckRegionConstraints, MirTypeckResults}; mod borrow_set; mod borrowck_errors; @@ -73,7 +83,6 @@ mod dataflow; mod def_use; mod diagnostics; mod handle_placeholders; -mod member_constraints; mod nll; mod path_utils; mod place_ext; @@ -110,7 +119,7 @@ pub fn provide(providers: &mut Providers) { fn mir_borrowck( tcx: TyCtxt<'_>, def: LocalDefId, -) -> Result<&ConcreteOpaqueTypes<'_>, ErrorGuaranteed> { +) -> Result<&DefinitionSiteHiddenTypes<'_>, ErrorGuaranteed> { assert!(!tcx.is_typeck_child(def.to_def_id())); let (input_body, _) = tcx.mir_promoted(def); debug!("run query mir_borrowck: {}", tcx.def_path_str(def)); @@ -121,22 +130,11 @@ fn mir_borrowck( Err(guar) } else if input_body.should_skip() { debug!("Skipping borrowck because of injected body"); - let opaque_types = ConcreteOpaqueTypes(Default::default()); + let opaque_types = DefinitionSiteHiddenTypes(Default::default()); Ok(tcx.arena.alloc(opaque_types)) } else { let mut root_cx = BorrowCheckRootCtxt::new(tcx, def, None); - // We need to manually borrowck all nested bodies from the HIR as - // we do not generate MIR for dead code. Not doing so causes us to - // never check closures in dead code. - let nested_bodies = tcx.nested_bodies_within(def); - for def_id in nested_bodies { - root_cx.get_or_insert_nested(def_id); - } - - let PropagatedBorrowCheckResults { closure_requirements, used_mut_upvars } = - do_mir_borrowck(&mut root_cx, def); - debug_assert!(closure_requirements.is_none()); - debug_assert!(used_mut_upvars.is_empty()); + root_cx.do_mir_borrowck(); root_cx.finalize() } } @@ -149,6 +147,8 @@ struct PropagatedBorrowCheckResults<'tcx> { used_mut_upvars: SmallVec<[FieldIdx; 8]>, } +type DeferredClosureRequirements<'tcx> = Vec<(LocalDefId, ty::GenericArgsRef<'tcx>, Locations)>; + /// After we borrow check a closure, we are left with various /// requirements that we have inferred between the free regions that /// appear in the closure's signature or on its field types. These @@ -278,7 +278,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> { mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>, ) -> Ty<'tcx> { fold_regions(tcx, self.inner, |r, depth| match r.kind() { - ty::ReBound(debruijn, br) => { + ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), br) => { debug_assert_eq!(debruijn, depth); map(ty::RegionVid::from_usize(br.var.index())) } @@ -287,16 +287,33 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> { } } -/// Perform the actual borrow checking. -/// -/// For nested bodies this should only be called through `root_cx.get_or_insert_nested`. -#[instrument(skip(root_cx), level = "debug")] -fn do_mir_borrowck<'tcx>( +struct CollectRegionConstraintsResult<'tcx> { + infcx: BorrowckInferCtxt<'tcx>, + body_owned: Body<'tcx>, + promoted: IndexVec>, + move_data: MoveData<'tcx>, + borrow_set: BorrowSet<'tcx>, + location_table: PoloniusLocationTable, + location_map: Rc, + universal_region_relations: Frozen>, + region_bound_pairs: Frozen>, + known_type_outlives_obligations: Frozen>>, + constraints: MirTypeckRegionConstraints<'tcx>, + deferred_closure_requirements: DeferredClosureRequirements<'tcx>, + deferred_opaque_type_errors: Vec>, + polonius_facts: Option>, + polonius_context: Option, +} + +/// Start borrow checking by collecting the region constraints for +/// the current body. This initializes the relevant data structures +/// and then type checks the MIR body. +fn borrowck_collect_region_constraints<'tcx>( root_cx: &mut BorrowCheckRootCtxt<'tcx>, def: LocalDefId, -) -> PropagatedBorrowCheckResults<'tcx> { +) -> CollectRegionConstraintsResult<'tcx> { let tcx = root_cx.tcx; - let infcx = BorrowckInferCtxt::new(tcx, def); + let infcx = BorrowckInferCtxt::new(tcx, def, root_cx.root_def_id()); let (input_body, promoted) = tcx.mir_promoted(def); let input_body: &Body<'_> = &input_body.borrow(); let input_promoted: &IndexSlice<_, _> = &promoted.borrow(); @@ -321,7 +338,84 @@ fn do_mir_borrowck<'tcx>( let locals_are_invalidated_at_exit = tcx.hir_body_owner_kind(def).is_fn_or_closure(); let borrow_set = BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data); - // Compute non-lexical lifetimes. + let location_map = Rc::new(DenseLocationMap::new(body)); + + let polonius_input = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_input()) + || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); + let mut polonius_facts = + (polonius_input || PoloniusFacts::enabled(infcx.tcx)).then_some(PoloniusFacts::default()); + + // Run the MIR type-checker. + let MirTypeckResults { + constraints, + universal_region_relations, + region_bound_pairs, + known_type_outlives_obligations, + deferred_closure_requirements, + polonius_context, + } = type_check::type_check( + root_cx, + &infcx, + body, + &promoted, + universal_regions, + &location_table, + &borrow_set, + &mut polonius_facts, + &move_data, + Rc::clone(&location_map), + ); + + CollectRegionConstraintsResult { + infcx, + body_owned, + promoted, + move_data, + borrow_set, + location_table, + location_map, + universal_region_relations, + region_bound_pairs, + known_type_outlives_obligations, + constraints, + deferred_closure_requirements, + deferred_opaque_type_errors: Default::default(), + polonius_facts, + polonius_context, + } +} + +/// Using the region constraints computed by [borrowck_collect_region_constraints] +/// and the additional constraints from [BorrowCheckRootCtxt::handle_opaque_type_uses], +/// compute the region graph and actually check for any borrowck errors. +fn borrowck_check_region_constraints<'tcx>( + root_cx: &mut BorrowCheckRootCtxt<'tcx>, + CollectRegionConstraintsResult { + infcx, + body_owned, + promoted, + move_data, + borrow_set, + location_table, + location_map, + universal_region_relations, + region_bound_pairs: _, + known_type_outlives_obligations: _, + constraints, + deferred_closure_requirements, + deferred_opaque_type_errors, + polonius_facts, + polonius_context, + }: CollectRegionConstraintsResult<'tcx>, +) -> PropagatedBorrowCheckResults<'tcx> { + assert!(!infcx.has_opaque_types_in_storage()); + assert!(deferred_closure_requirements.is_empty()); + let tcx = root_cx.tcx; + let body = &body_owned; + let def = body.source.def_id().expect_local(); + + // Compute non-lexical lifetimes using the constraints computed + // by typechecking the MIR body. let nll::NllOutput { regioncx, polonius_input, @@ -332,12 +426,15 @@ fn do_mir_borrowck<'tcx>( } = nll::compute_regions( root_cx, &infcx, - universal_regions, body, - &promoted, &location_table, &move_data, &borrow_set, + location_map, + universal_region_relations, + constraints, + polonius_facts, + polonius_context, ); // Dump MIR results into a file, if that is enabled. This lets us @@ -434,15 +531,17 @@ fn do_mir_borrowck<'tcx>( }; // Compute and report region errors, if any. - mbcx.report_region_errors(nll_errors); + if nll_errors.is_empty() { + mbcx.report_opaque_type_errors(deferred_opaque_type_errors); + } else { + mbcx.report_region_errors(nll_errors); + } - let (mut flow_analysis, flow_entry_states) = - get_flow_results(tcx, body, &move_data, &borrow_set, ®ioncx); + let flow_results = get_flow_results(tcx, body, &move_data, &borrow_set, ®ioncx); visit_results( body, traversal::reverse_postorder(body).map(|(bb, _)| bb), - &mut flow_analysis, - &flow_entry_states, + &flow_results, &mut mbcx, ); @@ -503,7 +602,7 @@ fn get_flow_results<'a, 'tcx>( move_data: &'a MoveData<'tcx>, borrow_set: &'a BorrowSet<'tcx>, regioncx: &RegionInferenceContext<'tcx>, -) -> (Borrowck<'a, 'tcx>, Results) { +) -> Results<'tcx, Borrowck<'a, 'tcx>> { // We compute these three analyses individually, but them combine them into // a single results so that `mbcx` can visit them all together. let borrows = Borrows::new(tcx, body, regioncx, borrow_set).iterate_to_fixpoint( @@ -528,24 +627,25 @@ fn get_flow_results<'a, 'tcx>( ever_inits: ever_inits.analysis, }; - assert_eq!(borrows.results.len(), uninits.results.len()); - assert_eq!(borrows.results.len(), ever_inits.results.len()); - let results: Results<_> = - itertools::izip!(borrows.results, uninits.results, ever_inits.results) + assert_eq!(borrows.entry_states.len(), uninits.entry_states.len()); + assert_eq!(borrows.entry_states.len(), ever_inits.entry_states.len()); + let entry_states: EntryStates<_> = + itertools::izip!(borrows.entry_states, uninits.entry_states, ever_inits.entry_states) .map(|(borrows, uninits, ever_inits)| BorrowckDomain { borrows, uninits, ever_inits }) .collect(); - (analysis, results) + Results { analysis, entry_states } } pub(crate) struct BorrowckInferCtxt<'tcx> { pub(crate) infcx: InferCtxt<'tcx>, - pub(crate) reg_var_to_origin: RefCell>, + pub(crate) root_def_id: LocalDefId, pub(crate) param_env: ParamEnv<'tcx>, + pub(crate) reg_var_to_origin: RefCell>, } impl<'tcx> BorrowckInferCtxt<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self { + pub(crate) fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId, root_def_id: LocalDefId) -> Self { let typing_mode = if tcx.use_typing_mode_borrowck() { TypingMode::borrowck(tcx, def_id) } else { @@ -553,7 +653,12 @@ impl<'tcx> BorrowckInferCtxt<'tcx> { }; let infcx = tcx.infer_ctxt().build(typing_mode); let param_env = tcx.param_env(def_id); - BorrowckInferCtxt { infcx, reg_var_to_origin: RefCell::new(Default::default()), param_env } + BorrowckInferCtxt { + infcx, + root_def_id, + reg_var_to_origin: RefCell::new(Default::default()), + param_env, + } } pub(crate) fn next_region_var( @@ -683,7 +788,7 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> { impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a, '_, 'tcx> { fn visit_after_early_statement_effect( &mut self, - _analysis: &mut Borrowck<'a, 'tcx>, + _analysis: &Borrowck<'a, 'tcx>, state: &BorrowckDomain, stmt: &Statement<'tcx>, location: Location, @@ -750,7 +855,6 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a, } StatementKind::Nop | StatementKind::Retag { .. } - | StatementKind::Deinit(..) | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") } @@ -759,7 +863,7 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a, fn visit_after_early_terminator_effect( &mut self, - _analysis: &mut Borrowck<'a, 'tcx>, + _analysis: &Borrowck<'a, 'tcx>, state: &BorrowckDomain, term: &Terminator<'tcx>, loc: Location, @@ -879,7 +983,7 @@ impl<'a, 'tcx> ResultsVisitor<'tcx, Borrowck<'a, 'tcx>> for MirBorrowckCtxt<'a, fn visit_after_primary_terminator_effect( &mut self, - _analysis: &mut Borrowck<'a, 'tcx>, + _analysis: &Borrowck<'a, 'tcx>, state: &BorrowckDomain, term: &Terminator<'tcx>, loc: Location, @@ -1432,27 +1536,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { self.consume_operand(location, (operand, span), state) } - &Rvalue::CopyForDeref(place) => { - self.access_place( - location, - (place, span), - (Deep, Read(ReadKind::Copy)), - LocalMutationIsAllowed::No, - state, - ); - - // Finally, check if path was already moved. - self.check_if_path_or_subpath_is_moved( - location, - InitializationRequiringAction::Use, - (place.as_ref(), span), - state, - ); - } - - &(Rvalue::Len(place) | Rvalue::Discriminant(place)) => { + &Rvalue::Discriminant(place) => { let af = match *rvalue { - Rvalue::Len(..) => Some(ArtificialField::ArrayLength), Rvalue::Discriminant(..) => None, _ => unreachable!(), }; @@ -1512,6 +1597,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { Rvalue::WrapUnsafeBinder(op, _) => { self.consume_operand(location, (op, span), state); } + + Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"), } } @@ -1797,7 +1884,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { | ty::Slice(_) | ty::FnDef(_, _) | ty::FnPtr(..) - | ty::Dynamic(_, _, _) + | ty::Dynamic(_, _) | ty::Closure(_, _) | ty::CoroutineClosure(_, _) | ty::Coroutine(_, _) @@ -1843,7 +1930,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { | ty::Ref(_, _, _) | ty::FnDef(_, _) | ty::FnPtr(..) - | ty::Dynamic(_, _, _) + | ty::Dynamic(_, _) | ty::CoroutineWitness(..) | ty::Never | ty::UnsafeBinder(_) @@ -1883,10 +1970,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { }, // `OpaqueCast`: only transmutes the type, so no moves there. // `Downcast` : only changes information about a `Place` without moving. - // `Subtype` : only transmutes the type, so no moves. // So it's safe to skip these. ProjectionElem::OpaqueCast(_) - | ProjectionElem::Subtype(_) | ProjectionElem::Downcast(_, _) | ProjectionElem::UnwrapUnsafeBinder(_) => (), } @@ -2112,7 +2197,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { for (place_base, elem) in place.iter_projections().rev() { match elem { ProjectionElem::Index(_/*operand*/) | - ProjectionElem::Subtype(_) | ProjectionElem::OpaqueCast(_) | ProjectionElem::ConstantIndex { .. } | // assigning to P[i] requires P to be valid. @@ -2498,13 +2582,22 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { _ => bug!("Deref of unexpected type: {:?}", base_ty), } } + // Check as the inner reference type if it is a field projection + // from the `&pin` pattern + ProjectionElem::Field(FieldIdx::ZERO, _) + if let Some(adt) = + place_base.ty(self.body(), self.infcx.tcx).ty.ty_adt_def() + && adt.is_pin() + && self.infcx.tcx.features().pin_ergonomics() => + { + self.is_mutable(place_base, is_local_mutation_allowed) + } // All other projections are owned by their base path, so mutable if // base path is mutable ProjectionElem::Field(..) | ProjectionElem::Index(..) | ProjectionElem::ConstantIndex { .. } | ProjectionElem::Subslice { .. } - | ProjectionElem::Subtype(..) | ProjectionElem::OpaqueCast { .. } | ProjectionElem::Downcast(..) | ProjectionElem::UnwrapUnsafeBinder(_) => { diff --git a/compiler/rustc_borrowck/src/member_constraints.rs b/compiler/rustc_borrowck/src/member_constraints.rs deleted file mode 100644 index bdd0f6fe11e0..000000000000 --- a/compiler/rustc_borrowck/src/member_constraints.rs +++ /dev/null @@ -1,226 +0,0 @@ -use std::hash::Hash; -use std::ops::Index; - -use rustc_data_structures::fx::FxIndexMap; -use rustc_index::{IndexSlice, IndexVec}; -use rustc_middle::ty::{self, Ty}; -use rustc_span::Span; -use tracing::instrument; - -/// Compactly stores a set of `R0 member of [R1...Rn]` constraints, -/// indexed by the region `R0`. -#[derive(Debug)] -pub(crate) struct MemberConstraintSet<'tcx, R> -where - R: Copy + Eq, -{ - /// Stores the first "member" constraint for a given `R0`. This is an - /// index into the `constraints` vector below. - first_constraints: FxIndexMap, - - /// Stores the data about each `R0 member of [R1..Rn]` constraint. - /// These are organized into a linked list, so each constraint - /// contains the index of the next constraint with the same `R0`. - constraints: IndexVec>, - - /// Stores the `R1..Rn` regions for *all* sets. For any given - /// constraint, we keep two indices so that we can pull out a - /// slice. - choice_regions: Vec, -} - -/// Represents a `R0 member of [R1..Rn]` constraint -#[derive(Debug)] -pub(crate) struct MemberConstraint<'tcx> { - next_constraint: Option, - - /// The span where the hidden type was instantiated. - pub(crate) definition_span: Span, - - /// The hidden type in which `R0` appears. (Used in error reporting.) - pub(crate) hidden_ty: Ty<'tcx>, - - pub(crate) key: ty::OpaqueTypeKey<'tcx>, - - /// The region `R0`. - pub(crate) member_region_vid: ty::RegionVid, - - /// Index of `R1` in `choice_regions` vector from `MemberConstraintSet`. - start_index: usize, - - /// Index of `Rn` in `choice_regions` vector from `MemberConstraintSet`. - end_index: usize, -} - -rustc_index::newtype_index! { - #[debug_format = "MemberConstraintIndex({})"] - pub(crate) struct NllMemberConstraintIndex {} -} - -impl Default for MemberConstraintSet<'_, ty::RegionVid> { - fn default() -> Self { - Self { - first_constraints: Default::default(), - constraints: Default::default(), - choice_regions: Default::default(), - } - } -} - -impl<'tcx> MemberConstraintSet<'tcx, ty::RegionVid> { - pub(crate) fn is_empty(&self) -> bool { - self.constraints.is_empty() - } - - /// Pushes a member constraint into the set. - #[instrument(level = "debug", skip(self))] - pub(crate) fn add_member_constraint( - &mut self, - key: ty::OpaqueTypeKey<'tcx>, - hidden_ty: Ty<'tcx>, - definition_span: Span, - member_region_vid: ty::RegionVid, - choice_regions: &[ty::RegionVid], - ) { - let next_constraint = self.first_constraints.get(&member_region_vid).cloned(); - let start_index = self.choice_regions.len(); - self.choice_regions.extend(choice_regions); - let end_index = self.choice_regions.len(); - let constraint_index = self.constraints.push(MemberConstraint { - next_constraint, - member_region_vid, - definition_span, - hidden_ty, - key, - start_index, - end_index, - }); - self.first_constraints.insert(member_region_vid, constraint_index); - } -} - -impl<'tcx, R1> MemberConstraintSet<'tcx, R1> -where - R1: Copy + Hash + Eq, -{ - /// Remap the "member region" key using `map_fn`, producing a new - /// member constraint set. This is used in the NLL code to map from - /// the original `RegionVid` to an scc index. In some cases, we - /// may have multiple `R1` values mapping to the same `R2` key -- that - /// is ok, the two sets will be merged. - pub(crate) fn into_mapped( - self, - mut map_fn: impl FnMut(R1) -> R2, - ) -> MemberConstraintSet<'tcx, R2> - where - R2: Copy + Hash + Eq, - { - // We can re-use most of the original data, just tweaking the - // linked list links a bit. - // - // For example if we had two keys `Ra` and `Rb` that both now - // wind up mapped to the same key `S`, we would append the - // linked list for `Ra` onto the end of the linked list for - // `Rb` (or vice versa) -- this basically just requires - // rewriting the final link from one list to point at the other - // other (see `append_list`). - - let MemberConstraintSet { first_constraints, mut constraints, choice_regions } = self; - - let mut first_constraints2 = FxIndexMap::default(); - first_constraints2.reserve(first_constraints.len()); - - for (r1, start1) in first_constraints { - let r2 = map_fn(r1); - if let Some(&start2) = first_constraints2.get(&r2) { - append_list(&mut constraints, start1, start2); - } - first_constraints2.insert(r2, start1); - } - - MemberConstraintSet { first_constraints: first_constraints2, constraints, choice_regions } - } -} - -impl<'tcx, R> MemberConstraintSet<'tcx, R> -where - R: Copy + Hash + Eq, -{ - pub(crate) fn all_indices(&self) -> impl Iterator { - self.constraints.indices() - } - - /// Iterate down the constraint indices associated with a given - /// peek-region. You can then use `choice_regions` and other - /// methods to access data. - pub(crate) fn indices( - &self, - member_region_vid: R, - ) -> impl Iterator { - let mut next = self.first_constraints.get(&member_region_vid).cloned(); - std::iter::from_fn(move || -> Option { - if let Some(current) = next { - next = self.constraints[current].next_constraint; - Some(current) - } else { - None - } - }) - } - - /// Returns the "choice regions" for a given member - /// constraint. This is the `R1..Rn` from a constraint like: - /// - /// ```text - /// R0 member of [R1..Rn] - /// ``` - pub(crate) fn choice_regions(&self, pci: NllMemberConstraintIndex) -> &[ty::RegionVid] { - let MemberConstraint { start_index, end_index, .. } = &self.constraints[pci]; - &self.choice_regions[*start_index..*end_index] - } -} - -impl<'tcx, R> Index for MemberConstraintSet<'tcx, R> -where - R: Copy + Eq, -{ - type Output = MemberConstraint<'tcx>; - - fn index(&self, i: NllMemberConstraintIndex) -> &MemberConstraint<'tcx> { - &self.constraints[i] - } -} - -/// Given a linked list starting at `source_list` and another linked -/// list starting at `target_list`, modify `target_list` so that it is -/// followed by `source_list`. -/// -/// Before: -/// -/// ```text -/// target_list: A -> B -> C -> (None) -/// source_list: D -> E -> F -> (None) -/// ``` -/// -/// After: -/// -/// ```text -/// target_list: A -> B -> C -> D -> E -> F -> (None) -/// ``` -fn append_list( - constraints: &mut IndexSlice>, - target_list: NllMemberConstraintIndex, - source_list: NllMemberConstraintIndex, -) { - let mut p = target_list; - loop { - let r = &mut constraints[p]; - match r.next_constraint { - Some(q) => p = q, - None => { - r.next_constraint = Some(source_list); - return; - } - } - } -} diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 41f67e78930f..5537d90e297a 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -5,10 +5,11 @@ use std::path::PathBuf; use std::rc::Rc; use std::str::FromStr; -use polonius_engine::{Algorithm, Output}; +use polonius_engine::{Algorithm, AllFacts, Output}; +use rustc_data_structures::frozen::Frozen; use rustc_index::IndexSlice; -use rustc_middle::mir::pretty::{PrettyPrintMirOptions, dump_mir_with_options}; -use rustc_middle::mir::{Body, PassWhere, Promoted, create_dump_file, dump_enabled, dump_mir}; +use rustc_middle::mir::pretty::PrettyPrintMirOptions; +use rustc_middle::mir::{Body, MirDumper, PassWhere, Promoted}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, TyCtxt}; use rustc_mir_dataflow::move_paths::MoveData; @@ -18,14 +19,16 @@ use rustc_span::sym; use tracing::{debug, instrument}; use crate::borrow_set::BorrowSet; +use crate::consumers::RustcFacts; use crate::diagnostics::RegionErrors; use crate::handle_placeholders::compute_sccs_applying_placeholder_outlives_constraints; -use crate::polonius::PoloniusDiagnosticsContext; use crate::polonius::legacy::{ PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput, }; +use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext}; use crate::region_infer::RegionInferenceContext; -use crate::type_check::{self, MirTypeckResults}; +use crate::type_check::MirTypeckRegionConstraints; +use crate::type_check::free_region_relations::UniversalRegionRelations; use crate::universal_regions::UniversalRegions; use crate::{ BorrowCheckRootCtxt, BorrowckInferCtxt, ClosureOutlivesSubject, ClosureRegionRequirements, @@ -65,52 +68,63 @@ pub(crate) fn replace_regions_in_mir<'tcx>( // Replace all remaining regions with fresh inference variables. renumber::renumber_mir(infcx, body, promoted); - dump_mir(infcx.tcx, false, "renumber", &0, body, |_, _| Ok(())); + if let Some(dumper) = MirDumper::new(infcx.tcx, "renumber", body) { + dumper.dump_mir(body); + } universal_regions } +/// Computes the closure requirements given the current inference state. +/// +/// This is intended to be used by before [BorrowCheckRootCtxt::handle_opaque_type_uses] +/// because applying member constraints may rely on closure requirements. +/// This is frequently the case of async functions where pretty much everything +/// happens inside of the inner async block but the opaque only gets constrained +/// in the parent function. +pub(crate) fn compute_closure_requirements_modulo_opaques<'tcx>( + infcx: &BorrowckInferCtxt<'tcx>, + body: &Body<'tcx>, + location_map: Rc, + universal_region_relations: &Frozen>, + constraints: &MirTypeckRegionConstraints<'tcx>, +) -> Option> { + // FIXME(#146079): we shouldn't have to clone all this stuff here. + // Computing the region graph should take at least some of it by reference/`Rc`. + let lowered_constraints = compute_sccs_applying_placeholder_outlives_constraints( + constraints.clone(), + &universal_region_relations, + infcx, + ); + let mut regioncx = RegionInferenceContext::new( + &infcx, + lowered_constraints, + universal_region_relations.clone(), + location_map, + ); + + let (closure_region_requirements, _nll_errors) = regioncx.solve(infcx, body, None); + closure_region_requirements +} + /// Computes the (non-lexical) regions from the input MIR. /// /// This may result in errors being reported. pub(crate) fn compute_regions<'tcx>( root_cx: &mut BorrowCheckRootCtxt<'tcx>, infcx: &BorrowckInferCtxt<'tcx>, - universal_regions: UniversalRegions<'tcx>, body: &Body<'tcx>, - promoted: &IndexSlice>, location_table: &PoloniusLocationTable, move_data: &MoveData<'tcx>, borrow_set: &BorrowSet<'tcx>, + location_map: Rc, + universal_region_relations: Frozen>, + constraints: MirTypeckRegionConstraints<'tcx>, + mut polonius_facts: Option>, + polonius_context: Option, ) -> NllOutput<'tcx> { - let is_polonius_legacy_enabled = infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); - let polonius_input = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_input()) - || is_polonius_legacy_enabled; let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output()) - || is_polonius_legacy_enabled; - let mut polonius_facts = - (polonius_input || PoloniusFacts::enabled(infcx.tcx)).then_some(PoloniusFacts::default()); - - let location_map = Rc::new(DenseLocationMap::new(body)); - - // Run the MIR type-checker. - let MirTypeckResults { - constraints, - universal_region_relations, - opaque_type_values, - polonius_context, - } = type_check::type_check( - root_cx, - infcx, - body, - promoted, - universal_regions, - location_table, - borrow_set, - &mut polonius_facts, - move_data, - Rc::clone(&location_map), - ); + || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); let lowered_constraints = compute_sccs_applying_placeholder_outlives_constraints( constraints, @@ -168,13 +182,6 @@ pub(crate) fn compute_regions<'tcx>( let (closure_region_requirements, nll_errors) = regioncx.solve(infcx, body, polonius_output.clone()); - if let Some(guar) = nll_errors.has_errors() { - // Suppress unhelpful extra errors in `infer_opaque_types`. - infcx.set_tainted_by_errors(guar); - } - - regioncx.infer_opaque_types(root_cx, infcx, opaque_type_values); - NllOutput { regioncx, polonius_input: polonius_facts.map(Box::new), @@ -202,9 +209,7 @@ pub(super) fn dump_nll_mir<'tcx>( borrow_set: &BorrowSet<'tcx>, ) { let tcx = infcx.tcx; - if !dump_enabled(tcx, "nll", body.source.def_id()) { - return; - } + let Some(dumper) = MirDumper::new(tcx, "nll", body) else { return }; // We want the NLL extra comments printed by default in NLL MIR dumps (they were removed in // #112346). Specifying `-Z mir-include-spans` on the CLI still has priority: for example, @@ -215,27 +220,24 @@ pub(super) fn dump_nll_mir<'tcx>( MirIncludeSpans::On | MirIncludeSpans::Nll ), }; - dump_mir_with_options( - tcx, - false, - "nll", - &0, - body, - |pass_where, out| { - emit_nll_mir(tcx, regioncx, closure_region_requirements, borrow_set, pass_where, out) - }, - options, - ); + + let extra_data = &|pass_where, out: &mut dyn std::io::Write| { + emit_nll_mir(tcx, regioncx, closure_region_requirements, borrow_set, pass_where, out) + }; + + let dumper = dumper.set_extra_data(extra_data).set_options(options); + + dumper.dump_mir(body); // Also dump the region constraint graph as a graphviz file. let _: io::Result<()> = try { - let mut file = create_dump_file(tcx, "regioncx.all.dot", false, "nll", &0, body)?; + let mut file = dumper.create_dump_file("regioncx.all.dot", body)?; regioncx.dump_graphviz_raw_constraints(tcx, &mut file)?; }; // Also dump the region constraint SCC graph as a graphviz file. let _: io::Result<()> = try { - let mut file = create_dump_file(tcx, "regioncx.scc.dot", false, "nll", &0, body)?; + let mut file = dumper.create_dump_file("regioncx.scc.dot", body)?; regioncx.dump_graphviz_scc_constraints(tcx, &mut file)?; }; } diff --git a/compiler/rustc_borrowck/src/places_conflict.rs b/compiler/rustc_borrowck/src/places_conflict.rs index cf3e82426e8a..60676ac6b864 100644 --- a/compiler/rustc_borrowck/src/places_conflict.rs +++ b/compiler/rustc_borrowck/src/places_conflict.rs @@ -249,7 +249,6 @@ fn place_components_conflict<'tcx>( | (ProjectionElem::ConstantIndex { .. }, _, _) | (ProjectionElem::Subslice { .. }, _, _) | (ProjectionElem::OpaqueCast { .. }, _, _) - | (ProjectionElem::Subtype(_), _, _) | (ProjectionElem::Downcast { .. }, _, _) | (ProjectionElem::UnwrapUnsafeBinder(_), _, _) => { // Recursive case. This can still be disjoint on a @@ -510,7 +509,6 @@ fn place_projection_conflict<'tcx>( | ProjectionElem::Field(..) | ProjectionElem::Index(..) | ProjectionElem::ConstantIndex { .. } - | ProjectionElem::Subtype(_) | ProjectionElem::OpaqueCast { .. } | ProjectionElem::Subslice { .. } | ProjectionElem::Downcast(..), diff --git a/compiler/rustc_borrowck/src/polonius/constraints.rs b/compiler/rustc_borrowck/src/polonius/constraints.rs index 50f59dd0dee6..525957578595 100644 --- a/compiler/rustc_borrowck/src/polonius/constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/constraints.rs @@ -7,9 +7,7 @@ use rustc_mir_dataflow::points::PointIndex; /// /// This models two sources of constraints: /// - constraints that traverse the subsets between regions at a given point, `a@p: b@p`. These -/// depend on typeck constraints generated via assignments, calls, etc. (In practice there are -/// subtleties where a statement's effect only starts being visible at the successor point, via -/// the "result" of that statement). +/// depend on typeck constraints generated via assignments, calls, etc. /// - constraints that traverse the CFG via the same region, `a@p: a@q`, where `p` is a predecessor /// of `q`. These depend on the liveness of the regions at these points, as well as their /// variance. diff --git a/compiler/rustc_borrowck/src/polonius/dump.rs b/compiler/rustc_borrowck/src/polonius/dump.rs index 6b13b5ad0814..62f9ae173474 100644 --- a/compiler/rustc_borrowck/src/polonius/dump.rs +++ b/compiler/rustc_borrowck/src/polonius/dump.rs @@ -2,9 +2,7 @@ use std::io; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_index::IndexVec; -use rustc_middle::mir::pretty::{ - PassWhere, PrettyPrintMirOptions, create_dump_file, dump_enabled, dump_mir_to_writer, -}; +use rustc_middle::mir::pretty::{MirDumper, PassWhere, PrettyPrintMirOptions}; use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::{RegionVid, TyCtxt}; use rustc_mir_dataflow::points::PointIndex; @@ -33,22 +31,41 @@ pub(crate) fn dump_polonius_mir<'tcx>( return; } - if !dump_enabled(tcx, "polonius", body.source.def_id()) { - return; - } + let Some(dumper) = MirDumper::new(tcx, "polonius", body) else { return }; let polonius_diagnostics = polonius_diagnostics.expect("missing diagnostics context with `-Zpolonius=next`"); - let _: io::Result<()> = try { - let mut file = create_dump_file(tcx, "html", false, "polonius", &0, body)?; - emit_polonius_dump( + let extra_data = &|pass_where, out: &mut dyn io::Write| { + emit_polonius_mir( tcx, + regioncx, + closure_region_requirements, + borrow_set, + &polonius_diagnostics.localized_outlives_constraints, + pass_where, + out, + ) + }; + // We want the NLL extra comments printed by default in NLL MIR dumps. Specifying `-Z + // mir-include-spans` on the CLI still has priority. + let options = PrettyPrintMirOptions { + include_extra_comments: matches!( + tcx.sess.opts.unstable_opts.mir_include_spans, + MirIncludeSpans::On | MirIncludeSpans::Nll + ), + }; + + let dumper = dumper.set_extra_data(extra_data).set_options(options); + + let _: io::Result<()> = try { + let mut file = dumper.create_dump_file("html", body)?; + emit_polonius_dump( + &dumper, body, regioncx, borrow_set, &polonius_diagnostics.localized_outlives_constraints, - closure_region_requirements, &mut file, )?; }; @@ -61,12 +78,11 @@ pub(crate) fn dump_polonius_mir<'tcx>( /// - a mermaid graph of the NLL regions and the constraints between them /// - a mermaid graph of the NLL SCCs and the constraints between them fn emit_polonius_dump<'tcx>( - tcx: TyCtxt<'tcx>, + dumper: &MirDumper<'_, '_, 'tcx>, body: &Body<'tcx>, regioncx: &RegionInferenceContext<'tcx>, borrow_set: &BorrowSet<'tcx>, localized_outlives_constraints: &LocalizedOutlivesConstraintSet, - closure_region_requirements: &Option>, out: &mut dyn io::Write, ) -> io::Result<()> { // Prepare the HTML dump file prologue. @@ -79,15 +95,7 @@ fn emit_polonius_dump<'tcx>( writeln!(out, "
")?; writeln!(out, "Raw MIR dump")?; writeln!(out, "
")?;
-    emit_html_mir(
-        tcx,
-        body,
-        regioncx,
-        borrow_set,
-        &localized_outlives_constraints,
-        closure_region_requirements,
-        out,
-    )?;
+    emit_html_mir(dumper, body, out)?;
     writeln!(out, "
")?; writeln!(out, "
")?; @@ -116,7 +124,7 @@ fn emit_polonius_dump<'tcx>( writeln!(out, "
")?; writeln!(out, "NLL regions")?; writeln!(out, "
")?;
-    emit_mermaid_nll_regions(tcx, regioncx, out)?;
+    emit_mermaid_nll_regions(dumper.tcx(), regioncx, out)?;
     writeln!(out, "
")?; writeln!(out, "
")?; @@ -124,7 +132,7 @@ fn emit_polonius_dump<'tcx>( writeln!(out, "
")?; writeln!(out, "NLL SCCs")?; writeln!(out, "
")?;
-    emit_mermaid_nll_sccs(tcx, regioncx, out)?;
+    emit_mermaid_nll_sccs(dumper.tcx(), regioncx, out)?;
     writeln!(out, "
")?; writeln!(out, "
")?; @@ -149,45 +157,14 @@ fn emit_polonius_dump<'tcx>( /// Emits the polonius MIR, as escaped HTML. fn emit_html_mir<'tcx>( - tcx: TyCtxt<'tcx>, + dumper: &MirDumper<'_, '_, 'tcx>, body: &Body<'tcx>, - regioncx: &RegionInferenceContext<'tcx>, - borrow_set: &BorrowSet<'tcx>, - localized_outlives_constraints: &LocalizedOutlivesConstraintSet, - closure_region_requirements: &Option>, out: &mut dyn io::Write, ) -> io::Result<()> { // Buffer the regular MIR dump to be able to escape it. let mut buffer = Vec::new(); - // We want the NLL extra comments printed by default in NLL MIR dumps. Specifying `-Z - // mir-include-spans` on the CLI still has priority. - let options = PrettyPrintMirOptions { - include_extra_comments: matches!( - tcx.sess.opts.unstable_opts.mir_include_spans, - MirIncludeSpans::On | MirIncludeSpans::Nll - ), - }; - - dump_mir_to_writer( - tcx, - "polonius", - &0, - body, - &mut buffer, - |pass_where, out| { - emit_polonius_mir( - tcx, - regioncx, - closure_region_requirements, - borrow_set, - localized_outlives_constraints, - pass_where, - out, - ) - }, - options, - )?; + dumper.dump_mir_to_writer(body, &mut buffer)?; // Escape the handful of characters that need it. We don't need to be particularly efficient: // we're actually writing into a buffered writer already. Note that MIR dumps are valid UTF-8. diff --git a/compiler/rustc_borrowck/src/polonius/legacy/facts.rs b/compiler/rustc_borrowck/src/polonius/legacy/facts.rs index 64389b11a651..1f8177477e68 100644 --- a/compiler/rustc_borrowck/src/polonius/legacy/facts.rs +++ b/compiler/rustc_borrowck/src/polonius/legacy/facts.rs @@ -184,22 +184,6 @@ where } } -impl FactRow for (A, B, C, D) -where - A: FactCell, - B: FactCell, - C: FactCell, - D: FactCell, -{ - fn write( - &self, - out: &mut dyn Write, - location_table: &PoloniusLocationTable, - ) -> Result<(), Box> { - write_row(out, location_table, &[&self.0, &self.1, &self.2, &self.3]) - } -} - fn write_row( out: &mut dyn Write, location_table: &PoloniusLocationTable, diff --git a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs index 99dd0b2dd466..722b59f145d9 100644 --- a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs +++ b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs @@ -84,7 +84,6 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> { StatementKind::ConstEvalCounter | StatementKind::Nop | StatementKind::Retag { .. } - | StatementKind::Deinit(..) | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") @@ -301,21 +300,11 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> { | Rvalue::Cast(_ /*cast_kind*/, operand, _ /*ty*/) | Rvalue::ShallowInitBox(operand, _ /*ty*/) => self.consume_operand(location, operand), - &Rvalue::CopyForDeref(place) => { - let op = &Operand::Copy(place); - self.consume_operand(location, op); - } - - &(Rvalue::Len(place) | Rvalue::Discriminant(place)) => { - let af = match rvalue { - Rvalue::Len(..) => Some(ArtificialField::ArrayLength), - Rvalue::Discriminant(..) => None, - _ => unreachable!(), - }; + &Rvalue::Discriminant(place) => { self.access_place( location, place, - (Shallow(af), Read(ReadKind::Copy)), + (Shallow(None), Read(ReadKind::Copy)), LocalMutationIsAllowed::No, ); } @@ -336,6 +325,8 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> { Rvalue::WrapUnsafeBinder(op, _) => { self.consume_operand(location, op); } + + Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in borrowck"), } } diff --git a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs index 6ab09f731c07..2ba72180d66a 100644 --- a/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/liveness_constraints.rs @@ -105,22 +105,14 @@ fn propagate_loans_between_points( }); } - let Some(current_live_regions) = live_regions.row(current_point) else { - // There are no constraints to add: there are no live regions at the current point. - return; - }; let Some(next_live_regions) = live_regions.row(next_point) else { // There are no constraints to add: there are no live regions at the next point. return; }; for region in next_live_regions.iter() { - if !current_live_regions.contains(region) { - continue; - } - - // `region` is indeed live at both points, add a constraint between them, according to - // variance. + // `region` could be live at the current point, and is live at the next point: add a + // constraint between them, according to variance. if let Some(&direction) = live_region_variances.get(®ion) { add_liveness_constraint( region, diff --git a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs index 5cd265e0db92..bdc3047e5ba0 100644 --- a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs +++ b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs @@ -1,27 +1,18 @@ -use std::collections::{BTreeMap, BTreeSet}; - use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{ - Body, Local, Location, Place, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, -}; -use rustc_middle::ty::{RegionVid, TyCtxt}; +use rustc_middle::ty::RegionVid; use rustc_mir_dataflow::points::PointIndex; use super::{LiveLoans, LocalizedOutlivesConstraintSet}; +use crate::BorrowSet; use crate::constraints::OutlivesConstraint; -use crate::dataflow::BorrowIndex; use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; -use crate::{BorrowSet, PlaceConflictBias, places_conflict}; -/// Compute loan reachability, stop at kills, and trace loan liveness throughout the CFG, by +/// Compute loan reachability to approximately trace loan liveness throughout the CFG, by /// traversing the full graph of constraints that combines: /// - the localized constraints (the physical edges), /// - with the constraints that hold at all points (the logical edges). pub(super) fn compute_loan_liveness<'tcx>( - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, liveness: &LivenessValues, outlives_constraints: impl Iterator>, borrow_set: &BorrowSet<'tcx>, @@ -29,11 +20,6 @@ pub(super) fn compute_loan_liveness<'tcx>( ) -> LiveLoans { let mut live_loans = LiveLoans::new(borrow_set.len()); - // FIXME: it may be preferable for kills to be encoded in the edges themselves, to simplify and - // likely make traversal (and constraint generation) more efficient. We also display kills on - // edges when visualizing the constraint graph anyways. - let kills = collect_kills(body, tcx, borrow_set); - // Create the full graph with the physical edges we've localized earlier, and the logical edges // of constraints that hold at all points. let logical_constraints = @@ -59,15 +45,15 @@ pub(super) fn compute_loan_liveness<'tcx>( continue; } - // Record the loan as being live on entry to this point. - live_loans.insert(node.point, loan_idx); - - // Here, we have a conundrum. There's currently a weakness in our theory, in that - // we're using a single notion of reachability to represent what used to be _two_ - // different transitive closures. It didn't seem impactful when coming up with the - // single-graph and reachability through space (regions) + time (CFG) concepts, but in - // practice the combination of time-traveling with kills is more impactful than - // initially anticipated. + // Record the loan as being live on entry to this point if it reaches a live region + // there. + // + // This is an approximation of liveness (which is the thing we want), in that we're + // using a single notion of reachability to represent what used to be _two_ different + // transitive closures. It didn't seem impactful when coming up with the single-graph + // and reachability through space (regions) + time (CFG) concepts, but in practice the + // combination of time-traveling with kills is more impactful than initially + // anticipated. // // Kills should prevent a loan from reaching its successor points in the CFG, but not // while time-traveling: we're not actually at that CFG point, but looking for @@ -92,40 +78,20 @@ pub(super) fn compute_loan_liveness<'tcx>( // two-step traversal described above: only kills encountered on exit via a backward // edge are ignored. // - // In our test suite, there are a couple of cases where kills are encountered while - // time-traveling, however as far as we can tell, always in cases where they would be - // unreachable. We have reason to believe that this is a property of the single-graph - // approach (but haven't proved it yet): - // - reachable kills while time-traveling would also be encountered via regular - // traversal - // - it makes _some_ sense to ignore unreachable kills, but subtleties around dead code - // in general need to be better thought through (like they were for NLLs). - // - ignoring kills is a conservative approximation: the loan is still live and could - // cause false positive errors at another place access. Soundness issues in this - // domain should look more like the absence of reachability instead. + // This version of the analysis, however, is enough in practice to pass the tests that + // we care about and NLLs reject, without regressions on crater, and is an actionable + // subset of the full analysis. It also naturally points to areas of improvement that we + // wish to explore later, namely handling kills appropriately during traversal, instead + // of continuing traversal to all the reachable nodes. // - // This is enough in practice to pass tests, and therefore is what we have implemented - // for now. - // - // FIXME: all of the above. Analyze potential unsoundness, possibly in concert with a - // borrowck implementation in a-mir-formality, fuzzing, or manually crafting - // counter-examples. + // FIXME: analyze potential unsoundness, possibly in concert with a borrowck + // implementation in a-mir-formality, fuzzing, or manually crafting counter-examples. - // Continuing traversal will depend on whether the loan is killed at this point, and - // whether we're time-traveling. - let current_location = liveness.location_from_point(node.point); - let is_loan_killed = - kills.get(¤t_location).is_some_and(|kills| kills.contains(&loan_idx)); + if liveness.is_live_at(node.region, liveness.location_from_point(node.point)) { + live_loans.insert(node.point, loan_idx); + } for succ in graph.outgoing_edges(node) { - // If the loan is killed at this point, it is killed _on exit_. But only during - // forward traversal. - if is_loan_killed { - let destination = liveness.location_from_point(succ.point); - if current_location.is_predecessor_of(destination, body) { - continue; - } - } stack.push(succ); } } @@ -192,116 +158,3 @@ impl LocalizedConstraintGraph { physical_edges.chain(materialized_edges) } } - -/// Traverses the MIR and collects kills. -fn collect_kills<'tcx>( - body: &Body<'tcx>, - tcx: TyCtxt<'tcx>, - borrow_set: &BorrowSet<'tcx>, -) -> BTreeMap> { - let mut collector = KillsCollector { borrow_set, tcx, body, kills: BTreeMap::default() }; - for (block, data) in body.basic_blocks.iter_enumerated() { - collector.visit_basic_block_data(block, data); - } - collector.kills -} - -struct KillsCollector<'a, 'tcx> { - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - borrow_set: &'a BorrowSet<'tcx>, - - /// The set of loans killed at each location. - kills: BTreeMap>, -} - -// This visitor has a similar structure to the `Borrows` dataflow computation with respect to kills, -// and the datalog polonius fact generation for the `loan_killed_at` relation. -impl<'tcx> KillsCollector<'_, 'tcx> { - /// Records the borrows on the specified place as `killed`. For example, when assigning to a - /// local, or on a call's return destination. - fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) { - // For the reasons described in graph traversal, we also filter out kills - // unreachable from the loan's introduction point, as they would stop traversal when - // e.g. checking for reachability in the subset graph through invariance constraints - // higher up. - let filter_unreachable_kills = |loan| { - let introduction = self.borrow_set[loan].reserve_location; - let reachable = introduction.is_predecessor_of(location, self.body); - reachable - }; - - let other_borrows_of_local = self - .borrow_set - .local_map - .get(&place.local) - .into_iter() - .flat_map(|bs| bs.iter()) - .copied(); - - // If the borrowed place is a local with no projections, all other borrows of this - // local must conflict. This is purely an optimization so we don't have to call - // `places_conflict` for every borrow. - if place.projection.is_empty() { - if !self.body.local_decls[place.local].is_ref_to_static() { - self.kills - .entry(location) - .or_default() - .extend(other_borrows_of_local.filter(|&loan| filter_unreachable_kills(loan))); - } - return; - } - - // By passing `PlaceConflictBias::NoOverlap`, we conservatively assume that any given - // pair of array indices are not equal, so that when `places_conflict` returns true, we - // will be assured that two places being compared definitely denotes the same sets of - // locations. - let definitely_conflicting_borrows = other_borrows_of_local - .filter(|&i| { - places_conflict( - self.tcx, - self.body, - self.borrow_set[i].borrowed_place, - place, - PlaceConflictBias::NoOverlap, - ) - }) - .filter(|&loan| filter_unreachable_kills(loan)); - - self.kills.entry(location).or_default().extend(definitely_conflicting_borrows); - } - - /// Records the borrows on the specified local as `killed`. - fn record_killed_borrows_for_local(&mut self, local: Local, location: Location) { - if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) { - self.kills.entry(location).or_default().extend(borrow_indices.iter()); - } - } -} - -impl<'tcx> Visitor<'tcx> for KillsCollector<'_, 'tcx> { - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { - // Make sure there are no remaining borrows for locals that have gone out of scope. - if let StatementKind::StorageDead(local) = statement.kind { - self.record_killed_borrows_for_local(local, location); - } - - self.super_statement(statement, location); - } - - fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { - // When we see `X = ...`, then kill borrows of `(*X).foo` and so forth. - self.record_killed_borrows_for_place(*place, location); - self.super_assign(place, rvalue, location); - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { - // A `Call` terminator's return value can be a local which has borrows, so we need to record - // those as killed as well. - if let TerminatorKind::Call { destination, .. } = terminator.kind { - self.record_killed_borrows_for_place(destination, location); - } - - self.super_terminator(terminator, location); - } -} diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 142ef8ba28ef..a9092b1981e1 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -146,8 +146,8 @@ impl PoloniusContext { /// - converting NLL typeck constraints to be localized /// - encoding liveness constraints /// - /// Then, this graph is traversed, and combined with kills, reachability is recorded as loan - /// liveness, to be used by the loan scope and active loans computations. + /// Then, this graph is traversed, reachability is recorded as loan liveness, to be used by the + /// loan scope and active loans computations. /// /// The constraint data will be used to compute errors and diagnostics. pub(crate) fn compute_loan_liveness<'tcx>( @@ -182,8 +182,6 @@ impl PoloniusContext { // Now that we have a complete graph, we can compute reachability to trace the liveness of // loans for the next step in the chain, the NLL loan scope and active loans computations. let live_loans = compute_loan_liveness( - tcx, - body, regioncx.liveness_constraints(), regioncx.outlives_constraints(), borrow_set, diff --git a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs index 1289b1899eb3..e4e52962bf7f 100644 --- a/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/typeck_constraints.rs @@ -47,9 +47,7 @@ pub(super) fn convert_typeck_constraints<'tcx>( tcx, body, stmt, - liveness, &outlives_constraint, - location, point, universal_regions, ) @@ -78,9 +76,7 @@ fn localize_statement_constraint<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, stmt: &Statement<'tcx>, - liveness: &LivenessValues, outlives_constraint: &OutlivesConstraint<'tcx>, - current_location: Location, current_point: PointIndex, universal_regions: &UniversalRegions<'tcx>, ) -> LocalizedOutlivesConstraint { @@ -98,8 +94,8 @@ fn localize_statement_constraint<'tcx>( // - and that should be impossible in MIR // // When we have a more complete implementation in the future, tested with crater, etc, - // we can relax this to a debug assert instead, or remove it. - assert!( + // we can remove this assertion. It's a debug assert because it can be expensive. + debug_assert!( { let mut lhs_regions = FxHashSet::default(); tcx.for_each_free_region(lhs, |region| { @@ -119,16 +115,8 @@ fn localize_statement_constraint<'tcx>( "there should be no common regions between the LHS and RHS of an assignment" ); - // As mentioned earlier, we should be tracking these better upstream but: we want to - // relate the types on entry to the type of the place on exit. That is, outlives - // constraints on the RHS are on entry, and outlives constraints to/from the LHS are on - // exit (i.e. on entry to the successor location). let lhs_ty = body.local_decls[lhs.local].ty; - let successor_location = Location { - block: current_location.block, - statement_index: current_location.statement_index + 1, - }; - let successor_point = liveness.point_from_location(successor_location); + let successor_point = current_point; compute_constraint_direction( tcx, outlives_constraint, @@ -195,6 +183,7 @@ fn localize_terminator_constraint<'tcx>( } } } + /// For a given outlives constraint and CFG edge, returns the localized constraint with the /// appropriate `from`-`to` direction. This is computed according to whether the constraint flows to /// or from a free region in the given `value`, some kind of result for an effectful operation, like diff --git a/compiler/rustc_borrowck/src/prefixes.rs b/compiler/rustc_borrowck/src/prefixes.rs index 83cca38a5c09..9e51264d8edc 100644 --- a/compiler/rustc_borrowck/src/prefixes.rs +++ b/compiler/rustc_borrowck/src/prefixes.rs @@ -77,9 +77,6 @@ impl<'tcx> Iterator for Prefixes<'tcx> { | ProjectionElem::Index(_) => { cursor = cursor_base; } - ProjectionElem::Subtype(..) => { - panic!("Subtype projection is not allowed before borrow check") - } ProjectionElem::Deref => { match self.kind { PrefixSet::Shallow => { diff --git a/compiler/rustc_borrowck/src/region_infer/graphviz.rs b/compiler/rustc_borrowck/src/region_infer/graphviz.rs index a3e29982e90f..526e1850c2ef 100644 --- a/compiler/rustc_borrowck/src/region_infer/graphviz.rs +++ b/compiler/rustc_borrowck/src/region_infer/graphviz.rs @@ -12,9 +12,13 @@ use rustc_middle::ty::UniverseIndex; use super::*; fn render_outlives_constraint(constraint: &OutlivesConstraint<'_>) -> String { - match constraint.locations { - Locations::All(_) => "All(...)".to_string(), - Locations::Single(loc) => format!("{loc:?}"), + if let ConstraintCategory::OutlivesUnnameablePlaceholder(unnameable) = constraint.category { + format!("{unnameable:?} unnameable") + } else { + match constraint.locations { + Locations::All(_) => "All(...)".to_string(), + Locations::Single(loc) => format!("{loc:?}"), + } } } @@ -41,7 +45,22 @@ fn render_region_vid<'tcx>( "".to_string() }; - format!("{:?}{universe_str}{external_name_str}", rvid) + let extra_info = match regioncx.region_definition(rvid).origin { + NllRegionVariableOrigin::FreeRegion => "".to_string(), + NllRegionVariableOrigin::Placeholder(p) => match p.bound.kind { + ty::BoundRegionKind::Named(def_id) => { + format!(" (for<{}>)", tcx.item_name(def_id)) + } + ty::BoundRegionKind::ClosureEnv | ty::BoundRegionKind::Anon => " (for<'_>)".to_string(), + ty::BoundRegionKind::NamedAnon(_) => { + bug!("only used for pretty printing") + } + }, + NllRegionVariableOrigin::Existential { name: Some(name), .. } => format!(" (ex<{name}>)"), + NllRegionVariableOrigin::Existential { .. } => format!(" (ex<'_>)"), + }; + + format!("{:?}{universe_str}{external_name_str}{extra_info}", rvid) } impl<'tcx> RegionInferenceContext<'tcx> { diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs index 68f1637e07ee..e98c60e63380 100644 --- a/compiler/rustc_borrowck/src/region_infer/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/mod.rs @@ -1,11 +1,9 @@ -use std::cell::OnceCell; use std::collections::VecDeque; use std::rc::Rc; -use rustc_data_structures::binary_search_util; use rustc_data_structures::frozen::Frozen; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; -use rustc_data_structures::graph::scc::{self, Sccs}; +use rustc_data_structures::graph::scc::Sccs; use rustc_errors::Diag; use rustc_hir::def_id::CRATE_DEF_ID; use rustc_index::IndexVec; @@ -24,15 +22,13 @@ use rustc_span::hygiene::DesugaringKind; use rustc_span::{DUMMY_SP, Span}; use tracing::{Level, debug, enabled, instrument, trace}; -use crate::constraints::graph::{self, NormalConstraintGraph, RegionGraph}; +use crate::constraints::graph::NormalConstraintGraph; use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstraintSet}; use crate::dataflow::BorrowIndex; use crate::diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo}; use crate::handle_placeholders::{LoweredConstraints, RegionTracker}; -use crate::member_constraints::{MemberConstraintSet, NllMemberConstraintIndex}; use crate::polonius::LiveLoans; use crate::polonius::legacy::PoloniusOutput; -use crate::region_infer::reverse_sccs::ReverseSccGraph; use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, ToElementIndex}; use crate::type_check::Locations; use crate::type_check::free_region_relations::UniversalRegionRelations; @@ -44,7 +40,7 @@ use crate::{ mod dump_mir; mod graphviz; -mod opaque_types; +pub(crate) mod opaque_types; mod reverse_sccs; pub(crate) mod values; @@ -78,18 +74,6 @@ impl Representative { } } -impl scc::Annotation for Representative { - fn merge_scc(self, other: Self) -> Self { - // Just pick the smallest one. Note that we order by tag first! - std::cmp::min(self, other) - } - - // For reachability, we do nothing since the representative doesn't change. - fn merge_reached(self, _other: Self) -> Self { - self - } -} - pub(crate) type ConstraintSccs = Sccs; pub struct RegionInferenceContext<'tcx> { @@ -120,20 +104,6 @@ pub struct RegionInferenceContext<'tcx> { scc_annotations: IndexVec, - /// Reverse of the SCC constraint graph -- i.e., an edge `A -> B` exists if - /// `B: A`. This is used to compute the universal regions that are required - /// to outlive a given SCC. - rev_scc_graph: OnceCell, - - /// The "R0 member of [R1..Rn]" constraints, indexed by SCC. - member_constraints: Rc>, - - /// Records the member constraints that we applied to each scc. - /// This is useful for error reporting. Once constraint - /// propagation is done, this vector is sorted according to - /// `member_region_scc`. - member_constraints_applied: Vec, - /// Map universe indexes to information on why we created it. universe_causes: FxIndexMap>, @@ -150,32 +120,6 @@ pub struct RegionInferenceContext<'tcx> { universal_region_relations: Frozen>, } -/// Each time that `apply_member_constraint` is successful, it appends -/// one of these structs to the `member_constraints_applied` field. -/// This is used in error reporting to trace out what happened. -/// -/// The way that `apply_member_constraint` works is that it effectively -/// adds a new lower bound to the SCC it is analyzing: so you wind up -/// with `'R: 'O` where `'R` is the pick-region and `'O` is the -/// minimal viable option. -#[derive(Debug)] -pub(crate) struct AppliedMemberConstraint { - /// The SCC that was affected. (The "member region".) - /// - /// The vector if `AppliedMemberConstraint` elements is kept sorted - /// by this field. - pub(crate) member_region_scc: ConstraintSccIndex, - - /// The "best option" that `apply_member_constraint` found -- this was - /// added as an "ad-hoc" lower-bound to `member_region_scc`. - pub(crate) min_choice: ty::RegionVid, - - /// The "member constraint index" -- we can find out details about - /// the constraint from - /// `set.member_constraints[member_constraint_index]`. - pub(crate) member_constraint_index: NllMemberConstraintIndex, -} - #[derive(Debug)] pub(crate) struct RegionDefinition<'tcx> { /// What kind of variable is this -- a free region? existential @@ -268,7 +212,6 @@ enum Trace<'a, 'tcx> { StartRegion, FromGraph(&'a OutlivesConstraint<'tcx>), FromStatic(RegionVid), - FromMember(RegionVid, RegionVid, Span), NotVisited, } @@ -363,7 +306,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { liveness_constraints, universe_causes, placeholder_indices, - member_constraints, } = lowered_constraints; debug!("universal_regions: {:#?}", universal_region_relations.universal_regions); @@ -385,9 +327,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { scc_values.merge_liveness(scc, region, &liveness_constraints); } - let member_constraints = - Rc::new(member_constraints.into_mapped(|r| constraint_sccs.scc(r))); - let mut result = Self { definitions, liveness_constraints, @@ -395,9 +334,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { constraint_graph, constraint_sccs, scc_annotations, - rev_scc_graph: OnceCell::new(), - member_constraints, - member_constraints_applied: Vec::new(), universe_causes, scc_values, type_tests, @@ -417,7 +353,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { /// minimum values. /// /// For example: - /// ``` + /// ```ignore (illustrative) /// fn foo<'a, 'b>( /* ... */ ) where 'a: 'b { /* ... */ } /// ``` /// would initialize two variables like so: @@ -550,19 +486,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { self.scc_values.placeholders_contained_in(scc) } - /// Once region solving has completed, this function will return the member constraints that - /// were applied to the value of a given SCC `scc`. See `AppliedMemberConstraint`. - pub(crate) fn applied_member_constraints( - &self, - scc: ConstraintSccIndex, - ) -> &[AppliedMemberConstraint] { - binary_search_util::binary_search_slice( - &self.member_constraints_applied, - |applied| applied.member_region_scc, - &scc, - ) - } - /// Performs region inference and report errors if we see any /// unsatisfiable constraints. If this is a closure, returns the /// region requirements to propagate to our creator, if any. @@ -607,12 +530,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { debug!(?errors_buffer); - if errors_buffer.is_empty() { - self.check_member_constraints(infcx, &mut errors_buffer); - } - - debug!(?errors_buffer); - let outlives_requirements = outlives_requirements.unwrap_or_default(); if outlives_requirements.is_empty() { @@ -642,154 +559,27 @@ impl<'tcx> RegionInferenceContext<'tcx> { }); // To propagate constraints, we walk the DAG induced by the - // SCC. For each SCC, we visit its successors and compute + // SCC. For each SCC `A`, we visit its successors and compute // their values, then we union all those values to get our // own. - for scc in self.constraint_sccs.all_sccs() { - self.compute_value_for_scc(scc); - } - - // Sort the applied member constraints so we can binary search - // through them later. - self.member_constraints_applied.sort_by_key(|applied| applied.member_region_scc); - } - - /// Computes the value of the SCC `scc_a`, which has not yet been - /// computed, by unioning the values of its successors. - /// Assumes that all successors have been computed already - /// (which is assured by iterating over SCCs in dependency order). - #[instrument(skip(self), level = "debug")] - fn compute_value_for_scc(&mut self, scc_a: ConstraintSccIndex) { - // Walk each SCC `B` such that `A: B`... - for &scc_b in self.constraint_sccs.successors(scc_a) { - debug!(?scc_b); - self.scc_values.add_region(scc_a, scc_b); - } - - // Now take member constraints into account. - let member_constraints = Rc::clone(&self.member_constraints); - for m_c_i in member_constraints.indices(scc_a) { - self.apply_member_constraint(scc_a, m_c_i, member_constraints.choice_regions(m_c_i)); - } - - debug!(value = ?self.scc_values.region_value_str(scc_a)); - } - - /// Invoked for each `R0 member of [R1..Rn]` constraint. - /// - /// `scc` is the SCC containing R0, and `choice_regions` are the - /// `R1..Rn` regions -- they are always known to be universal - /// regions (and if that's not true, we just don't attempt to - /// enforce the constraint). - /// - /// The current value of `scc` at the time the method is invoked - /// is considered a *lower bound*. If possible, we will modify - /// the constraint to set it equal to one of the option regions. - /// If we make any changes, returns true, else false. - /// - /// This function only adds the member constraints to the region graph, - /// it does not check them. They are later checked in - /// `check_member_constraints` after the region graph has been computed. - #[instrument(skip(self, member_constraint_index), level = "debug")] - fn apply_member_constraint( - &mut self, - scc: ConstraintSccIndex, - member_constraint_index: NllMemberConstraintIndex, - choice_regions: &[ty::RegionVid], - ) { - // Create a mutable vector of the options. We'll try to winnow - // them down. - let mut choice_regions: Vec = choice_regions.to_vec(); - - // Convert to the SCC representative: sometimes we have inference - // variables in the member constraint that wind up equated with - // universal regions. The scc representative is the minimal numbered - // one from the corresponding scc so it will be the universal region - // if one exists. - for c_r in &mut choice_regions { - let scc = self.constraint_sccs.scc(*c_r); - *c_r = self.scc_representative(scc); - } - - // If the member region lives in a higher universe, we currently choose - // the most conservative option by leaving it unchanged. - if !self.max_nameable_universe(scc).is_root() { - return; - } - - // The existing value for `scc` is a lower-bound. This will - // consist of some set `{P} + {LB}` of points `{P}` and - // lower-bound free regions `{LB}`. As each choice region `O` - // is a free region, it will outlive the points. But we can - // only consider the option `O` if `O: LB`. - choice_regions.retain(|&o_r| { - self.scc_values - .universal_regions_outlived_by(scc) - .all(|lb| self.universal_region_relations.outlives(o_r, lb)) - }); - debug!(?choice_regions, "after lb"); - - // Now find all the *upper bounds* -- that is, each UB is a - // free region that must outlive the member region `R0` (`UB: - // R0`). Therefore, we need only keep an option `O` if `UB: O` - // for all UB. - let universal_region_relations = &self.universal_region_relations; - for ub in self.reverse_scc_graph().upper_bounds(scc) { - debug!(?ub); - choice_regions.retain(|&o_r| universal_region_relations.outlives(ub, o_r)); - } - debug!(?choice_regions, "after ub"); - - // At this point we can pick any member of `choice_regions` and would like to choose - // it to be a small as possible. To avoid potential non-determinism we will pick the - // smallest such choice. - // - // Because universal regions are only partially ordered (i.e, not every two regions are - // comparable), we will ignore any region that doesn't compare to all others when picking - // the minimum choice. - // - // For example, consider `choice_regions = ['static, 'a, 'b, 'c, 'd, 'e]`, where - // `'static: 'a, 'static: 'b, 'a: 'c, 'b: 'c, 'c: 'd, 'c: 'e`. - // `['d, 'e]` are ignored because they do not compare - the same goes for `['a, 'b]`. - let totally_ordered_subset = choice_regions.iter().copied().filter(|&r1| { - choice_regions.iter().all(|&r2| { - self.universal_region_relations.outlives(r1, r2) - || self.universal_region_relations.outlives(r2, r1) - }) - }); - // Now we're left with `['static, 'c]`. Pick `'c` as the minimum! - let Some(min_choice) = totally_ordered_subset.reduce(|r1, r2| { - let r1_outlives_r2 = self.universal_region_relations.outlives(r1, r2); - let r2_outlives_r1 = self.universal_region_relations.outlives(r2, r1); - match (r1_outlives_r2, r2_outlives_r1) { - (true, true) => r1.min(r2), - (true, false) => r2, - (false, true) => r1, - (false, false) => bug!("incomparable regions in total order"), + for scc_a in self.constraint_sccs.all_sccs() { + // Walk each SCC `B` such that `A: B`... + for &scc_b in self.constraint_sccs.successors(scc_a) { + debug!(?scc_b); + self.scc_values.add_region(scc_a, scc_b); } - }) else { - debug!("no unique minimum choice"); - return; - }; - - // As we require `'scc: 'min_choice`, we have definitely already computed - // its `scc_values` at this point. - let min_choice_scc = self.constraint_sccs.scc(min_choice); - debug!(?min_choice, ?min_choice_scc); - if self.scc_values.add_region(scc, min_choice_scc) { - self.member_constraints_applied.push(AppliedMemberConstraint { - member_region_scc: scc, - min_choice, - member_constraint_index, - }); } } - /// Returns `true` if all the elements in the value of `scc_b` are nameable + /// Returns `true` if all the placeholders in the value of `scc_b` are nameable /// in `scc_a`. Used during constraint propagation, and only once /// the value of `scc_b` has been computed. - fn universe_compatible(&self, scc_b: ConstraintSccIndex, scc_a: ConstraintSccIndex) -> bool { - self.scc_annotations[scc_a].universe_compatible_with(self.scc_annotations[scc_b]) + fn can_name_all_placeholders( + &self, + scc_a: ConstraintSccIndex, + scc_b: ConstraintSccIndex, + ) -> bool { + self.scc_annotations[scc_a].can_name_all_placeholders(self.scc_annotations[scc_b]) } /// Once regions have been propagated, this method is used to see @@ -829,7 +619,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { } // Type-test failed. Report the error. - let erased_generic_kind = infcx.tcx.erase_regions(type_test.generic_kind); + let erased_generic_kind = infcx.tcx.erase_and_anonymize_regions(type_test.generic_kind); // Skip duplicate-ish errors. if deduplicate_errors.insert(( @@ -1178,16 +968,22 @@ impl<'tcx> RegionInferenceContext<'tcx> { return true; } + let fr_static = self.universal_regions().fr_static; + // If we are checking that `'sup: 'sub`, and `'sub` contains // some placeholder that `'sup` cannot name, then this is only // true if `'sup` outlives static. - if !self.universe_compatible(sub_region_scc, sup_region_scc) { + // + // Avoid infinite recursion if `sub_region` is already `'static` + if sub_region != fr_static + && !self.can_name_all_placeholders(sup_region_scc, sub_region_scc) + { debug!( "sub universe `{sub_region_scc:?}` is not nameable \ by super `{sup_region_scc:?}`, promoting to static", ); - return self.eval_outlives(sup_region, self.universal_regions().fr_static); + return self.eval_outlives(sup_region, fr_static); } // Both the `sub_region` and `sup_region` consist of the union @@ -1487,11 +1283,9 @@ impl<'tcx> RegionInferenceContext<'tcx> { { debug!("try_propagate_universal_region_error: fr_minus={:?}", fr_minus); - let blame_span_category = self.find_outlives_blame_span( - longer_fr, - NllRegionVariableOrigin::FreeRegion, - shorter_fr, - ); + let blame_constraint = self + .best_blame_constraint(longer_fr, NllRegionVariableOrigin::FreeRegion, shorter_fr) + .0; // Grow `shorter_fr` until we find some non-local regions. (We // always will.) We'll call them `shorter_fr+` -- they're ever @@ -1504,8 +1298,8 @@ impl<'tcx> RegionInferenceContext<'tcx> { propagated_outlives_requirements.push(ClosureOutlivesRequirement { subject: ClosureOutlivesSubject::Region(fr_minus), outlived_free_region: fr, - blame_span: blame_span_category.1.span, - category: blame_span_category.0, + blame_span: blame_constraint.cause.span, + category: blame_constraint.category, }); } return RegionRelationCheckResult::Propagated; @@ -1544,118 +1338,67 @@ impl<'tcx> RegionInferenceContext<'tcx> { } } - #[instrument(level = "debug", skip(self, infcx, errors_buffer))] - fn check_member_constraints( + pub(crate) fn constraint_path_between_regions( &self, - infcx: &InferCtxt<'tcx>, - errors_buffer: &mut RegionErrors<'tcx>, - ) { - let member_constraints = Rc::clone(&self.member_constraints); - for m_c_i in member_constraints.all_indices() { - debug!(?m_c_i); - let m_c = &member_constraints[m_c_i]; - let member_region_vid = m_c.member_region_vid; - debug!( - ?member_region_vid, - value = ?self.region_value_str(member_region_vid), - ); - let choice_regions = member_constraints.choice_regions(m_c_i); - debug!(?choice_regions); - - // Did the member region wind up equal to any of the option regions? - if let Some(o) = - choice_regions.iter().find(|&&o_r| self.eval_equal(o_r, m_c.member_region_vid)) - { - debug!("evaluated as equal to {:?}", o); - continue; - } - - // If not, report an error. - let member_region = ty::Region::new_var(infcx.tcx, member_region_vid); - errors_buffer.push(RegionErrorKind::UnexpectedHiddenRegion { - span: m_c.definition_span, - hidden_ty: m_c.hidden_ty, - key: m_c.key, - member_region, - }); + from_region: RegionVid, + to_region: RegionVid, + ) -> Option>> { + if from_region == to_region { + bug!("Tried to find a path between {from_region:?} and itself!"); } - } - - /// We have a constraint `fr1: fr2` that is not satisfied, where - /// `fr2` represents some universal region. Here, `r` is some - /// region where we know that `fr1: r` and this function has the - /// job of determining whether `r` is "to blame" for the fact that - /// `fr1: fr2` is required. - /// - /// This is true under two conditions: - /// - /// - `r == fr2` - /// - `fr2` is `'static` and `r` is some placeholder in a universe - /// that cannot be named by `fr1`; in that case, we will require - /// that `fr1: 'static` because it is the only way to `fr1: r` to - /// be satisfied. (See `add_incompatible_universe`.) - pub(crate) fn provides_universal_region( - &self, - r: RegionVid, - fr1: RegionVid, - fr2: RegionVid, - ) -> bool { - debug!("provides_universal_region(r={:?}, fr1={:?}, fr2={:?})", r, fr1, fr2); - let result = { - r == fr2 || { - fr2 == self.universal_regions().fr_static && self.cannot_name_placeholder(fr1, r) - } - }; - debug!("provides_universal_region: result = {:?}", result); - result - } - - /// If `r2` represents a placeholder region, then this returns - /// `true` if `r1` cannot name that placeholder in its - /// value; otherwise, returns `false`. - pub(crate) fn cannot_name_placeholder(&self, r1: RegionVid, r2: RegionVid) -> bool { - match self.definitions[r2].origin { - NllRegionVariableOrigin::Placeholder(placeholder) => { - let r1_universe = self.definitions[r1].universe; - debug!( - "cannot_name_value_of: universe1={r1_universe:?} placeholder={:?}", - placeholder - ); - r1_universe.cannot_name(placeholder.universe) - } - - NllRegionVariableOrigin::FreeRegion | NllRegionVariableOrigin::Existential { .. } => { - false - } - } - } - - /// Finds a good `ObligationCause` to blame for the fact that `fr1` outlives `fr2`. - pub(crate) fn find_outlives_blame_span( - &self, - fr1: RegionVid, - fr1_origin: NllRegionVariableOrigin, - fr2: RegionVid, - ) -> (ConstraintCategory<'tcx>, ObligationCause<'tcx>) { - let BlameConstraint { category, cause, .. } = self - .best_blame_constraint(fr1, fr1_origin, |r| self.provides_universal_region(r, fr1, fr2)) - .0; - (category, cause) + self.constraint_path_to(from_region, |to| to == to_region, true).map(|o| o.0) } /// Walks the graph of constraints (where `'a: 'b` is considered - /// an edge `'a -> 'b`) to find all paths from `from_region` to - /// `to_region`. The paths are accumulated into the vector - /// `results`. The paths are stored as a series of - /// `ConstraintIndex` values -- in other words, a list of *edges*. + /// an edge `'a -> 'b`) to find a path from `from_region` to + /// `to_region`. /// /// Returns: a series of constraints as well as the region `R` /// that passed the target test. + /// If `include_static_outlives_all` is `true`, then the synthetic + /// outlives constraints `'static -> a` for every region `a` are + /// considered in the search, otherwise they are ignored. #[instrument(skip(self, target_test), ret)] - pub(crate) fn find_constraint_paths_between_regions( + pub(crate) fn constraint_path_to( &self, from_region: RegionVid, target_test: impl Fn(RegionVid) -> bool, + include_placeholder_static: bool, + ) -> Option<(Vec>, RegionVid)> { + self.find_constraint_path_between_regions_inner( + true, + from_region, + &target_test, + include_placeholder_static, + ) + .or_else(|| { + self.find_constraint_path_between_regions_inner( + false, + from_region, + &target_test, + include_placeholder_static, + ) + }) + } + + /// The constraints we get from equating the hidden type of each use of an opaque + /// with its final hidden type may end up getting preferred over other, potentially + /// longer constraint paths. + /// + /// Given that we compute the final hidden type by relying on this existing constraint + /// path, this can easily end up hiding the actual reason for why we require these regions + /// to be equal. + /// + /// To handle this, we first look at the path while ignoring these constraints and then + /// retry while considering them. This is not perfect, as the `from_region` may have already + /// been partially related to its argument region, so while we rely on a member constraint + /// to get a complete path, the most relevant step of that path already existed before then. + fn find_constraint_path_between_regions_inner( + &self, + ignore_opaque_type_constraints: bool, + from_region: RegionVid, + target_test: impl Fn(RegionVid) -> bool, + include_placeholder_static: bool, ) -> Option<(Vec>, RegionVid)> { let mut context = IndexVec::from_elem(Trace::NotVisited, &self.definitions); context[from_region] = Trace::StartRegion; @@ -1670,7 +1413,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { while let Some(r) = deque.pop_front() { debug!( - "find_constraint_paths_between_regions: from_region={:?} r={:?} value={}", + "constraint_path_to: from_region={:?} r={:?} value={}", from_region, r, self.region_value_str(r), @@ -1704,20 +1447,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { result.push(c); } - Trace::FromMember(sup, sub, span) => { - let c = OutlivesConstraint { - sup, - sub, - locations: Locations::All(span), - span, - category: ConstraintCategory::OpaqueType, - variance_info: ty::VarianceDiagInfo::default(), - from_closure: false, - }; - p = c.sup; - result.push(c); - } - Trace::StartRegion => { result.reverse(); return Some((result, r)); @@ -1756,23 +1485,24 @@ impl<'tcx> RegionInferenceContext<'tcx> { let edges = self.constraint_graph.outgoing_edges_from_graph(r, &self.constraints); // This loop can be hot. for constraint in edges { - if matches!(constraint.category, ConstraintCategory::IllegalUniverse) { - debug!("Ignoring illegal universe constraint: {constraint:?}"); - continue; + match constraint.category { + ConstraintCategory::OutlivesUnnameablePlaceholder(_) + if !include_placeholder_static => + { + debug!("Ignoring illegal placeholder constraint: {constraint:?}"); + continue; + } + ConstraintCategory::OpaqueType if ignore_opaque_type_constraints => { + debug!("Ignoring member constraint: {constraint:?}"); + continue; + } + _ => {} } + debug_assert_eq!(constraint.sup, r); handle_trace(constraint.sub, Trace::FromGraph(constraint)); } } - - // Member constraints can also give rise to `'r: 'x` edges that - // were not part of the graph initially, so watch out for those. - // (But they are extremely rare; this loop is very cold.) - for constraint in self.applied_member_constraints(self.constraint_sccs.scc(r)) { - let sub = constraint.min_choice; - let p_c = &self.member_constraints[constraint.member_constraint_index]; - handle_trace(sub, Trace::FromMember(r, sub, p_c.definition_span)); - } } None @@ -1783,37 +1513,10 @@ impl<'tcx> RegionInferenceContext<'tcx> { pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, location: Location) -> RegionVid { trace!(scc = ?self.constraint_sccs.scc(fr1)); trace!(universe = ?self.max_nameable_universe(self.constraint_sccs.scc(fr1))); - self.find_constraint_paths_between_regions(fr1, |r| { - // First look for some `r` such that `fr1: r` and `r` is live at `location` + self.constraint_path_to(fr1, |r| { trace!(?r, liveness_constraints=?self.liveness_constraints.pretty_print_live_points(r)); self.liveness_constraints.is_live_at(r, location) - }) - .or_else(|| { - // If we fail to find that, we may find some `r` such that - // `fr1: r` and `r` is a placeholder from some universe - // `fr1` cannot name. This would force `fr1` to be - // `'static`. - self.find_constraint_paths_between_regions(fr1, |r| { - self.cannot_name_placeholder(fr1, r) - }) - }) - .or_else(|| { - // If we fail to find THAT, it may be that `fr1` is a - // placeholder that cannot "fit" into its SCC. In that - // case, there should be some `r` where `fr1: r` and `fr1` is a - // placeholder that `r` cannot name. We can blame that - // edge. - // - // Remember that if `R1: R2`, then the universe of R1 - // must be able to name the universe of R2, because R2 will - // be at least `'empty(Universe(R2))`, and `R1` must be at - // larger than that. - self.find_constraint_paths_between_regions(fr1, |r| { - self.cannot_name_placeholder(r, fr1) - }) - }) - .map(|(_path, r)| r) - .unwrap() + }, true).unwrap().1 } /// Get the region outlived by `longer_fr` and live at `element`. @@ -1857,22 +1560,38 @@ impl<'tcx> RegionInferenceContext<'tcx> { /// creating a constraint path that forces `R` to outlive /// `from_region`, and then finding the best choices within that /// path to blame. - #[instrument(level = "debug", skip(self, target_test))] + #[instrument(level = "debug", skip(self))] pub(crate) fn best_blame_constraint( &self, from_region: RegionVid, from_region_origin: NllRegionVariableOrigin, - target_test: impl Fn(RegionVid) -> bool, + to_region: RegionVid, ) -> (BlameConstraint<'tcx>, Vec>) { - // Find all paths - let (path, target_region) = self - .find_constraint_paths_between_regions(from_region, target_test) - .or_else(|| { - self.find_constraint_paths_between_regions(from_region, |r| { - self.cannot_name_placeholder(from_region, r) - }) - }) - .unwrap(); + assert!(from_region != to_region, "Trying to blame a region for itself!"); + + let path = self.constraint_path_between_regions(from_region, to_region).unwrap(); + + // If we are passing through a constraint added because we reached an unnameable placeholder `'unnameable`, + // redirect search towards `'unnameable`. + let due_to_placeholder_outlives = path.iter().find_map(|c| { + if let ConstraintCategory::OutlivesUnnameablePlaceholder(unnameable) = c.category { + Some(unnameable) + } else { + None + } + }); + + // Edge case: it's possible that `'from_region` is an unnameable placeholder. + let path = if let Some(unnameable) = due_to_placeholder_outlives + && unnameable != from_region + { + // We ignore the extra edges due to unnameable placeholders to get + // an explanation that was present in the original constraint graph. + self.constraint_path_to(from_region, |r| r == unnameable, false).unwrap().0 + } else { + path + }; + debug!( "path={:#?}", path.iter() @@ -1939,10 +1658,15 @@ impl<'tcx> RegionInferenceContext<'tcx> { // // and here we prefer to blame the source (the y = x statement). let blame_source = match from_region_origin { - NllRegionVariableOrigin::FreeRegion - | NllRegionVariableOrigin::Existential { from_forall: false } => true, - NllRegionVariableOrigin::Placeholder(_) - | NllRegionVariableOrigin::Existential { from_forall: true } => false, + NllRegionVariableOrigin::FreeRegion => true, + NllRegionVariableOrigin::Placeholder(_) => false, + // `'existential: 'whatever` never results in a region error by itself. + // We may always infer it to `'static` afterall. This means while an error + // path may go through an existential, these existentials are never the + // `from_region`. + NllRegionVariableOrigin::Existential { name: _ } => { + unreachable!("existentials can outlive everything") + } }; // To pick a constraint to blame, we organize constraints by how interesting we expect them @@ -1975,7 +1699,7 @@ impl<'tcx> RegionInferenceContext<'tcx> { ConstraintCategory::Cast { unsize_to: Some(unsize_ty), is_implicit_coercion: true, - } if target_region == self.universal_regions().fr_static + } if to_region == self.universal_regions().fr_static // Mirror the note's condition, to minimize how often this diverts blame. && let ty::Adt(_, args) = unsize_ty.kind() && args.iter().any(|arg| arg.as_type().is_some_and(|ty| ty.is_trait())) @@ -2012,9 +1736,10 @@ impl<'tcx> RegionInferenceContext<'tcx> { // `BoringNoLocation` constraints can point to user-written code, but are less // specific, and are not used for relations that would make sense to blame. ConstraintCategory::BoringNoLocation => 6, - // Do not blame internal constraints. - ConstraintCategory::IllegalUniverse => 7, - ConstraintCategory::Internal => 8, + // Do not blame internal constraints if we can avoid it. Never blame + // the `'region: 'static` constraints introduced by placeholder outlives. + ConstraintCategory::Internal => 7, + ConstraintCategory::OutlivesUnnameablePlaceholder(_) => 8, }; debug!("constraint {constraint:?} category: {category:?}, interest: {interest:?}"); @@ -2050,6 +1775,14 @@ impl<'tcx> RegionInferenceContext<'tcx> { path[best_choice] }; + assert!( + !matches!( + best_constraint.category, + ConstraintCategory::OutlivesUnnameablePlaceholder(_) + ), + "Illegal placeholder constraint blamed; should have redirected to other region relation" + ); + let blame_constraint = BlameConstraint { category: best_constraint.category, from_closure: best_constraint.from_closure, @@ -2099,11 +1832,6 @@ impl<'tcx> RegionInferenceContext<'tcx> { &self.constraint_sccs } - /// Access to the region graph, built from the outlives constraints. - pub(crate) fn region_graph(&self) -> RegionGraph<'_, 'tcx, graph::Normal> { - self.constraint_graph.region_graph(&self.constraints, self.universal_regions().fr_static) - } - /// Returns the representative `RegionVid` for a given SCC. /// See `RegionTracker` for how a region variable ID is chosen. /// diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs deleted file mode 100644 index 6270e6d9a60e..000000000000 --- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs +++ /dev/null @@ -1,291 +0,0 @@ -use rustc_data_structures::fx::FxIndexMap; -use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin}; -use rustc_macros::extension; -use rustc_middle::ty::{ - self, DefiningScopeKind, OpaqueHiddenType, OpaqueTypeKey, Ty, TyCtxt, TypeFoldable, - TypeVisitableExt, fold_regions, -}; -use rustc_span::Span; -use rustc_trait_selection::opaque_types::check_opaque_type_parameter_valid; -use tracing::{debug, instrument}; - -use super::RegionInferenceContext; -use crate::BorrowCheckRootCtxt; -use crate::session_diagnostics::LifetimeMismatchOpaqueParam; -use crate::universal_regions::RegionClassification; - -impl<'tcx> RegionInferenceContext<'tcx> { - /// Resolve any opaque types that were encountered while borrow checking - /// this item. This is then used to get the type in the `type_of` query. - /// - /// For example consider `fn f<'a>(x: &'a i32) -> impl Sized + 'a { x }`. - /// This is lowered to give HIR something like - /// - /// type f<'a>::_Return<'_x> = impl Sized + '_x; - /// fn f<'a>(x: &'a i32) -> f<'a>::_Return<'a> { x } - /// - /// When checking the return type record the type from the return and the - /// type used in the return value. In this case they might be `_Return<'1>` - /// and `&'2 i32` respectively. - /// - /// Once we to this method, we have completed region inference and want to - /// call `infer_opaque_definition_from_instantiation` to get the inferred - /// type of `_Return<'_x>`. `infer_opaque_definition_from_instantiation` - /// compares lifetimes directly, so we need to map the inference variables - /// back to concrete lifetimes: `'static`, `ReEarlyParam` or `ReLateParam`. - /// - /// First we map the regions in the generic parameters `_Return<'1>` to - /// their `external_name` giving `_Return<'a>`. This step is a bit involved. - /// See the [rustc-dev-guide chapter] for more info. - /// - /// Then we map all the lifetimes in the concrete type to an equal - /// universal region that occurs in the opaque type's args, in this case - /// this would result in `&'a i32`. We only consider regions in the args - /// in case there is an equal region that does not. For example, this should - /// be allowed: - /// `fn f<'a: 'b, 'b: 'a>(x: *mut &'b i32) -> impl Sized + 'a { x }` - /// - /// This will then allow `infer_opaque_definition_from_instantiation` to - /// determine that `_Return<'_x> = &'_x i32`. - /// - /// There's a slight complication around closures. Given - /// `fn f<'a: 'a>() { || {} }` the closure's type is something like - /// `f::<'a>::{{closure}}`. The region parameter from f is essentially - /// ignored by type checking so ends up being inferred to an empty region. - /// Calling `universal_upper_bound` for such a region gives `fr_fn_body`, - /// which has no `external_name` in which case we use `'{erased}` as the - /// region to pass to `infer_opaque_definition_from_instantiation`. - /// - /// [rustc-dev-guide chapter]: - /// https://rustc-dev-guide.rust-lang.org/opaque-types-region-infer-restrictions.html - #[instrument(level = "debug", skip(self, root_cx, infcx), ret)] - pub(crate) fn infer_opaque_types( - &self, - root_cx: &mut BorrowCheckRootCtxt<'tcx>, - infcx: &InferCtxt<'tcx>, - opaque_ty_decls: FxIndexMap, OpaqueHiddenType<'tcx>>, - ) { - let mut decls_modulo_regions: FxIndexMap, (OpaqueTypeKey<'tcx>, Span)> = - FxIndexMap::default(); - - for (opaque_type_key, concrete_type) in opaque_ty_decls { - debug!(?opaque_type_key, ?concrete_type); - - let mut arg_regions: Vec<(ty::RegionVid, ty::Region<'_>)> = - vec![(self.universal_regions().fr_static, infcx.tcx.lifetimes.re_static)]; - - let opaque_type_key = - opaque_type_key.fold_captured_lifetime_args(infcx.tcx, |region| { - // Use the SCC representative instead of directly using `region`. - // See [rustc-dev-guide chapter] § "Strict lifetime equality". - let scc = self.constraint_sccs.scc(region.as_var()); - let vid = self.scc_representative(scc); - let named = match self.definitions[vid].origin { - // Iterate over all universal regions in a consistent order and find the - // *first* equal region. This makes sure that equal lifetimes will have - // the same name and simplifies subsequent handling. - // See [rustc-dev-guide chapter] § "Semantic lifetime equality". - NllRegionVariableOrigin::FreeRegion => self - .universal_regions() - .universal_regions_iter() - .filter(|&ur| { - // See [rustc-dev-guide chapter] § "Closure restrictions". - !matches!( - self.universal_regions().region_classification(ur), - Some(RegionClassification::External) - ) - }) - .find(|&ur| self.universal_region_relations.equal(vid, ur)) - .map(|ur| self.definitions[ur].external_name.unwrap()), - NllRegionVariableOrigin::Placeholder(placeholder) => { - Some(ty::Region::new_placeholder(infcx.tcx, placeholder)) - } - NllRegionVariableOrigin::Existential { .. } => None, - } - .unwrap_or_else(|| { - ty::Region::new_error_with_message( - infcx.tcx, - concrete_type.span, - "opaque type with non-universal region args", - ) - }); - - arg_regions.push((vid, named)); - named - }); - debug!(?opaque_type_key, ?arg_regions); - - let concrete_type = fold_regions(infcx.tcx, concrete_type, |region, _| { - arg_regions - .iter() - .find(|&&(arg_vid, _)| self.eval_equal(region.as_var(), arg_vid)) - .map(|&(_, arg_named)| arg_named) - .unwrap_or(infcx.tcx.lifetimes.re_erased) - }); - debug!(?concrete_type); - - let ty = - infcx.infer_opaque_definition_from_instantiation(opaque_type_key, concrete_type); - - // Sometimes, when the hidden type is an inference variable, it can happen that - // the hidden type becomes the opaque type itself. In this case, this was an opaque - // usage of the opaque type and we can ignore it. This check is mirrored in typeck's - // writeback. - if !infcx.next_trait_solver() { - if let ty::Alias(ty::Opaque, alias_ty) = ty.kind() - && alias_ty.def_id == opaque_type_key.def_id.to_def_id() - && alias_ty.args == opaque_type_key.args - { - continue; - } - } - - root_cx.add_concrete_opaque_type( - opaque_type_key.def_id, - OpaqueHiddenType { span: concrete_type.span, ty }, - ); - - // Check that all opaque types have the same region parameters if they have the same - // non-region parameters. This is necessary because within the new solver we perform - // various query operations modulo regions, and thus could unsoundly select some impls - // that don't hold. - if !ty.references_error() - && let Some((prev_decl_key, prev_span)) = decls_modulo_regions.insert( - infcx.tcx.erase_regions(opaque_type_key), - (opaque_type_key, concrete_type.span), - ) - && let Some((arg1, arg2)) = std::iter::zip( - prev_decl_key.iter_captured_args(infcx.tcx).map(|(_, arg)| arg), - opaque_type_key.iter_captured_args(infcx.tcx).map(|(_, arg)| arg), - ) - .find(|(arg1, arg2)| arg1 != arg2) - { - infcx.dcx().emit_err(LifetimeMismatchOpaqueParam { - arg: arg1, - prev: arg2, - span: prev_span, - prev_span: concrete_type.span, - }); - } - } - } - - /// Map the regions in the type to named regions. This is similar to what - /// `infer_opaque_types` does, but can infer any universal region, not only - /// ones from the args for the opaque type. It also doesn't double check - /// that the regions produced are in fact equal to the named region they are - /// replaced with. This is fine because this function is only to improve the - /// region names in error messages. - /// - /// This differs from `MirBorrowckCtxt::name_regions` since it is particularly - /// lax with mapping region vids that are *shorter* than a universal region to - /// that universal region. This is useful for member region constraints since - /// we want to suggest a universal region name to capture even if it's technically - /// not equal to the error region. - pub(crate) fn name_regions_for_member_constraint(&self, tcx: TyCtxt<'tcx>, ty: T) -> T - where - T: TypeFoldable>, - { - fold_regions(tcx, ty, |region, _| match region.kind() { - ty::ReVar(vid) => { - let scc = self.constraint_sccs.scc(vid); - - // Special handling of higher-ranked regions. - if !self.max_nameable_universe(scc).is_root() { - match self.scc_values.placeholders_contained_in(scc).enumerate().last() { - // If the region contains a single placeholder then they're equal. - Some((0, placeholder)) => { - return ty::Region::new_placeholder(tcx, placeholder); - } - - // Fallback: this will produce a cryptic error message. - _ => return region, - } - } - - // Find something that we can name - let upper_bound = self.approx_universal_upper_bound(vid); - if let Some(universal_region) = self.definitions[upper_bound].external_name { - return universal_region; - } - - // Nothing exact found, so we pick a named upper bound, if there's only one. - // If there's >1 universal region, then we probably are dealing w/ an intersection - // region which cannot be mapped back to a universal. - // FIXME: We could probably compute the LUB if there is one. - let scc = self.constraint_sccs.scc(vid); - let upper_bounds: Vec<_> = self - .reverse_scc_graph() - .upper_bounds(scc) - .filter_map(|vid| self.definitions[vid].external_name) - .filter(|r| !r.is_static()) - .collect(); - match &upper_bounds[..] { - [universal_region] => *universal_region, - _ => region, - } - } - _ => region, - }) - } -} - -#[extension(pub trait InferCtxtExt<'tcx>)] -impl<'tcx> InferCtxt<'tcx> { - /// Given the fully resolved, instantiated type for an opaque - /// type, i.e., the value of an inference variable like C1 or C2 - /// (*), computes the "definition type" for an opaque type - /// definition -- that is, the inferred value of `Foo1<'x>` or - /// `Foo2<'x>` that we would conceptually use in its definition: - /// ```ignore (illustrative) - /// type Foo1<'x> = impl Bar<'x> = AAA; // <-- this type AAA - /// type Foo2<'x> = impl Bar<'x> = BBB; // <-- or this type BBB - /// fn foo<'a, 'b>(..) -> (Foo1<'a>, Foo2<'b>) { .. } - /// ``` - /// Note that these values are defined in terms of a distinct set of - /// generic parameters (`'x` instead of `'a`) from C1 or C2. The main - /// purpose of this function is to do that translation. - /// - /// (*) C1 and C2 were introduced in the comments on - /// `register_member_constraints`. Read that comment for more context. - /// - /// # Parameters - /// - /// - `def_id`, the `impl Trait` type - /// - `args`, the args used to instantiate this opaque type - /// - `instantiated_ty`, the inferred type C1 -- fully resolved, lifted version of - /// `opaque_defn.concrete_ty` - #[instrument(level = "debug", skip(self))] - fn infer_opaque_definition_from_instantiation( - &self, - opaque_type_key: OpaqueTypeKey<'tcx>, - instantiated_ty: OpaqueHiddenType<'tcx>, - ) -> Ty<'tcx> { - if let Some(e) = self.tainted_by_errors() { - return Ty::new_error(self.tcx, e); - } - - if let Err(err) = check_opaque_type_parameter_valid( - self, - opaque_type_key, - instantiated_ty.span, - DefiningScopeKind::MirBorrowck, - ) { - return Ty::new_error(self.tcx, err.report(self)); - } - - let definition_ty = instantiated_ty - .remap_generic_params_to_declaration_params( - opaque_type_key, - self.tcx, - DefiningScopeKind::MirBorrowck, - ) - .ty; - - if let Err(e) = definition_ty.error_reported() { - return Ty::new_error(self.tcx, e); - } - - definition_ty - } -} diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types/member_constraints.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types/member_constraints.rs new file mode 100644 index 000000000000..a2e2b61ae2d3 --- /dev/null +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types/member_constraints.rs @@ -0,0 +1,195 @@ +use rustc_data_structures::fx::FxHashMap; +use rustc_hir::def_id::DefId; +use rustc_middle::bug; +use rustc_middle::ty::{ + self, GenericArgsRef, Region, RegionVid, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, + TypeVisitor, +}; +use tracing::{debug, instrument}; + +use super::DefiningUse; +use super::region_ctxt::RegionCtxt; +use crate::constraints::ConstraintSccIndex; + +pub(super) fn apply_member_constraints<'tcx>( + rcx: &mut RegionCtxt<'_, 'tcx>, + defining_uses: &[DefiningUse<'tcx>], +) { + // Start by collecting the member constraints of all defining uses. + // + // Applying member constraints can influence other member constraints, + // so we first collect and then apply them. + let mut member_constraints = Default::default(); + for defining_use in defining_uses { + let mut visitor = CollectMemberConstraintsVisitor { + rcx, + defining_use, + member_constraints: &mut member_constraints, + }; + defining_use.hidden_type.ty.visit_with(&mut visitor); + } + + // Now walk over the region graph, visiting the smallest regions first and then all + // regions which have to outlive that one. + // + // Whenever we encounter a member region, we mutate the value of this SCC. This is + // as if we'd introduce new outlives constraints. However, we discard these region + // values after we've inferred the hidden types of opaques and apply the region + // constraints by simply equating the actual hidden type with the inferred one. + debug!(?member_constraints); + for scc_a in rcx.constraint_sccs.all_sccs() { + debug!(?scc_a); + // Start by adding the region values required by outlives constraints. This + // matches how we compute the final region values in `fn compute_regions`. + // + // We need to do this here to get a lower bound when applying member constraints. + // This propagates the region values added by previous member constraints. + for &scc_b in rcx.constraint_sccs.successors(scc_a) { + debug!(?scc_b); + rcx.scc_values.add_region(scc_a, scc_b); + } + + for defining_use in member_constraints.get(&scc_a).into_iter().flatten() { + apply_member_constraint(rcx, scc_a, &defining_use.arg_regions); + } + } +} + +#[instrument(level = "debug", skip(rcx))] +fn apply_member_constraint<'tcx>( + rcx: &mut RegionCtxt<'_, 'tcx>, + member: ConstraintSccIndex, + arg_regions: &[RegionVid], +) { + // If the member region lives in a higher universe, we currently choose + // the most conservative option by leaving it unchanged. + if !rcx.max_placeholder_universe_reached(member).is_root() { + debug!("member region reached non root universe, bailing"); + return; + } + + // The existing value of `'member` is a lower-bound. If its is already larger than + // some universal region, we cannot equate it with that region. Said differently, we + // ignore choice regions which are smaller than this member region. + let mut choice_regions = arg_regions + .iter() + .copied() + .map(|r| rcx.representative(r).rvid()) + .filter(|&choice_region| { + rcx.scc_values.universal_regions_outlived_by(member).all(|lower_bound| { + rcx.universal_region_relations.outlives(choice_region, lower_bound) + }) + }) + .collect::>(); + debug!(?choice_regions, "after enforcing lower-bound"); + + // Now find all the *upper bounds* -- that is, each UB is a + // free region that must outlive the member region `R0` (`UB: + // R0`). Therefore, we need only keep an option `O` if `UB: O` + // for all UB. + // + // If we have a requirement `'upper_bound: 'member`, equating `'member` + // with some region `'choice` means we now also require `'upper_bound: 'choice`. + // Avoid choice regions for which this does not hold. + for ub in rcx.rev_scc_graph.upper_bounds(member) { + choice_regions + .retain(|&choice_region| rcx.universal_region_relations.outlives(ub, choice_region)); + } + debug!(?choice_regions, "after enforcing upper-bound"); + + // At this point we can pick any member of `choice_regions` and would like to choose + // it to be a small as possible. To avoid potential non-determinism we will pick the + // smallest such choice. + // + // Because universal regions are only partially ordered (i.e, not every two regions are + // comparable), we will ignore any region that doesn't compare to all others when picking + // the minimum choice. + // + // For example, consider `choice_regions = ['static, 'a, 'b, 'c, 'd, 'e]`, where + // `'static: 'a, 'static: 'b, 'a: 'c, 'b: 'c, 'c: 'd, 'c: 'e`. + // `['d, 'e]` are ignored because they do not compare - the same goes for `['a, 'b]`. + let totally_ordered_subset = choice_regions.iter().copied().filter(|&r1| { + choice_regions.iter().all(|&r2| { + rcx.universal_region_relations.outlives(r1, r2) + || rcx.universal_region_relations.outlives(r2, r1) + }) + }); + // Now we're left with `['static, 'c]`. Pick `'c` as the minimum! + let Some(min_choice) = totally_ordered_subset.reduce(|r1, r2| { + let r1_outlives_r2 = rcx.universal_region_relations.outlives(r1, r2); + let r2_outlives_r1 = rcx.universal_region_relations.outlives(r2, r1); + match (r1_outlives_r2, r2_outlives_r1) { + (true, true) => r1.min(r2), + (true, false) => r2, + (false, true) => r1, + (false, false) => bug!("incomparable regions in total order"), + } + }) else { + debug!("no unique minimum choice"); + return; + }; + + debug!(?min_choice); + // Lift the member region to be at least as large as this `min_choice` by directly + // mutating the `scc_values` as we compute it. This acts as if we've added a + // `'member: 'min_choice` while not recomputing sccs. This means different sccs + // may now actually be equal. + let min_choice_scc = rcx.constraint_sccs.scc(min_choice); + rcx.scc_values.add_region(member, min_choice_scc); +} + +struct CollectMemberConstraintsVisitor<'a, 'b, 'tcx> { + rcx: &'a RegionCtxt<'a, 'tcx>, + defining_use: &'b DefiningUse<'tcx>, + member_constraints: &'a mut FxHashMap>>, +} +impl<'tcx> CollectMemberConstraintsVisitor<'_, '_, 'tcx> { + fn cx(&self) -> TyCtxt<'tcx> { + self.rcx.infcx.tcx + } + fn visit_closure_args(&mut self, def_id: DefId, args: GenericArgsRef<'tcx>) { + let generics = self.cx().generics_of(def_id); + for arg in args.iter().skip(generics.parent_count) { + arg.visit_with(self); + } + } +} +impl<'tcx> TypeVisitor> for CollectMemberConstraintsVisitor<'_, '_, 'tcx> { + fn visit_region(&mut self, r: Region<'tcx>) { + match r.kind() { + ty::ReBound(..) => return, + ty::ReVar(vid) => { + let scc = self.rcx.constraint_sccs.scc(vid); + self.member_constraints.entry(scc).or_default().push(self.defining_use); + } + _ => unreachable!(), + } + } + + fn visit_ty(&mut self, ty: Ty<'tcx>) { + if !ty.flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS) { + return; + } + + match *ty.kind() { + ty::Closure(def_id, args) + | ty::CoroutineClosure(def_id, args) + | ty::Coroutine(def_id, args) => self.visit_closure_args(def_id, args), + + ty::Alias(kind, ty::AliasTy { def_id, args, .. }) + if let Some(variances) = self.cx().opt_alias_variances(kind, def_id) => + { + // Skip lifetime parameters that are not captured, since they do + // not need member constraints registered for them; we'll erase + // them (and hopefully in the future replace them with placeholders). + for (&v, arg) in std::iter::zip(variances, args.iter()) { + if v != ty::Bivariant { + arg.visit_with(self) + } + } + } + + _ => ty.super_visit_with(self), + } + } +} diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs new file mode 100644 index 000000000000..67d1d44b1e1f --- /dev/null +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types/mod.rs @@ -0,0 +1,699 @@ +use std::iter; +use std::rc::Rc; + +use rustc_data_structures::frozen::Frozen; +use rustc_data_structures::fx::FxIndexMap; +use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_infer::infer::outlives::env::RegionBoundPairs; +use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, OpaqueTypeStorageEntries}; +use rustc_infer::traits::ObligationCause; +use rustc_macros::extension; +use rustc_middle::mir::{Body, ConstraintCategory, DefinitionSiteHiddenTypes}; +use rustc_middle::ty::{ + self, DefiningScopeKind, EarlyBinder, FallibleTypeFolder, GenericArg, GenericArgsRef, + OpaqueHiddenType, OpaqueTypeKey, Region, RegionVid, Ty, TyCtxt, TypeFoldable, + TypeSuperFoldable, TypeVisitableExt, fold_regions, +}; +use rustc_mir_dataflow::points::DenseLocationMap; +use rustc_span::Span; +use rustc_trait_selection::opaque_types::{ + NonDefiningUseReason, opaque_type_has_defining_use_args, +}; +use rustc_trait_selection::solve::NoSolution; +use rustc_trait_selection::traits::query::type_op::custom::CustomTypeOp; +use tracing::{debug, instrument}; + +use super::reverse_sccs::ReverseSccGraph; +use crate::BorrowckInferCtxt; +use crate::consumers::RegionInferenceContext; +use crate::session_diagnostics::LifetimeMismatchOpaqueParam; +use crate::type_check::canonical::fully_perform_op_raw; +use crate::type_check::free_region_relations::UniversalRegionRelations; +use crate::type_check::{Locations, MirTypeckRegionConstraints}; +use crate::universal_regions::{RegionClassification, UniversalRegions}; + +mod member_constraints; +mod region_ctxt; + +use member_constraints::apply_member_constraints; +use region_ctxt::RegionCtxt; + +/// We defer errors from [fn handle_opaque_type_uses] and only report them +/// if there are no `RegionErrors`. If there are region errors, it's likely +/// that errors here are caused by them and don't need to be handled separately. +pub(crate) enum DeferredOpaqueTypeError<'tcx> { + InvalidOpaqueTypeArgs(NonDefiningUseReason<'tcx>), + LifetimeMismatchOpaqueParam(LifetimeMismatchOpaqueParam<'tcx>), + UnexpectedHiddenRegion { + /// The opaque type. + opaque_type_key: OpaqueTypeKey<'tcx>, + /// The hidden type containing the member region. + hidden_type: OpaqueHiddenType<'tcx>, + /// The unexpected region. + member_region: Region<'tcx>, + }, + NonDefiningUseInDefiningScope { + span: Span, + opaque_type_key: OpaqueTypeKey<'tcx>, + }, +} + +/// We eagerly map all regions to NLL vars here, as we need to make sure we've +/// introduced nll vars for all used placeholders. +/// +/// We need to resolve inference vars as even though we're in MIR typeck, we may still +/// encounter inference variables, e.g. when checking user types. +pub(crate) fn clone_and_resolve_opaque_types<'tcx>( + infcx: &BorrowckInferCtxt<'tcx>, + universal_region_relations: &Frozen>, + constraints: &mut MirTypeckRegionConstraints<'tcx>, +) -> (OpaqueTypeStorageEntries, Vec<(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)>) { + let opaque_types = infcx.clone_opaque_types(); + let opaque_types_storage_num_entries = infcx.inner.borrow_mut().opaque_types().num_entries(); + let opaque_types = opaque_types + .into_iter() + .map(|entry| { + fold_regions(infcx.tcx, infcx.resolve_vars_if_possible(entry), |r, _| { + let vid = if let ty::RePlaceholder(placeholder) = r.kind() { + constraints.placeholder_region(infcx, placeholder).as_var() + } else { + universal_region_relations.universal_regions.to_region_vid(r) + }; + Region::new_var(infcx.tcx, vid) + }) + }) + .collect::>(); + (opaque_types_storage_num_entries, opaque_types) +} + +/// Maps an NLL var to a deterministically chosen equal universal region. +/// +/// See the corresponding [rustc-dev-guide chapter] for more details. This +/// ignores changes to the region values due to member constraints. Applying +/// member constraints does not impact the result of this function. +/// +/// [rustc-dev-guide chapter]: https://rustc-dev-guide.rust-lang.org/borrow_check/opaque-types-region-inference-restrictions.html +fn nll_var_to_universal_region<'tcx>( + rcx: &RegionCtxt<'_, 'tcx>, + r: RegionVid, +) -> Option> { + // Use the SCC representative instead of directly using `region`. + // See [rustc-dev-guide chapter] § "Strict lifetime equality". + let vid = rcx.representative(r).rvid(); + match rcx.definitions[vid].origin { + // Iterate over all universal regions in a consistent order and find the + // *first* equal region. This makes sure that equal lifetimes will have + // the same name and simplifies subsequent handling. + // See [rustc-dev-guide chapter] § "Semantic lifetime equality". + NllRegionVariableOrigin::FreeRegion => rcx + .universal_regions() + .universal_regions_iter() + .filter(|&ur| { + // See [rustc-dev-guide chapter] § "Closure restrictions". + !matches!( + rcx.universal_regions().region_classification(ur), + Some(RegionClassification::External) + ) + }) + .find(|&ur| rcx.universal_region_relations.equal(vid, ur)) + .map(|ur| rcx.definitions[ur].external_name.unwrap()), + NllRegionVariableOrigin::Placeholder(placeholder) => { + Some(ty::Region::new_placeholder(rcx.infcx.tcx, placeholder)) + } + // If `r` were equal to any universal region, its SCC representative + // would have been set to a free region. + NllRegionVariableOrigin::Existential { .. } => None, + } +} + +/// Collect all defining uses of opaque types inside of this typeck root. This +/// expects the hidden type to be mapped to the definition parameters of the opaque +/// and errors if we end up with distinct hidden types. +fn add_hidden_type<'tcx>( + tcx: TyCtxt<'tcx>, + hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>, + def_id: LocalDefId, + hidden_ty: OpaqueHiddenType<'tcx>, +) { + // Sometimes two opaque types are the same only after we remap the generic parameters + // back to the opaque type definition. E.g. we may have `OpaqueType` mapped to + // `(X, Y)` and `OpaqueType` mapped to `(Y, X)`, and those are the same, but we + // only know that once we convert the generic parameters to those of the opaque type. + if let Some(prev) = hidden_types.0.get_mut(&def_id) { + if prev.ty != hidden_ty.ty { + let guar = hidden_ty.ty.error_reported().err().unwrap_or_else(|| { + let (Ok(e) | Err(e)) = prev.build_mismatch_error(&hidden_ty, tcx).map(|d| d.emit()); + e + }); + prev.ty = Ty::new_error(tcx, guar); + } + // Pick a better span if there is one. + // FIXME(oli-obk): collect multiple spans for better diagnostics down the road. + prev.span = prev.span.substitute_dummy(hidden_ty.span); + } else { + hidden_types.0.insert(def_id, hidden_ty); + } +} + +#[derive(Debug)] +struct DefiningUse<'tcx> { + /// The opaque type using non NLL vars. This uses the actual + /// free regions and placeholders. This is necessary + /// to interact with code outside of `rustc_borrowck`. + opaque_type_key: OpaqueTypeKey<'tcx>, + arg_regions: Vec, + hidden_type: OpaqueHiddenType<'tcx>, +} + +/// This computes the actual hidden types of the opaque types and maps them to their +/// definition sites. Outside of registering the computed hidden types this function +/// does not mutate the current borrowck state. +/// +/// While it may fail to infer the hidden type and return errors, we always apply +/// the computed hidden type to all opaque type uses to check whether they +/// are correct. This is necessary to support non-defining uses of opaques in their +/// defining scope. +/// +/// It also means that this whole function is not really soundness critical as we +/// recheck all uses of the opaques regardless. +pub(crate) fn compute_definition_site_hidden_types<'tcx>( + infcx: &BorrowckInferCtxt<'tcx>, + universal_region_relations: &Frozen>, + constraints: &MirTypeckRegionConstraints<'tcx>, + location_map: Rc, + hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>, + opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)], +) -> Vec> { + let mut errors = Vec::new(); + // When computing the hidden type we need to track member constraints. + // We don't mutate the region graph used by `fn compute_regions` but instead + // manually track region information via a `RegionCtxt`. We discard this + // information at the end of this function. + let mut rcx = RegionCtxt::new(infcx, universal_region_relations, location_map, constraints); + + // We start by checking each use of an opaque type during type check and + // check whether the generic arguments of the opaque type are fully + // universal, if so, it's a defining use. + let defining_uses = collect_defining_uses(&mut rcx, hidden_types, opaque_types, &mut errors); + + // We now compute and apply member constraints for all regions in the hidden + // types of each defining use. This mutates the region values of the `rcx` which + // is used when mapping the defining uses to the definition site. + apply_member_constraints(&mut rcx, &defining_uses); + + // After applying member constraints, we now check whether all member regions ended + // up equal to one of their choice regions and compute the actual hidden type of + // the opaque type definition. This is stored in the `root_cx`. + compute_definition_site_hidden_types_from_defining_uses( + &rcx, + hidden_types, + &defining_uses, + &mut errors, + ); + errors +} + +#[instrument(level = "debug", skip_all, ret)] +fn collect_defining_uses<'tcx>( + rcx: &mut RegionCtxt<'_, 'tcx>, + hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>, + opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)], + errors: &mut Vec>, +) -> Vec> { + let infcx = rcx.infcx; + let mut defining_uses = vec![]; + for &(opaque_type_key, hidden_type) in opaque_types { + let non_nll_opaque_type_key = opaque_type_key.fold_captured_lifetime_args(infcx.tcx, |r| { + nll_var_to_universal_region(&rcx, r.as_var()).unwrap_or(r) + }); + if let Err(err) = opaque_type_has_defining_use_args( + infcx, + non_nll_opaque_type_key, + hidden_type.span, + DefiningScopeKind::MirBorrowck, + ) { + // A non-defining use. This is a hard error on stable and gets ignored + // with `TypingMode::Borrowck`. + if infcx.tcx.use_typing_mode_borrowck() { + match err { + NonDefiningUseReason::Tainted(guar) => add_hidden_type( + infcx.tcx, + hidden_types, + opaque_type_key.def_id, + OpaqueHiddenType::new_error(infcx.tcx, guar), + ), + _ => debug!(?non_nll_opaque_type_key, ?err, "ignoring non-defining use"), + } + } else { + errors.push(DeferredOpaqueTypeError::InvalidOpaqueTypeArgs(err)); + debug!( + "collect_defining_uses: InvalidOpaqueTypeArgs for {:?} := {:?}", + non_nll_opaque_type_key, hidden_type + ); + } + continue; + } + + // We use the original `opaque_type_key` to compute the `arg_regions`. + let arg_regions = iter::once(rcx.universal_regions().fr_static) + .chain( + opaque_type_key + .iter_captured_args(infcx.tcx) + .filter_map(|(_, arg)| arg.as_region()) + .map(Region::as_var), + ) + .collect(); + defining_uses.push(DefiningUse { + opaque_type_key: non_nll_opaque_type_key, + arg_regions, + hidden_type, + }); + } + + defining_uses +} + +#[instrument(level = "debug", skip(rcx, hidden_types, defining_uses, errors))] +fn compute_definition_site_hidden_types_from_defining_uses<'tcx>( + rcx: &RegionCtxt<'_, 'tcx>, + hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>, + defining_uses: &[DefiningUse<'tcx>], + errors: &mut Vec>, +) { + let infcx = rcx.infcx; + let tcx = infcx.tcx; + let mut decls_modulo_regions: FxIndexMap, (OpaqueTypeKey<'tcx>, Span)> = + FxIndexMap::default(); + for &DefiningUse { opaque_type_key, ref arg_regions, hidden_type } in defining_uses { + debug!(?opaque_type_key, ?arg_regions, ?hidden_type); + // After applying member constraints, we now map all regions in the hidden type + // to the `arg_regions` of this defining use. In case a region in the hidden type + // ended up not being equal to any such region, we error. + let hidden_type = + match hidden_type.try_fold_with(&mut ToArgRegionsFolder::new(rcx, arg_regions)) { + Ok(hidden_type) => hidden_type, + Err(r) => { + debug!("UnexpectedHiddenRegion: {:?}", r); + errors.push(DeferredOpaqueTypeError::UnexpectedHiddenRegion { + hidden_type, + opaque_type_key, + member_region: ty::Region::new_var(tcx, r), + }); + let guar = tcx.dcx().span_delayed_bug( + hidden_type.span, + "opaque type with non-universal region args", + ); + ty::OpaqueHiddenType::new_error(tcx, guar) + } + }; + + // Now that we mapped the member regions to their final value, + // map the arguments of the opaque type key back to the parameters + // of the opaque type definition. + let ty = infcx + .infer_opaque_definition_from_instantiation(opaque_type_key, hidden_type) + .unwrap_or_else(|_| { + Ty::new_error_with_message( + rcx.infcx.tcx, + hidden_type.span, + "deferred invalid opaque type args", + ) + }); + + // Sometimes, when the hidden type is an inference variable, it can happen that + // the hidden type becomes the opaque type itself. In this case, this was an opaque + // usage of the opaque type and we can ignore it. This check is mirrored in typeck's + // writeback. + if !rcx.infcx.tcx.use_typing_mode_borrowck() { + if let ty::Alias(ty::Opaque, alias_ty) = ty.kind() + && alias_ty.def_id == opaque_type_key.def_id.to_def_id() + && alias_ty.args == opaque_type_key.args + { + continue; + } + } + + // Check that all opaque types have the same region parameters if they have the same + // non-region parameters. This is necessary because within the new solver we perform + // various query operations modulo regions, and thus could unsoundly select some impls + // that don't hold. + // + // FIXME(-Znext-solver): This isn't necessary after all. We can remove this check again. + if let Some((prev_decl_key, prev_span)) = decls_modulo_regions.insert( + rcx.infcx.tcx.erase_and_anonymize_regions(opaque_type_key), + (opaque_type_key, hidden_type.span), + ) && let Some((arg1, arg2)) = std::iter::zip( + prev_decl_key.iter_captured_args(infcx.tcx).map(|(_, arg)| arg), + opaque_type_key.iter_captured_args(infcx.tcx).map(|(_, arg)| arg), + ) + .find(|(arg1, arg2)| arg1 != arg2) + { + errors.push(DeferredOpaqueTypeError::LifetimeMismatchOpaqueParam( + LifetimeMismatchOpaqueParam { + arg: arg1, + prev: arg2, + span: prev_span, + prev_span: hidden_type.span, + }, + )); + } + add_hidden_type( + tcx, + hidden_types, + opaque_type_key.def_id, + OpaqueHiddenType { span: hidden_type.span, ty }, + ); + } +} + +/// A folder to map the regions in the hidden type to their corresponding `arg_regions`. +/// +/// This folder has to differentiate between member regions and other regions in the hidden +/// type. Member regions have to be equal to one of the `arg_regions` while other regions simply +/// get treated as an existential region in the opaque if they are not. Existential +/// regions are currently represented using `'erased`. +struct ToArgRegionsFolder<'a, 'tcx> { + rcx: &'a RegionCtxt<'a, 'tcx>, + // When folding closure args or bivariant alias arguments, we simply + // ignore non-member regions. However, we still need to map member + // regions to their arg region even if its in a closure argument. + // + // See tests/ui/type-alias-impl-trait/closure_wf_outlives.rs for an example. + erase_unknown_regions: bool, + arg_regions: &'a [RegionVid], +} + +impl<'a, 'tcx> ToArgRegionsFolder<'a, 'tcx> { + fn new( + rcx: &'a RegionCtxt<'a, 'tcx>, + arg_regions: &'a [RegionVid], + ) -> ToArgRegionsFolder<'a, 'tcx> { + ToArgRegionsFolder { rcx, erase_unknown_regions: false, arg_regions } + } + + fn fold_non_member_arg(&mut self, arg: GenericArg<'tcx>) -> GenericArg<'tcx> { + let prev = self.erase_unknown_regions; + self.erase_unknown_regions = true; + let res = arg.try_fold_with(self).unwrap(); + self.erase_unknown_regions = prev; + res + } + + fn fold_closure_args( + &mut self, + def_id: DefId, + args: GenericArgsRef<'tcx>, + ) -> Result, RegionVid> { + let generics = self.cx().generics_of(def_id); + self.cx().mk_args_from_iter(args.iter().enumerate().map(|(index, arg)| { + if index < generics.parent_count { + Ok(self.fold_non_member_arg(arg)) + } else { + arg.try_fold_with(self) + } + })) + } +} +impl<'tcx> FallibleTypeFolder> for ToArgRegionsFolder<'_, 'tcx> { + type Error = RegionVid; + fn cx(&self) -> TyCtxt<'tcx> { + self.rcx.infcx.tcx + } + + fn try_fold_region(&mut self, r: Region<'tcx>) -> Result, RegionVid> { + match r.kind() { + // ignore bound regions, keep visiting + ty::ReBound(_, _) => Ok(r), + _ => { + let r = r.as_var(); + if let Some(arg_region) = self + .arg_regions + .iter() + .copied() + .find(|&arg_vid| self.rcx.eval_equal(r, arg_vid)) + .and_then(|r| nll_var_to_universal_region(self.rcx, r)) + { + Ok(arg_region) + } else if self.erase_unknown_regions { + Ok(self.cx().lifetimes.re_erased) + } else { + Err(r) + } + } + } + } + + fn try_fold_ty(&mut self, ty: Ty<'tcx>) -> Result, RegionVid> { + if !ty.flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS) { + return Ok(ty); + } + + let tcx = self.cx(); + Ok(match *ty.kind() { + ty::Closure(def_id, args) => { + Ty::new_closure(tcx, def_id, self.fold_closure_args(def_id, args)?) + } + + ty::CoroutineClosure(def_id, args) => { + Ty::new_coroutine_closure(tcx, def_id, self.fold_closure_args(def_id, args)?) + } + + ty::Coroutine(def_id, args) => { + Ty::new_coroutine(tcx, def_id, self.fold_closure_args(def_id, args)?) + } + + ty::Alias(kind, ty::AliasTy { def_id, args, .. }) + if let Some(variances) = tcx.opt_alias_variances(kind, def_id) => + { + let args = tcx.mk_args_from_iter(std::iter::zip(variances, args.iter()).map( + |(&v, s)| { + if v == ty::Bivariant { + Ok(self.fold_non_member_arg(s)) + } else { + s.try_fold_with(self) + } + }, + ))?; + ty::AliasTy::new_from_args(tcx, def_id, args).to_ty(tcx) + } + + _ => ty.try_super_fold_with(self)?, + }) + } +} + +/// This function is what actually applies member constraints to the borrowck +/// state. It is also responsible to check all uses of the opaques in their +/// defining scope. +/// +/// It does this by equating the hidden type of each use with the instantiated final +/// hidden type of the opaque. +pub(crate) fn apply_definition_site_hidden_types<'tcx>( + infcx: &BorrowckInferCtxt<'tcx>, + body: &Body<'tcx>, + universal_regions: &UniversalRegions<'tcx>, + region_bound_pairs: &RegionBoundPairs<'tcx>, + known_type_outlives_obligations: &[ty::PolyTypeOutlivesPredicate<'tcx>], + constraints: &mut MirTypeckRegionConstraints<'tcx>, + hidden_types: &mut DefinitionSiteHiddenTypes<'tcx>, + opaque_types: &[(OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>)], +) -> Vec> { + let tcx = infcx.tcx; + let mut errors = Vec::new(); + for &(key, hidden_type) in opaque_types { + let Some(expected) = hidden_types.0.get(&key.def_id).map(|ty| EarlyBinder::bind(*ty)) + else { + if !tcx.use_typing_mode_borrowck() { + if let ty::Alias(ty::Opaque, alias_ty) = hidden_type.ty.kind() + && alias_ty.def_id == key.def_id.to_def_id() + && alias_ty.args == key.args + { + continue; + } else { + unreachable!("non-defining use in defining scope"); + } + } + errors.push(DeferredOpaqueTypeError::NonDefiningUseInDefiningScope { + span: hidden_type.span, + opaque_type_key: key, + }); + let guar = tcx.dcx().span_delayed_bug( + hidden_type.span, + "non-defining use in the defining scope with no defining uses", + ); + add_hidden_type(tcx, hidden_types, key.def_id, OpaqueHiddenType::new_error(tcx, guar)); + continue; + }; + + // We erase all non-member region of the opaque and need to treat these as existentials. + let expected = ty::fold_regions(tcx, expected.instantiate(tcx, key.args), |re, _dbi| { + match re.kind() { + ty::ReErased => infcx.next_nll_region_var( + NllRegionVariableOrigin::Existential { name: None }, + || crate::RegionCtxt::Existential(None), + ), + _ => re, + } + }); + + // We now simply equate the expected with the actual hidden type. + let locations = Locations::All(hidden_type.span); + if let Err(guar) = fully_perform_op_raw( + infcx, + body, + universal_regions, + region_bound_pairs, + known_type_outlives_obligations, + constraints, + locations, + ConstraintCategory::OpaqueType, + CustomTypeOp::new( + |ocx| { + let cause = ObligationCause::misc( + hidden_type.span, + body.source.def_id().expect_local(), + ); + // We need to normalize both types in the old solver before equatingt them. + let actual_ty = ocx.normalize(&cause, infcx.param_env, hidden_type.ty); + let expected_ty = ocx.normalize(&cause, infcx.param_env, expected.ty); + ocx.eq(&cause, infcx.param_env, actual_ty, expected_ty).map_err(|_| NoSolution) + }, + "equating opaque types", + ), + ) { + add_hidden_type(tcx, hidden_types, key.def_id, OpaqueHiddenType::new_error(tcx, guar)); + } + } + errors +} + +/// In theory `apply_definition_site_hidden_types` could introduce new uses of opaque types. +/// We do not check these new uses so this could be unsound. +/// +/// We detect any new uses and simply delay a bug if they occur. If this results in +/// an ICE we can properly handle this, but we haven't encountered any such test yet. +/// +/// See the related comment in `FnCtxt::detect_opaque_types_added_during_writeback`. +pub(crate) fn detect_opaque_types_added_while_handling_opaque_types<'tcx>( + infcx: &InferCtxt<'tcx>, + opaque_types_storage_num_entries: OpaqueTypeStorageEntries, +) { + for (key, hidden_type) in infcx + .inner + .borrow_mut() + .opaque_types() + .opaque_types_added_since(opaque_types_storage_num_entries) + { + let opaque_type_string = infcx.tcx.def_path_str(key.def_id); + let msg = format!("unexpected cyclic definition of `{opaque_type_string}`"); + infcx.dcx().span_delayed_bug(hidden_type.span, msg); + } + + let _ = infcx.take_opaque_types(); +} + +impl<'tcx> RegionInferenceContext<'tcx> { + /// Map the regions in the type to named regions. This is similar to what + /// `infer_opaque_types` does, but can infer any universal region, not only + /// ones from the args for the opaque type. It also doesn't double check + /// that the regions produced are in fact equal to the named region they are + /// replaced with. This is fine because this function is only to improve the + /// region names in error messages. + /// + /// This differs from `MirBorrowckCtxt::name_regions` since it is particularly + /// lax with mapping region vids that are *shorter* than a universal region to + /// that universal region. This is useful for member region constraints since + /// we want to suggest a universal region name to capture even if it's technically + /// not equal to the error region. + pub(crate) fn name_regions_for_member_constraint(&self, tcx: TyCtxt<'tcx>, ty: T) -> T + where + T: TypeFoldable>, + { + fold_regions(tcx, ty, |region, _| match region.kind() { + ty::ReVar(vid) => { + let scc = self.constraint_sccs.scc(vid); + + // Special handling of higher-ranked regions. + if !self.max_nameable_universe(scc).is_root() { + match self.scc_values.placeholders_contained_in(scc).enumerate().last() { + // If the region contains a single placeholder then they're equal. + Some((0, placeholder)) => { + return ty::Region::new_placeholder(tcx, placeholder); + } + + // Fallback: this will produce a cryptic error message. + _ => return region, + } + } + + // Find something that we can name + let upper_bound = self.approx_universal_upper_bound(vid); + if let Some(universal_region) = self.definitions[upper_bound].external_name { + return universal_region; + } + + // Nothing exact found, so we pick a named upper bound, if there's only one. + // If there's >1 universal region, then we probably are dealing w/ an intersection + // region which cannot be mapped back to a universal. + // FIXME: We could probably compute the LUB if there is one. + let scc = self.constraint_sccs.scc(vid); + let rev_scc_graph = + ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions()); + let upper_bounds: Vec<_> = rev_scc_graph + .upper_bounds(scc) + .filter_map(|vid| self.definitions[vid].external_name) + .filter(|r| !r.is_static()) + .collect(); + match &upper_bounds[..] { + [universal_region] => *universal_region, + _ => region, + } + } + _ => region, + }) + } +} + +#[extension(pub trait InferCtxtExt<'tcx>)] +impl<'tcx> InferCtxt<'tcx> { + /// Given the fully resolved, instantiated type for an opaque + /// type, i.e., the value of an inference variable like C1 or C2 + /// (*), computes the "definition type" for an opaque type + /// definition -- that is, the inferred value of `Foo1<'x>` or + /// `Foo2<'x>` that we would conceptually use in its definition: + /// ```ignore (illustrative) + /// type Foo1<'x> = impl Bar<'x> = AAA; // <-- this type AAA + /// type Foo2<'x> = impl Bar<'x> = BBB; // <-- or this type BBB + /// fn foo<'a, 'b>(..) -> (Foo1<'a>, Foo2<'b>) { .. } + /// ``` + /// Note that these values are defined in terms of a distinct set of + /// generic parameters (`'x` instead of `'a`) from C1 or C2. The main + /// purpose of this function is to do that translation. + /// + /// (*) C1 and C2 were introduced in the comments on + /// `register_member_constraints`. Read that comment for more context. + #[instrument(level = "debug", skip(self))] + fn infer_opaque_definition_from_instantiation( + &self, + opaque_type_key: OpaqueTypeKey<'tcx>, + instantiated_ty: OpaqueHiddenType<'tcx>, + ) -> Result, NonDefiningUseReason<'tcx>> { + opaque_type_has_defining_use_args( + self, + opaque_type_key, + instantiated_ty.span, + DefiningScopeKind::MirBorrowck, + )?; + + let definition_ty = instantiated_ty + .remap_generic_params_to_declaration_params( + opaque_type_key, + self.tcx, + DefiningScopeKind::MirBorrowck, + ) + .ty; + + definition_ty.error_reported()?; + Ok(definition_ty) + } +} diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types/region_ctxt.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types/region_ctxt.rs new file mode 100644 index 000000000000..90b15cbdd2cc --- /dev/null +++ b/compiler/rustc_borrowck/src/region_infer/opaque_types/region_ctxt.rs @@ -0,0 +1,136 @@ +use std::rc::Rc; + +use rustc_data_structures::frozen::Frozen; +use rustc_index::IndexVec; +use rustc_infer::infer::NllRegionVariableOrigin; +use rustc_middle::ty::{RegionVid, UniverseIndex}; +use rustc_mir_dataflow::points::DenseLocationMap; + +use crate::BorrowckInferCtxt; +use crate::constraints::ConstraintSccIndex; +use crate::handle_placeholders::{SccAnnotations, region_definitions}; +use crate::region_infer::reverse_sccs::ReverseSccGraph; +use crate::region_infer::values::RegionValues; +use crate::region_infer::{ + ConstraintSccs, OutlivesConstraintSet, RegionDefinition, RegionTracker, Representative, +}; +use crate::type_check::MirTypeckRegionConstraints; +use crate::type_check::free_region_relations::UniversalRegionRelations; +use crate::universal_regions::UniversalRegions; + +/// A slimmed down version of [crate::region_infer::RegionInferenceContext] used +/// only by opaque type handling. +pub(super) struct RegionCtxt<'a, 'tcx> { + pub(super) infcx: &'a BorrowckInferCtxt<'tcx>, + pub(super) definitions: Frozen>>, + pub(super) universal_region_relations: &'a UniversalRegionRelations<'tcx>, + pub(super) constraint_sccs: ConstraintSccs, + pub(super) scc_annotations: IndexVec, + pub(super) rev_scc_graph: ReverseSccGraph, + pub(super) scc_values: RegionValues, +} + +impl<'a, 'tcx> RegionCtxt<'a, 'tcx> { + /// Creates a new `RegionCtxt` used to compute defining opaque type uses. + /// + /// This does not yet propagate region values. This is instead done lazily + /// when applying member constraints. + pub(super) fn new( + infcx: &'a BorrowckInferCtxt<'tcx>, + universal_region_relations: &'a Frozen>, + location_map: Rc, + constraints: &MirTypeckRegionConstraints<'tcx>, + ) -> RegionCtxt<'a, 'tcx> { + let mut outlives_constraints = constraints.outlives_constraints.clone(); + let universal_regions = &universal_region_relations.universal_regions; + let (definitions, _has_placeholders) = region_definitions(infcx, universal_regions); + + let compute_sccs = + |outlives_constraints: &OutlivesConstraintSet<'tcx>, + annotations: &mut SccAnnotations<'_, 'tcx, RegionTracker>| { + ConstraintSccs::new_with_annotation( + &outlives_constraints + .graph(definitions.len()) + .region_graph(outlives_constraints, universal_regions.fr_static), + annotations, + ) + }; + + let mut scc_annotations = SccAnnotations::init(&definitions); + let mut constraint_sccs = compute_sccs(&outlives_constraints, &mut scc_annotations); + + let added_constraints = crate::handle_placeholders::rewrite_placeholder_outlives( + &constraint_sccs, + &scc_annotations, + universal_regions.fr_static, + &mut outlives_constraints, + ); + + if added_constraints { + scc_annotations = SccAnnotations::init(&definitions); + constraint_sccs = compute_sccs(&outlives_constraints, &mut scc_annotations); + } + + let scc_annotations = scc_annotations.scc_to_annotation; + + // Unlike the `RegionInferenceContext`, we only care about free regions + // and fully ignore liveness and placeholders. + let placeholder_indices = Default::default(); + let mut scc_values = + RegionValues::new(location_map, universal_regions.len(), placeholder_indices); + for variable in definitions.indices() { + let scc = constraint_sccs.scc(variable); + match definitions[variable].origin { + NllRegionVariableOrigin::FreeRegion => { + scc_values.add_element(scc, variable); + } + _ => {} + } + } + + let rev_scc_graph = ReverseSccGraph::compute(&constraint_sccs, universal_regions); + RegionCtxt { + infcx, + definitions, + universal_region_relations, + constraint_sccs, + scc_annotations, + rev_scc_graph, + scc_values, + } + } + + pub(super) fn representative(&self, vid: RegionVid) -> Representative { + let scc = self.constraint_sccs.scc(vid); + self.scc_annotations[scc].representative + } + + pub(crate) fn max_placeholder_universe_reached( + &self, + scc: ConstraintSccIndex, + ) -> UniverseIndex { + self.scc_annotations[scc].max_placeholder_universe_reached() + } + + pub(super) fn universal_regions(&self) -> &UniversalRegions<'tcx> { + &self.universal_region_relations.universal_regions + } + + pub(super) fn eval_equal(&self, r1_vid: RegionVid, r2_vid: RegionVid) -> bool { + let r1 = self.constraint_sccs.scc(r1_vid); + let r2 = self.constraint_sccs.scc(r2_vid); + + if r1 == r2 { + return true; + } + + let universal_outlives = |sub, sup| { + self.scc_values.universal_regions_outlived_by(sub).all(|r1| { + self.scc_values + .universal_regions_outlived_by(sup) + .any(|r2| self.universal_region_relations.outlives(r2, r1)) + }) + }; + universal_outlives(r1, r2) && universal_outlives(r2, r1) + } +} diff --git a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs index 604265f89408..e8da85eccef1 100644 --- a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs +++ b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs @@ -5,7 +5,6 @@ use rustc_data_structures::graph; use rustc_data_structures::graph::vec_graph::VecGraph; use rustc_middle::ty::RegionVid; -use crate::RegionInferenceContext; use crate::constraints::ConstraintSccIndex; use crate::region_infer::ConstraintSccs; use crate::universal_regions::UniversalRegions; @@ -57,12 +56,3 @@ impl ReverseSccGraph { .filter(move |r| duplicates.insert(*r)) } } - -impl RegionInferenceContext<'_> { - /// Return the reverse graph of the region SCCs, initialising it if needed. - pub(super) fn reverse_scc_graph(&self) -> &ReverseSccGraph { - self.rev_scc_graph.get_or_init(|| { - ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions()) - }) - } -} diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index f1427218cdb0..eb611fa34757 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -37,6 +37,7 @@ pub(crate) enum RegionElement { /// Records the CFG locations where each region is live. When we initially compute liveness, we use /// an interval matrix storing liveness ranges for each region-vid. +#[derive(Clone)] // FIXME(#146079) pub(crate) struct LivenessValues { /// The map from locations to points. location_map: Rc, @@ -194,6 +195,7 @@ impl LivenessValues { /// rustc to the internal `PlaceholderIndex` values that are used in /// NLL. #[derive(Debug, Default)] +#[derive(Clone)] // FIXME(#146079) pub(crate) struct PlaceholderIndices { indices: FxIndexSet, } diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs index ff92b4168a86..d6dbc7dd831f 100644 --- a/compiler/rustc_borrowck/src/renumber.rs +++ b/compiler/rustc_borrowck/src/renumber.rs @@ -21,10 +21,10 @@ pub(crate) fn renumber_mir<'tcx>( let mut renumberer = RegionRenumberer { infcx }; for body in promoted.iter_mut() { - renumberer.visit_body(body); + renumberer.visit_body_preserves_cfg(body); } - renumberer.visit_body(body); + renumberer.visit_body_preserves_cfg(body); } // The fields are used only for debugging output in `sccs_info`. @@ -66,7 +66,7 @@ impl<'a, 'tcx> RegionRenumberer<'a, 'tcx> { T: TypeFoldable>, F: Fn() -> RegionCtxt, { - let origin = NllRegionVariableOrigin::Existential { from_forall: false }; + let origin = NllRegionVariableOrigin::Existential { name: None }; fold_regions(self.infcx.tcx, value, |_region, _depth| { self.infcx.next_nll_region_var(origin, || region_ctxt_fn()) }) diff --git a/compiler/rustc_borrowck/src/root_cx.rs b/compiler/rustc_borrowck/src/root_cx.rs index 9b1d12aede51..21c11e128735 100644 --- a/compiler/rustc_borrowck/src/root_cx.rs +++ b/compiler/rustc_borrowck/src/root_cx.rs @@ -1,25 +1,43 @@ +use std::mem; +use std::rc::Rc; + use rustc_abi::FieldIdx; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_hir::def_id::LocalDefId; -use rustc_middle::bug; -use rustc_middle::ty::{OpaqueHiddenType, Ty, TyCtxt, TypeVisitableExt}; +use rustc_middle::mir::ConstraintCategory; +use rustc_middle::ty::{self, TyCtxt}; use rustc_span::ErrorGuaranteed; use smallvec::SmallVec; use crate::consumers::BorrowckConsumer; -use crate::{ClosureRegionRequirements, ConcreteOpaqueTypes, PropagatedBorrowCheckResults}; +use crate::nll::compute_closure_requirements_modulo_opaques; +use crate::region_infer::opaque_types::{ + apply_definition_site_hidden_types, clone_and_resolve_opaque_types, + compute_definition_site_hidden_types, detect_opaque_types_added_while_handling_opaque_types, +}; +use crate::type_check::{Locations, constraint_conversion}; +use crate::{ + ClosureRegionRequirements, CollectRegionConstraintsResult, DefinitionSiteHiddenTypes, + PropagatedBorrowCheckResults, borrowck_check_region_constraints, + borrowck_collect_region_constraints, +}; /// The shared context used by both the root as well as all its nested /// items. pub(super) struct BorrowCheckRootCtxt<'tcx> { pub tcx: TyCtxt<'tcx>, root_def_id: LocalDefId, - concrete_opaque_types: ConcreteOpaqueTypes<'tcx>, - nested_bodies: FxHashMap>, + hidden_types: DefinitionSiteHiddenTypes<'tcx>, + /// The region constraints computed by [borrowck_collect_region_constraints]. This uses + /// an [FxIndexMap] to guarantee that iterating over it visits nested bodies before + /// their parents. + collect_region_constraints_results: + FxIndexMap>, + propagated_borrowck_results: FxHashMap>, tainted_by_errors: Option, /// This should be `None` during normal compilation. See [`crate::consumers`] for more /// information on how this is used. - pub(crate) consumer: Option>, + pub consumer: Option>, } impl<'tcx> BorrowCheckRootCtxt<'tcx> { @@ -31,83 +49,244 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> { BorrowCheckRootCtxt { tcx, root_def_id, - concrete_opaque_types: Default::default(), - nested_bodies: Default::default(), + hidden_types: Default::default(), + collect_region_constraints_results: Default::default(), + propagated_borrowck_results: Default::default(), tainted_by_errors: None, consumer, } } - /// Collect all defining uses of opaque types inside of this typeck root. This - /// expects the hidden type to be mapped to the definition parameters of the opaque - /// and errors if we end up with distinct hidden types. - pub(super) fn add_concrete_opaque_type( - &mut self, - def_id: LocalDefId, - hidden_ty: OpaqueHiddenType<'tcx>, - ) { - // Sometimes two opaque types are the same only after we remap the generic parameters - // back to the opaque type definition. E.g. we may have `OpaqueType` mapped to - // `(X, Y)` and `OpaqueType` mapped to `(Y, X)`, and those are the same, but we - // only know that once we convert the generic parameters to those of the opaque type. - if let Some(prev) = self.concrete_opaque_types.0.get_mut(&def_id) { - if prev.ty != hidden_ty.ty { - let guar = hidden_ty.ty.error_reported().err().unwrap_or_else(|| { - let (Ok(e) | Err(e)) = - prev.build_mismatch_error(&hidden_ty, self.tcx).map(|d| d.emit()); - e - }); - prev.ty = Ty::new_error(self.tcx, guar); - } - // Pick a better span if there is one. - // FIXME(oli-obk): collect multiple spans for better diagnostics down the road. - prev.span = prev.span.substitute_dummy(hidden_ty.span); - } else { - self.concrete_opaque_types.0.insert(def_id, hidden_ty); - } + pub(super) fn root_def_id(&self) -> LocalDefId { + self.root_def_id } pub(super) fn set_tainted_by_errors(&mut self, guar: ErrorGuaranteed) { self.tainted_by_errors = Some(guar); } - pub(super) fn get_or_insert_nested( - &mut self, - def_id: LocalDefId, - ) -> &PropagatedBorrowCheckResults<'tcx> { - debug_assert_eq!( - self.tcx.typeck_root_def_id(def_id.to_def_id()), - self.root_def_id.to_def_id() - ); - if !self.nested_bodies.contains_key(&def_id) { - let result = super::do_mir_borrowck(self, def_id); - if let Some(prev) = self.nested_bodies.insert(def_id, result) { - bug!("unexpected previous nested body: {prev:?}"); - } - } - - self.nested_bodies.get(&def_id).unwrap() - } - - pub(super) fn closure_requirements( - &mut self, - nested_body_def_id: LocalDefId, - ) -> &Option> { - &self.get_or_insert_nested(nested_body_def_id).closure_requirements - } - pub(super) fn used_mut_upvars( &mut self, nested_body_def_id: LocalDefId, ) -> &SmallVec<[FieldIdx; 8]> { - &self.get_or_insert_nested(nested_body_def_id).used_mut_upvars + &self.propagated_borrowck_results[&nested_body_def_id].used_mut_upvars } - pub(super) fn finalize(self) -> Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> { + pub(super) fn finalize(self) -> Result<&'tcx DefinitionSiteHiddenTypes<'tcx>, ErrorGuaranteed> { if let Some(guar) = self.tainted_by_errors { Err(guar) } else { - Ok(self.tcx.arena.alloc(self.concrete_opaque_types)) + Ok(self.tcx.arena.alloc(self.hidden_types)) + } + } + + fn handle_opaque_type_uses(&mut self) { + let mut per_body_info = Vec::new(); + for input in self.collect_region_constraints_results.values_mut() { + let (num_entries, opaque_types) = clone_and_resolve_opaque_types( + &input.infcx, + &input.universal_region_relations, + &mut input.constraints, + ); + input.deferred_opaque_type_errors = compute_definition_site_hidden_types( + &input.infcx, + &input.universal_region_relations, + &input.constraints, + Rc::clone(&input.location_map), + &mut self.hidden_types, + &opaque_types, + ); + per_body_info.push((num_entries, opaque_types)); + } + + for (input, (opaque_types_storage_num_entries, opaque_types)) in + self.collect_region_constraints_results.values_mut().zip(per_body_info) + { + if input.deferred_opaque_type_errors.is_empty() { + input.deferred_opaque_type_errors = apply_definition_site_hidden_types( + &input.infcx, + &input.body_owned, + &input.universal_region_relations.universal_regions, + &input.region_bound_pairs, + &input.known_type_outlives_obligations, + &mut input.constraints, + &mut self.hidden_types, + &opaque_types, + ); + } + + detect_opaque_types_added_while_handling_opaque_types( + &input.infcx, + opaque_types_storage_num_entries, + ) + } + } + + /// Computing defining uses of opaques may depend on the propagated region + /// requirements of nested bodies, while applying defining uses may introduce + /// additional region requirements we need to propagate. + /// + /// This results in cyclic dependency. To compute the defining uses in parent + /// bodies, we need the closure requirements of its nested bodies, but to check + /// non-defining uses in nested bodies, we may rely on the defining uses in the + /// parent. + /// + /// We handle this issue by applying closure requirements twice. Once using the + /// region constraints from before we've handled opaque types in the nested body + /// - which is used by the parent to handle its defining uses - and once after. + /// + /// As a performance optimization, we also eagerly finish borrowck for bodies + /// which don't depend on opaque types. In this case they get removed from + /// `collect_region_constraints_results` and the final result gets put into + /// `propagated_borrowck_results`. + fn apply_closure_requirements_modulo_opaques(&mut self) { + let mut closure_requirements_modulo_opaques = FxHashMap::default(); + // We need to `mem::take` both `self.collect_region_constraints_results` and + // `input.deferred_closure_requirements` as we otherwise can't iterate over + // them while mutably using the containing struct. + let collect_region_constraints_results = + mem::take(&mut self.collect_region_constraints_results); + // We iterate over all bodies here, visiting nested bodies before their parent. + for (def_id, mut input) in collect_region_constraints_results { + // A body depends on opaque types if it either has any opaque type uses itself, + // or it has a nested body which does. + // + // If the current body does not depend on any opaque types, we eagerly compute + // its final result and write it into `self.propagated_borrowck_results`. This + // avoids having to compute its closure requirements modulo regions, as they + // are just the same as its final closure requirements. + let mut depends_on_opaques = input.infcx.has_opaque_types_in_storage(); + + // Iterate over all nested bodies of `input`. If that nested body depends on + // opaque types, we apply its closure requirements modulo opaques. Otherwise + // we use the closure requirements from its final borrowck result. + // + // In case we've only applied the closure requirements modulo opaques, we have + // to later apply its closure requirements considering opaques, so we put that + // nested body back into `deferred_closure_requirements`. + for (def_id, args, locations) in mem::take(&mut input.deferred_closure_requirements) { + let closure_requirements = match self.propagated_borrowck_results.get(&def_id) { + None => { + depends_on_opaques = true; + input.deferred_closure_requirements.push((def_id, args, locations)); + &closure_requirements_modulo_opaques[&def_id] + } + Some(result) => &result.closure_requirements, + }; + + Self::apply_closure_requirements( + &mut input, + closure_requirements, + def_id, + args, + locations, + ); + } + + // In case the current body does depend on opaques and is a nested body, + // we need to compute its closure requirements modulo opaques so that + // we're able to use it when visiting its parent later in this function. + // + // If the current body does not depend on opaque types, we finish borrowck + // and write its result into `propagated_borrowck_results`. + if depends_on_opaques { + if def_id != self.root_def_id { + let req = Self::compute_closure_requirements_modulo_opaques(&input); + closure_requirements_modulo_opaques.insert(def_id, req); + } + self.collect_region_constraints_results.insert(def_id, input); + } else { + assert!(input.deferred_closure_requirements.is_empty()); + let result = borrowck_check_region_constraints(self, input); + self.propagated_borrowck_results.insert(def_id, result); + } + } + } + + fn compute_closure_requirements_modulo_opaques( + input: &CollectRegionConstraintsResult<'tcx>, + ) -> Option> { + compute_closure_requirements_modulo_opaques( + &input.infcx, + &input.body_owned, + Rc::clone(&input.location_map), + &input.universal_region_relations, + &input.constraints, + ) + } + + fn apply_closure_requirements( + input: &mut CollectRegionConstraintsResult<'tcx>, + closure_requirements: &Option>, + closure_def_id: LocalDefId, + args: ty::GenericArgsRef<'tcx>, + locations: Locations, + ) { + if let Some(closure_requirements) = closure_requirements { + constraint_conversion::ConstraintConversion::new( + &input.infcx, + &input.universal_region_relations.universal_regions, + &input.region_bound_pairs, + &input.known_type_outlives_obligations, + locations, + input.body_owned.span, // irrelevant; will be overridden. + ConstraintCategory::Boring, // same as above. + &mut input.constraints, + ) + .apply_closure_requirements(closure_requirements, closure_def_id, args); + } + } + + pub(super) fn do_mir_borrowck(&mut self) { + // The list of all bodies we need to borrowck. This first looks at + // nested bodies, and then their parents. This means accessing e.g. + // `used_mut_upvars` for a closure can assume that we've already + // checked that closure. + let all_bodies = self + .tcx + .nested_bodies_within(self.root_def_id) + .iter() + .chain(std::iter::once(self.root_def_id)); + for def_id in all_bodies { + let result = borrowck_collect_region_constraints(self, def_id); + self.collect_region_constraints_results.insert(def_id, result); + } + + // We now apply the closure requirements of nested bodies modulo + // regions. In case a body does not depend on opaque types, we + // eagerly check its region constraints and use the final closure + // requirements. + // + // We eagerly finish borrowck for bodies which don't depend on + // opaques. + self.apply_closure_requirements_modulo_opaques(); + + // We handle opaque type uses for all bodies together. + self.handle_opaque_type_uses(); + + // Now walk over all bodies which depend on opaque types and finish borrowck. + // + // We first apply the final closure requirements from nested bodies which also + // depend on opaque types and then finish borrow checking the parent. Bodies + // which don't depend on opaques have already been fully borrowchecked in + // `apply_closure_requirements_modulo_opaques` as an optimization. + for (def_id, mut input) in mem::take(&mut self.collect_region_constraints_results) { + for (def_id, args, locations) in mem::take(&mut input.deferred_closure_requirements) { + // We visit nested bodies before their parent, so we're already + // done with nested bodies at this point. + let closure_requirements = + &self.propagated_borrowck_results[&def_id].closure_requirements; + Self::apply_closure_requirements( + &mut input, + closure_requirements, + def_id, + args, + locations, + ); + } + + let result = borrowck_check_region_constraints(self, input); + self.propagated_borrowck_results.insert(def_id, result); } } } diff --git a/compiler/rustc_borrowck/src/type_check/canonical.rs b/compiler/rustc_borrowck/src/type_check/canonical.rs index b3fa786a5177..aece0bda3469 100644 --- a/compiler/rustc_borrowck/src/type_check/canonical.rs +++ b/compiler/rustc_borrowck/src/type_check/canonical.rs @@ -2,8 +2,9 @@ use std::fmt; use rustc_errors::ErrorGuaranteed; use rustc_infer::infer::canonical::Canonical; +use rustc_infer::infer::outlives::env::RegionBoundPairs; use rustc_middle::bug; -use rustc_middle::mir::ConstraintCategory; +use rustc_middle::mir::{Body, ConstraintCategory}; use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, Upcast}; use rustc_span::Span; use rustc_span::def_id::DefId; @@ -14,7 +15,68 @@ use rustc_trait_selection::traits::query::type_op::{self, TypeOpOutput}; use tracing::{debug, instrument}; use super::{Locations, NormalizeLocation, TypeChecker}; +use crate::BorrowckInferCtxt; use crate::diagnostics::ToUniverseInfo; +use crate::type_check::{MirTypeckRegionConstraints, constraint_conversion}; +use crate::universal_regions::UniversalRegions; + +#[instrument(skip(infcx, constraints, op), level = "trace")] +pub(crate) fn fully_perform_op_raw<'tcx, R: fmt::Debug, Op>( + infcx: &BorrowckInferCtxt<'tcx>, + body: &Body<'tcx>, + universal_regions: &UniversalRegions<'tcx>, + region_bound_pairs: &RegionBoundPairs<'tcx>, + known_type_outlives_obligations: &[ty::PolyTypeOutlivesPredicate<'tcx>], + constraints: &mut MirTypeckRegionConstraints<'tcx>, + locations: Locations, + category: ConstraintCategory<'tcx>, + op: Op, +) -> Result +where + Op: type_op::TypeOp<'tcx, Output = R>, + Op::ErrorInfo: ToUniverseInfo<'tcx>, +{ + let old_universe = infcx.universe(); + + let TypeOpOutput { output, constraints: query_constraints, error_info } = + op.fully_perform(infcx, infcx.root_def_id, locations.span(body))?; + if cfg!(debug_assertions) { + let data = infcx.take_and_reset_region_constraints(); + if !data.is_empty() { + panic!("leftover region constraints: {data:#?}"); + } + } + + debug!(?output, ?query_constraints); + + if let Some(data) = query_constraints { + constraint_conversion::ConstraintConversion::new( + infcx, + universal_regions, + region_bound_pairs, + known_type_outlives_obligations, + locations, + locations.span(body), + category, + constraints, + ) + .convert_all(data); + } + + // If the query has created new universes and errors are going to be emitted, register the + // cause of these new universes for improved diagnostics. + let universe = infcx.universe(); + if old_universe != universe + && let Some(error_info) = error_info + { + let universe_info = error_info.to_universe_info(old_universe); + for u in (old_universe + 1)..=universe { + constraints.universe_causes.insert(u, universe_info.clone()); + } + } + + Ok(output) +} impl<'a, 'tcx> TypeChecker<'a, 'tcx> { /// Given some operation `op` that manipulates types, proves @@ -38,36 +100,17 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { Op: type_op::TypeOp<'tcx, Output = R>, Op::ErrorInfo: ToUniverseInfo<'tcx>, { - let old_universe = self.infcx.universe(); - - let TypeOpOutput { output, constraints, error_info } = - op.fully_perform(self.infcx, locations.span(self.body))?; - if cfg!(debug_assertions) { - let data = self.infcx.take_and_reset_region_constraints(); - if !data.is_empty() { - panic!("leftover region constraints: {data:#?}"); - } - } - - debug!(?output, ?constraints); - - if let Some(data) = constraints { - self.push_region_constraints(locations, category, data); - } - - // If the query has created new universes and errors are going to be emitted, register the - // cause of these new universes for improved diagnostics. - let universe = self.infcx.universe(); - if old_universe != universe - && let Some(error_info) = error_info - { - let universe_info = error_info.to_universe_info(old_universe); - for u in (old_universe + 1)..=universe { - self.constraints.universe_causes.insert(u, universe_info.clone()); - } - } - - Ok(output) + fully_perform_op_raw( + self.infcx, + self.body, + self.universal_regions, + self.region_bound_pairs, + self.known_type_outlives_obligations, + self.constraints, + locations, + category, + op, + ) } pub(super) fn instantiate_canonical( @@ -187,8 +230,14 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { location: impl NormalizeLocation, ) -> Ty<'tcx> { let tcx = self.tcx(); + let body = self.body; + + let cause = ObligationCause::misc( + location.to_locations().span(body), + body.source.def_id().expect_local(), + ); + if self.infcx.next_trait_solver() { - let body = self.body; let param_env = self.infcx.param_env; // FIXME: Make this into a real type op? self.fully_perform_op( @@ -198,10 +247,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { |ocx| { let structurally_normalize = |ty| { ocx.structurally_normalize_ty( - &ObligationCause::misc( - location.to_locations().span(body), - body.source.def_id().expect_local(), - ), + &cause, param_env, ty, ) @@ -210,6 +256,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let tail = tcx.struct_tail_raw( ty, + &cause, structurally_normalize, || {}, ); @@ -222,7 +269,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { .unwrap_or_else(|guar| Ty::new_error(tcx, guar)) } else { let mut normalize = |ty| self.normalize(ty, location); - let tail = tcx.struct_tail_raw(ty, &mut normalize, || {}); + let tail = tcx.struct_tail_raw(ty, &cause, &mut normalize, || {}); normalize(tail) } } diff --git a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs index 9bb96b94506a..703223e2e54a 100644 --- a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs +++ b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs @@ -1,10 +1,10 @@ use rustc_data_structures::fx::FxHashSet; use rustc_hir::def_id::LocalDefId; +use rustc_infer::infer::SubregionOrigin; use rustc_infer::infer::canonical::QueryRegionConstraints; use rustc_infer::infer::outlives::env::RegionBoundPairs; use rustc_infer::infer::outlives::obligations::{TypeOutlives, TypeOutlivesDelegate}; use rustc_infer::infer::region_constraints::{GenericKind, VerifyBound}; -use rustc_infer::infer::{InferCtxt, SubregionOrigin}; use rustc_infer::traits::query::type_op::DeeplyNormalize; use rustc_middle::bug; use rustc_middle::ty::{ @@ -18,10 +18,12 @@ use crate::constraints::OutlivesConstraint; use crate::region_infer::TypeTest; use crate::type_check::{Locations, MirTypeckRegionConstraints}; use crate::universal_regions::UniversalRegions; -use crate::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory}; +use crate::{ + BorrowckInferCtxt, ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory, +}; pub(crate) struct ConstraintConversion<'a, 'tcx> { - infcx: &'a InferCtxt<'tcx>, + infcx: &'a BorrowckInferCtxt<'tcx>, universal_regions: &'a UniversalRegions<'tcx>, /// Each RBP `GK: 'a` is assumed to be true. These encode /// relationships like `T: 'a` that are added via implicit bounds @@ -34,7 +36,6 @@ pub(crate) struct ConstraintConversion<'a, 'tcx> { /// logic expecting to see (e.g.) `ReStatic`, and if we supplied /// our special inference variable there, we would mess that up. region_bound_pairs: &'a RegionBoundPairs<'tcx>, - param_env: ty::ParamEnv<'tcx>, known_type_outlives_obligations: &'a [ty::PolyTypeOutlivesPredicate<'tcx>], locations: Locations, span: Span, @@ -45,10 +46,9 @@ pub(crate) struct ConstraintConversion<'a, 'tcx> { impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> { pub(crate) fn new( - infcx: &'a InferCtxt<'tcx>, + infcx: &'a BorrowckInferCtxt<'tcx>, universal_regions: &'a UniversalRegions<'tcx>, region_bound_pairs: &'a RegionBoundPairs<'tcx>, - param_env: ty::ParamEnv<'tcx>, known_type_outlives_obligations: &'a [ty::PolyTypeOutlivesPredicate<'tcx>], locations: Locations, span: Span, @@ -59,7 +59,6 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> { infcx, universal_regions, region_bound_pairs, - param_env, known_type_outlives_obligations, locations, span, @@ -286,8 +285,11 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> { ConstraintCategory<'tcx>, )>, ) -> Ty<'tcx> { - match self.param_env.and(DeeplyNormalize { value: ty }).fully_perform(self.infcx, self.span) - { + match self.infcx.param_env.and(DeeplyNormalize { value: ty }).fully_perform( + self.infcx, + self.infcx.root_def_id, + self.span, + ) { Ok(TypeOpOutput { output: ty, constraints, .. }) => { // FIXME(higher_ranked_auto): What should we do with the assumptions here? if let Some(QueryRegionConstraints { outlives, assumptions: _ }) = constraints { diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs index f642d34ea673..d27a73535bab 100644 --- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs +++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs @@ -2,9 +2,9 @@ use rustc_data_structures::frozen::Frozen; use rustc_data_structures::transitive_relation::{TransitiveRelation, TransitiveRelationBuilder}; use rustc_hir::def::DefKind; use rustc_infer::infer::canonical::QueryRegionConstraints; +use rustc_infer::infer::outlives; use rustc_infer::infer::outlives::env::RegionBoundPairs; use rustc_infer::infer::region_constraints::GenericKind; -use rustc_infer::infer::{InferCtxt, outlives}; use rustc_infer::traits::query::type_op::DeeplyNormalize; use rustc_middle::mir::ConstraintCategory; use rustc_middle::traits::query::OutlivesBound; @@ -14,10 +14,12 @@ use rustc_trait_selection::traits::query::type_op::{self, TypeOp}; use tracing::{debug, instrument}; use type_op::TypeOpOutput; +use crate::BorrowckInferCtxt; use crate::type_check::{Locations, MirTypeckRegionConstraints, constraint_conversion}; use crate::universal_regions::UniversalRegions; #[derive(Debug)] +#[derive(Clone)] // FIXME(#146079) pub(crate) struct UniversalRegionRelations<'tcx> { pub(crate) universal_regions: UniversalRegions<'tcx>, @@ -47,14 +49,12 @@ pub(crate) struct CreateResult<'tcx> { } pub(crate) fn create<'tcx>( - infcx: &InferCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + infcx: &BorrowckInferCtxt<'tcx>, universal_regions: UniversalRegions<'tcx>, constraints: &mut MirTypeckRegionConstraints<'tcx>, ) -> CreateResult<'tcx> { UniversalRegionRelationsBuilder { infcx, - param_env, constraints, universal_regions, region_bound_pairs: Default::default(), @@ -177,8 +177,7 @@ impl UniversalRegionRelations<'_> { } struct UniversalRegionRelationsBuilder<'a, 'tcx> { - infcx: &'a InferCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + infcx: &'a BorrowckInferCtxt<'tcx>, universal_regions: UniversalRegions<'tcx>, constraints: &'a mut MirTypeckRegionConstraints<'tcx>, @@ -205,7 +204,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { // Insert the `'a: 'b` we know from the predicates. // This does not consider the type-outlives. - let param_env = self.param_env; + let param_env = self.infcx.param_env; self.add_outlives_bounds(outlives::explicit_outlives_bounds(param_env)); // - outlives is reflexive, so `'r: 'r` for every region `'r` @@ -263,7 +262,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { let TypeOpOutput { output: norm_ty, constraints: constraints_normalize, .. } = param_env .and(DeeplyNormalize { value: ty }) - .fully_perform(self.infcx, span) + .fully_perform(self.infcx, self.infcx.root_def_id, span) .unwrap_or_else(|guar| TypeOpOutput { output: Ty::new_error(self.infcx.tcx, guar), constraints: None, @@ -298,8 +297,9 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { // Add implied bounds from impl header. if matches!(tcx.def_kind(defining_ty_def_id), DefKind::AssocFn | DefKind::AssocConst) { for &(ty, _) in tcx.assumed_wf_types(tcx.local_parent(defining_ty_def_id)) { - let result: Result<_, ErrorGuaranteed> = - param_env.and(DeeplyNormalize { value: ty }).fully_perform(self.infcx, span); + let result: Result<_, ErrorGuaranteed> = param_env + .and(DeeplyNormalize { value: ty }) + .fully_perform(self.infcx, self.infcx.root_def_id, span); let Ok(TypeOpOutput { output: norm_ty, constraints: c, .. }) = result else { continue; }; @@ -318,7 +318,6 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { self.infcx, &self.universal_regions, &self.region_bound_pairs, - param_env, &known_type_outlives_obligations, Locations::All(span), span, @@ -353,10 +352,11 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { output: normalized_outlives, constraints: constraints_normalize, error_info: _, - }) = self - .param_env - .and(DeeplyNormalize { value: outlives }) - .fully_perform(self.infcx, span) + }) = self.infcx.param_env.and(DeeplyNormalize { value: outlives }).fully_perform( + self.infcx, + self.infcx.root_def_id, + span, + ) else { self.infcx.dcx().delayed_bug(format!("could not normalize {outlives:?}")); return; @@ -381,9 +381,10 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> { span: Span, ) -> Option<&'tcx QueryRegionConstraints<'tcx>> { let TypeOpOutput { output: bounds, constraints, .. } = self + .infcx .param_env .and(type_op::ImpliedOutlivesBounds { ty }) - .fully_perform(self.infcx, span) + .fully_perform(self.infcx, self.infcx.root_def_id, span) .map_err(|_: ErrorGuaranteed| debug!("failed to compute implied bounds {:?}", ty)) .ok()?; debug!(?bounds, ?constraints); diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs index 99392ea19151..eb31b5de05d2 100644 --- a/compiler/rustc_borrowck/src/type_check/input_output.rs +++ b/compiler/rustc_borrowck/src/type_check/input_output.rs @@ -86,7 +86,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // them with fresh ty vars. resume_ty: next_ty_var(), yield_ty: next_ty_var(), - witness: next_ty_var(), }, ) .args, diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index 5d30fa71e92c..7ac94020de03 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -640,7 +640,7 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { let op = typeck.infcx.param_env.and(DropckOutlives { dropped_ty }); - match op.fully_perform(typeck.infcx, DUMMY_SP) { + match op.fully_perform(typeck.infcx, typeck.root_cx.root_def_id(), DUMMY_SP) { Ok(TypeOpOutput { output, constraints, .. }) => { DropData { dropck_result: output, region_constraint_data: constraints } } @@ -657,7 +657,7 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> { let errors = match dropck_outlives::compute_dropck_outlives_with_errors( &ocx, op, span, ) { - Ok(_) => ocx.select_all_or_error(), + Ok(_) => ocx.evaluate_obligations_error_on_ambiguity(), Err(e) => e, }; diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index f5fedbf95c1c..d1d0bff8fe06 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -26,8 +26,7 @@ use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::cast::CastTy; use rustc_middle::ty::{ self, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, CoroutineArgsExt, - GenericArgsRef, OpaqueHiddenType, OpaqueTypeKey, RegionVid, Ty, TyCtxt, TypeVisitableExt, - UserArgs, UserTypeAnnotationIndex, fold_regions, + GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::move_paths::MoveData; @@ -35,6 +34,7 @@ use rustc_mir_dataflow::points::DenseLocationMap; use rustc_span::def_id::CRATE_DEF_ID; use rustc_span::source_map::Spanned; use rustc_span::{Span, sym}; +use rustc_trait_selection::infer::InferCtxtExt; use rustc_trait_selection::traits::query::type_op::custom::scrape_region_constraints; use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput}; use tracing::{debug, instrument, trace}; @@ -42,7 +42,6 @@ use tracing::{debug, instrument, trace}; use crate::borrow_set::BorrowSet; use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet}; use crate::diagnostics::UniverseInfo; -use crate::member_constraints::MemberConstraintSet; use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable}; use crate::polonius::{PoloniusContext, PoloniusLivenessContext}; use crate::region_infer::TypeTest; @@ -50,7 +49,7 @@ use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderI use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst}; use crate::type_check::free_region_relations::{CreateResult, UniversalRegionRelations}; use crate::universal_regions::{DefiningTy, UniversalRegions}; -use crate::{BorrowCheckRootCtxt, BorrowckInferCtxt, path_utils}; +use crate::{BorrowCheckRootCtxt, BorrowckInferCtxt, DeferredClosureRequirements, path_utils}; macro_rules! span_mirbug { ($context:expr, $elem:expr, $($message:tt)*) => ({ @@ -67,12 +66,11 @@ macro_rules! span_mirbug { }) } -mod canonical; -mod constraint_conversion; +pub(crate) mod canonical; +pub(crate) mod constraint_conversion; pub(crate) mod free_region_relations; mod input_output; pub(crate) mod liveness; -mod opaque_types; mod relate_tys; /// Type checks the given `mir` in the context of the inference @@ -114,7 +112,6 @@ pub(crate) fn type_check<'tcx>( placeholder_index_to_region: IndexVec::default(), liveness_constraints: LivenessValues::with_specific_points(Rc::clone(&location_map)), outlives_constraints: OutlivesConstraintSet::default(), - member_constraints: MemberConstraintSet::default(), type_tests: Vec::default(), universe_causes: FxIndexMap::default(), }; @@ -124,7 +121,7 @@ pub(crate) fn type_check<'tcx>( region_bound_pairs, normalized_inputs_and_output, known_type_outlives_obligations, - } = free_region_relations::create(infcx, infcx.param_env, universal_regions, &mut constraints); + } = free_region_relations::create(infcx, universal_regions, &mut constraints); let pre_obligations = infcx.take_registered_region_obligations(); assert!( @@ -145,6 +142,7 @@ pub(crate) fn type_check<'tcx>( None }; + let mut deferred_closure_requirements = Default::default(); let mut typeck = TypeChecker { root_cx, infcx, @@ -160,6 +158,7 @@ pub(crate) fn type_check<'tcx>( polonius_facts, borrow_set, constraints: &mut constraints, + deferred_closure_requirements: &mut deferred_closure_requirements, polonius_liveness, }; @@ -170,9 +169,6 @@ pub(crate) fn type_check<'tcx>( liveness::generate(&mut typeck, &location_map, move_data); - let opaque_type_values = - opaque_types::take_opaques_and_register_member_constraints(&mut typeck); - // We're done with typeck, we can finalize the polonius liveness context for region inference. let polonius_context = typeck.polonius_liveness.take().map(|liveness_context| { PoloniusContext::create_from_liveness( @@ -182,10 +178,22 @@ pub(crate) fn type_check<'tcx>( ) }); + // In case type check encountered an error region, we suppress unhelpful extra + // errors in by clearing out all outlives bounds that we may end up checking. + if let Some(guar) = universal_region_relations.universal_regions.encountered_re_error() { + debug!("encountered an error region; removing constraints!"); + constraints.outlives_constraints = Default::default(); + constraints.type_tests = Default::default(); + root_cx.set_tainted_by_errors(guar); + infcx.set_tainted_by_errors(guar); + } + MirTypeckResults { constraints, universal_region_relations, - opaque_type_values, + region_bound_pairs, + known_type_outlives_obligations, + deferred_closure_requirements, polonius_context, } } @@ -225,6 +233,7 @@ struct TypeChecker<'a, 'tcx> { polonius_facts: &'a mut Option, borrow_set: &'a BorrowSet<'tcx>, constraints: &'a mut MirTypeckRegionConstraints<'tcx>, + deferred_closure_requirements: &'a mut DeferredClosureRequirements<'tcx>, /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints. polonius_liveness: Option, } @@ -234,12 +243,15 @@ struct TypeChecker<'a, 'tcx> { pub(crate) struct MirTypeckResults<'tcx> { pub(crate) constraints: MirTypeckRegionConstraints<'tcx>, pub(crate) universal_region_relations: Frozen>, - pub(crate) opaque_type_values: FxIndexMap, OpaqueHiddenType<'tcx>>, + pub(crate) region_bound_pairs: Frozen>, + pub(crate) known_type_outlives_obligations: Frozen>>, + pub(crate) deferred_closure_requirements: DeferredClosureRequirements<'tcx>, pub(crate) polonius_context: Option, } /// A collection of region constraints that must be satisfied for the /// program to be considered well-typed. +#[derive(Clone)] // FIXME(#146079) pub(crate) struct MirTypeckRegionConstraints<'tcx> { /// Maps from a `ty::Placeholder` to the corresponding /// `PlaceholderIndex` bit that we will use for it. @@ -266,8 +278,6 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> { pub(crate) outlives_constraints: OutlivesConstraintSet<'tcx>, - pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>, - pub(crate) universe_causes: FxIndexMap>, pub(crate) type_tests: Vec>, @@ -276,7 +286,7 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> { impl<'tcx> MirTypeckRegionConstraints<'tcx> { /// Creates a `Region` for a given `PlaceholderRegion`, or returns the /// region that corresponds to a previously created one. - fn placeholder_region( + pub(crate) fn placeholder_region( &mut self, infcx: &InferCtxt<'tcx>, placeholder: ty::PlaceholderRegion, @@ -369,14 +379,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { self.body } - fn to_region_vid(&mut self, r: ty::Region<'tcx>) -> RegionVid { - if let ty::RePlaceholder(placeholder) = r.kind() { - self.constraints.placeholder_region(self.infcx, placeholder).as_var() - } else { - self.universal_regions.to_region_vid(r) - } - } - fn unsized_feature_enabled(&self) -> bool { self.tcx().features().unsized_fn_params() } @@ -413,7 +415,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { self.infcx, self.universal_regions, self.region_bound_pairs, - self.infcx.param_env, self.known_type_outlives_obligations, locations, locations.span(self.body), @@ -504,6 +505,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let mut constraints = Default::default(); let mut liveness_constraints = LivenessValues::without_specific_points(Rc::new(DenseLocationMap::new(promoted_body))); + let mut deferred_closure_requirements = Default::default(); // Don't try to add borrow_region facts for the promoted MIR as they refer // to the wrong locations. @@ -511,6 +513,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { mem::swap(this.polonius_facts, polonius_facts); mem::swap(&mut this.constraints.outlives_constraints, &mut constraints); mem::swap(&mut this.constraints.liveness_constraints, &mut liveness_constraints); + mem::swap(this.deferred_closure_requirements, &mut deferred_closure_requirements); }; swap_constraints(self); @@ -535,6 +538,17 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { } self.constraints.outlives_constraints.push(constraint) } + + // If there are nested bodies in promoteds, we also need to update their + // location to something in the actual body, not the promoted. + // + // We don't update the constraint categories of the resulting constraints + // as returns in nested bodies are a proper return, even if that nested body + // is in a promoted. + for (closure_def_id, args, _locations) in deferred_closure_requirements { + self.deferred_closure_requirements.push((closure_def_id, args, locations)); + } + // If the region is live at least one location in the promoted MIR, // then add a liveness constraint to the main MIR for this region // at the location provided as an argument to this method @@ -734,7 +748,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => {} StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(..)) - | StatementKind::Deinit(..) | StatementKind::SetDiscriminant { .. } => { bug!("Statement not allowed in this MIR phase") } @@ -769,9 +782,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } TerminatorKind::Call { func, args, .. } | TerminatorKind::TailCall { func, args, .. } => { - let call_source = match term.kind { - TerminatorKind::Call { call_source, .. } => call_source, - TerminatorKind::TailCall { .. } => CallSource::Normal, + let (call_source, destination, is_diverging) = match term.kind { + TerminatorKind::Call { call_source, destination, target, .. } => { + (call_source, destination, target.is_none()) + } + TerminatorKind::TailCall { .. } => { + (CallSource::Normal, RETURN_PLACE.into(), false) + } _ => unreachable!(), }; @@ -845,9 +862,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } - if let TerminatorKind::Call { destination, target, .. } = term.kind { - self.check_call_dest(term, &sig, destination, target, term_location); - } + self.check_call_dest(term, &sig, destination, is_diverging, term_location); // The ordinary liveness rules will ensure that all // regions in the type of the callee are live here. We @@ -1031,16 +1046,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } - &Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, ty) => { - let trait_ref = - ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, span), [ty]); - - self.prove_trait_ref( - trait_ref, - location.to_locations(), - ConstraintCategory::SizedBound, - ); - } &Rvalue::NullaryOp(NullOp::ContractChecks, _) => {} &Rvalue::NullaryOp(NullOp::UbChecks, _) => {} @@ -1458,68 +1463,77 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } CastKind::PtrToPtr => { let ty_from = op.ty(self.body, tcx); - let cast_ty_from = CastTy::from_ty(ty_from); - let cast_ty_to = CastTy::from_ty(*ty); - match (cast_ty_from, cast_ty_to) { - (Some(CastTy::Ptr(src)), Some(CastTy::Ptr(dst))) => { - let src_tail = self.struct_tail(src.ty, location); - let dst_tail = self.struct_tail(dst.ty, location); + let Some(CastTy::Ptr(src)) = CastTy::from_ty(ty_from) else { + unreachable!(); + }; + let Some(CastTy::Ptr(dst)) = CastTy::from_ty(*ty) else { + unreachable!(); + }; - // This checks (lifetime part of) vtable validity for pointer casts, - // which is irrelevant when there are aren't principal traits on - // both sides (aka only auto traits). - // - // Note that other checks (such as denying `dyn Send` -> `dyn - // Debug`) are in `rustc_hir_typeck`. - if let ty::Dynamic(src_tty, _src_lt, ty::Dyn) = *src_tail.kind() - && let ty::Dynamic(dst_tty, dst_lt, ty::Dyn) = *dst_tail.kind() - && src_tty.principal().is_some() - && dst_tty.principal().is_some() - { - // Remove auto traits. - // Auto trait checks are handled in `rustc_hir_typeck` as FCW. - let src_obj = Ty::new_dynamic( - tcx, - tcx.mk_poly_existential_predicates( - &src_tty.without_auto_traits().collect::>(), - ), - // FIXME: Once we disallow casting `*const dyn Trait + 'short` - // to `*const dyn Trait + 'long`, then this can just be `src_lt`. - dst_lt, - ty::Dyn, - ); - let dst_obj = Ty::new_dynamic( - tcx, - tcx.mk_poly_existential_predicates( - &dst_tty.without_auto_traits().collect::>(), - ), - dst_lt, - ty::Dyn, - ); + if self.infcx.type_is_sized_modulo_regions(self.infcx.param_env, dst.ty) { + // Wide to thin ptr cast. This may even occur in an env with + // impossible predicates, such as `where dyn Trait: Sized`. + // In this case, we don't want to fall into the case below, + // since the types may not actually be equatable, but it's + // fine to perform this operation in an impossible env. + let trait_ref = ty::TraitRef::new( + tcx, + tcx.require_lang_item(LangItem::Sized, self.last_span), + [dst.ty], + ); + self.prove_trait_ref( + trait_ref, + location.to_locations(), + ConstraintCategory::Cast { + is_implicit_coercion: true, + unsize_to: None, + }, + ); + } else if let ty::Dynamic(src_tty, _src_lt) = + *self.struct_tail(src.ty, location).kind() + && let ty::Dynamic(dst_tty, dst_lt) = + *self.struct_tail(dst.ty, location).kind() + && src_tty.principal().is_some() + && dst_tty.principal().is_some() + { + // This checks (lifetime part of) vtable validity for pointer casts, + // which is irrelevant when there are aren't principal traits on + // both sides (aka only auto traits). + // + // Note that other checks (such as denying `dyn Send` -> `dyn + // Debug`) are in `rustc_hir_typeck`. - debug!(?src_tty, ?dst_tty, ?src_obj, ?dst_obj); + // Remove auto traits. + // Auto trait checks are handled in `rustc_hir_typeck` as FCW. + let src_obj = Ty::new_dynamic( + tcx, + tcx.mk_poly_existential_predicates( + &src_tty.without_auto_traits().collect::>(), + ), + // FIXME: Once we disallow casting `*const dyn Trait + 'short` + // to `*const dyn Trait + 'long`, then this can just be `src_lt`. + dst_lt, + ); + let dst_obj = Ty::new_dynamic( + tcx, + tcx.mk_poly_existential_predicates( + &dst_tty.without_auto_traits().collect::>(), + ), + dst_lt, + ); - self.sub_types( - src_obj, - dst_obj, - location.to_locations(), - ConstraintCategory::Cast { - is_implicit_coercion: false, - unsize_to: None, - }, - ) - .unwrap(); - } - } - _ => { - span_mirbug!( - self, - rvalue, - "Invalid PtrToPtr cast {:?} -> {:?}", - ty_from, - ty - ) - } + debug!(?src_tty, ?dst_tty, ?src_obj, ?dst_obj); + + self.sub_types( + src_obj, + dst_obj, + location.to_locations(), + ConstraintCategory::Cast { + is_implicit_coercion: false, + unsize_to: None, + }, + ) + .unwrap(); } } CastKind::Transmute => { @@ -1533,6 +1547,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ), } } + CastKind::Subtype => { + bug!("CastKind::Subtype shouldn't exist in borrowck") + } } } @@ -1617,7 +1634,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { | Rvalue::BinaryOp(..) | Rvalue::RawPtr(..) | Rvalue::ThreadLocalRef(..) - | Rvalue::Len(..) | Rvalue::Discriminant(..) | Rvalue::NullaryOp(NullOp::OffsetOf(..), _) => {} } @@ -1760,10 +1776,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { locations, ); - assert!(!matches!( - tcx.impl_of_assoc(def_id).map(|imp| tcx.def_kind(imp)), - Some(DefKind::Impl { of_trait: true }) - )); + assert_eq!(tcx.trait_impl_of_assoc(def_id), None); self.prove_predicates( args.types().map(|ty| ty::ClauseKind::WellFormed(ty.into())), locations, @@ -1861,9 +1874,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ) .unwrap(); } - ProjectionElem::Subtype(_) => { - bug!("ProjectionElem::Subtype shouldn't exist in borrowck") - } } } } @@ -1874,65 +1884,61 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { term: &Terminator<'tcx>, sig: &ty::FnSig<'tcx>, destination: Place<'tcx>, - target: Option, + is_diverging: bool, term_location: Location, ) { let tcx = self.tcx(); - match target { - Some(_) => { - let dest_ty = destination.ty(self.body, tcx).ty; - let dest_ty = self.normalize(dest_ty, term_location); - let category = match destination.as_local() { - Some(RETURN_PLACE) => { - if let DefiningTy::Const(def_id, _) | DefiningTy::InlineConst(def_id, _) = - self.universal_regions.defining_ty - { - if tcx.is_static(def_id) { - ConstraintCategory::UseAsStatic - } else { - ConstraintCategory::UseAsConst - } - } else { - ConstraintCategory::Return(ReturnConstraint::Normal) - } - } - Some(l) if !self.body.local_decls[l].is_user_variable() => { - ConstraintCategory::Boring - } - // The return type of a call is interesting for diagnostics. - _ => ConstraintCategory::Assignment, - }; - - let locations = term_location.to_locations(); - - if let Err(terr) = self.sub_types(sig.output(), dest_ty, locations, category) { - span_mirbug!( - self, - term, - "call dest mismatch ({:?} <- {:?}): {:?}", - dest_ty, - sig.output(), - terr - ); - } - - // When `unsized_fn_params` is not enabled, - // this check is done at `check_local`. - if self.unsized_feature_enabled() { - let span = term.source_info.span; - self.ensure_place_sized(dest_ty, span); - } + if is_diverging { + // The signature in this call can reference region variables, + // so erase them before calling a query. + let output_ty = self.tcx().erase_and_anonymize_regions(sig.output()); + if !output_ty + .is_privately_uninhabited(self.tcx(), self.infcx.typing_env(self.infcx.param_env)) + { + span_mirbug!(self, term, "call to non-diverging function {:?} w/o dest", sig); } - None => { - // The signature in this call can reference region variables, - // so erase them before calling a query. - let output_ty = self.tcx().erase_regions(sig.output()); - if !output_ty.is_privately_uninhabited( - self.tcx(), - self.infcx.typing_env(self.infcx.param_env), - ) { - span_mirbug!(self, term, "call to converging function {:?} w/o dest", sig); + } else { + let dest_ty = destination.ty(self.body, tcx).ty; + let dest_ty = self.normalize(dest_ty, term_location); + let category = match destination.as_local() { + Some(RETURN_PLACE) => { + if let DefiningTy::Const(def_id, _) | DefiningTy::InlineConst(def_id, _) = + self.universal_regions.defining_ty + { + if tcx.is_static(def_id) { + ConstraintCategory::UseAsStatic + } else { + ConstraintCategory::UseAsConst + } + } else { + ConstraintCategory::Return(ReturnConstraint::Normal) + } } + Some(l) if !self.body.local_decls[l].is_user_variable() => { + ConstraintCategory::Boring + } + // The return type of a call is interesting for diagnostics. + _ => ConstraintCategory::Assignment, + }; + + let locations = term_location.to_locations(); + + if let Err(terr) = self.sub_types(sig.output(), dest_ty, locations, category) { + span_mirbug!( + self, + term, + "call dest mismatch ({:?} <- {:?}): {:?}", + dest_ty, + sig.output(), + terr + ); + } + + // When `unsized_fn_params` is not enabled, + // this check is done at `check_local`. + if self.unsized_feature_enabled() { + let span = term.source_info.span; + self.ensure_place_sized(dest_ty, span); } } } @@ -1979,7 +1985,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { let op_arg_ty = self.normalize(op_arg_ty, term_location); let category = if call_source.from_hir_call() { - ConstraintCategory::CallArgument(Some(self.infcx.tcx.erase_regions(func_ty))) + ConstraintCategory::CallArgument(Some( + self.infcx.tcx.erase_and_anonymize_regions(func_ty), + )) } else { ConstraintCategory::Boring }; @@ -2113,7 +2121,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // Erase the regions from `ty` to get a global type. The // `Sized` bound in no way depends on precise regions, so this // shouldn't affect `is_sized`. - let erased_ty = tcx.erase_regions(ty); + let erased_ty = tcx.erase_and_anonymize_regions(ty); // FIXME(#132279): Using `Ty::is_sized` causes us to incorrectly handle opaques here. if !erased_ty.is_sized(tcx, self.infcx.typing_env(self.infcx.param_env)) { // in current MIR construction, all non-control-flow rvalue @@ -2192,7 +2200,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { | Rvalue::Repeat(..) | Rvalue::Ref(..) | Rvalue::RawPtr(..) - | Rvalue::Len(..) | Rvalue::Cast(..) | Rvalue::ShallowInitBox(..) | Rvalue::BinaryOp(..) @@ -2394,9 +2401,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { | ProjectionElem::UnwrapUnsafeBinder(_) => { // other field access } - ProjectionElem::Subtype(_) => { - bug!("ProjectionElem::Subtype shouldn't exist in borrowck") - } } } } @@ -2468,24 +2472,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { args: GenericArgsRef<'tcx>, locations: Locations, ) -> ty::InstantiatedPredicates<'tcx> { - if let Some(closure_requirements) = &self.root_cx.closure_requirements(def_id) { - constraint_conversion::ConstraintConversion::new( - self.infcx, - self.universal_regions, - self.region_bound_pairs, - self.infcx.param_env, - self.known_type_outlives_obligations, - locations, - self.body.span, // irrelevant; will be overridden. - ConstraintCategory::Boring, // same as above. - self.constraints, - ) - .apply_closure_requirements(closure_requirements, def_id, args); - } + let root_def_id = self.root_cx.root_def_id(); + // We will have to handle propagated closure requirements for this closure, + // but need to defer this until the nested body has been fully borrow checked. + self.deferred_closure_requirements.push((def_id, args, locations)); - // Now equate closure args to regions inherited from `typeck_root_def_id`. Fixes #98589. - let typeck_root_def_id = tcx.typeck_root_def_id(self.body.source.def_id()); - let typeck_root_args = ty::GenericArgs::identity_for_item(tcx, typeck_root_def_id); + // Equate closure args to regions inherited from `root_def_id`. Fixes #98589. + let typeck_root_args = ty::GenericArgs::identity_for_item(tcx, root_def_id); let parent_args = match tcx.def_kind(def_id) { // We don't want to dispatch on 3 different kind of closures here, so take @@ -2560,17 +2553,14 @@ impl<'tcx> TypeOp<'tcx> for InstantiateOpaqueType<'tcx> { fn fully_perform( mut self, infcx: &InferCtxt<'tcx>, + root_def_id: LocalDefId, span: Span, ) -> Result, ErrorGuaranteed> { - let (mut output, region_constraints) = scrape_region_constraints( - infcx, - |ocx| { + let (mut output, region_constraints) = + scrape_region_constraints(infcx, root_def_id, "InstantiateOpaqueType", span, |ocx| { ocx.register_obligations(self.obligations.clone()); Ok(()) - }, - "InstantiateOpaqueType", - span, - )?; + })?; self.region_constraints = Some(region_constraints); output.error_info = Some(self); Ok(output) diff --git a/compiler/rustc_borrowck/src/type_check/opaque_types.rs b/compiler/rustc_borrowck/src/type_check/opaque_types.rs deleted file mode 100644 index 5a422483eef4..000000000000 --- a/compiler/rustc_borrowck/src/type_check/opaque_types.rs +++ /dev/null @@ -1,333 +0,0 @@ -use std::iter; - -use rustc_data_structures::fx::FxIndexMap; -use rustc_middle::span_bug; -use rustc_middle::ty::{ - self, GenericArgKind, OpaqueHiddenType, OpaqueTypeKey, Ty, TyCtxt, TypeSuperVisitable, - TypeVisitable, TypeVisitableExt, TypeVisitor, fold_regions, -}; -use tracing::{debug, trace}; - -use super::{MemberConstraintSet, TypeChecker}; - -/// Once we're done with typechecking the body, we take all the opaque types -/// defined by this function and add their 'member constraints'. -pub(super) fn take_opaques_and_register_member_constraints<'tcx>( - typeck: &mut TypeChecker<'_, 'tcx>, -) -> FxIndexMap, OpaqueHiddenType<'tcx>> { - let infcx = typeck.infcx; - // Annoying: to invoke `typeck.to_region_vid`, we need access to - // `typeck.constraints`, but we also want to be mutating - // `typeck.member_constraints`. For now, just swap out the value - // we want and replace at the end. - let mut member_constraints = std::mem::take(&mut typeck.constraints.member_constraints); - let opaque_types = infcx - .take_opaque_types() - .into_iter() - .map(|(opaque_type_key, hidden_type)| { - let hidden_type = infcx.resolve_vars_if_possible(hidden_type); - register_member_constraints( - typeck, - &mut member_constraints, - opaque_type_key, - hidden_type, - ); - trace!("finalized opaque type {:?} to {:#?}", opaque_type_key, hidden_type.ty.kind()); - if hidden_type.has_non_region_infer() { - span_bug!(hidden_type.span, "could not resolve {:?}", hidden_type.ty); - } - - // Convert all regions to nll vars. - let (opaque_type_key, hidden_type) = - fold_regions(infcx.tcx, (opaque_type_key, hidden_type), |r, _| { - ty::Region::new_var(infcx.tcx, typeck.to_region_vid(r)) - }); - - (opaque_type_key, hidden_type) - }) - .collect(); - assert!(typeck.constraints.member_constraints.is_empty()); - typeck.constraints.member_constraints = member_constraints; - opaque_types -} - -/// Given the map `opaque_types` containing the opaque -/// `impl Trait` types whose underlying, hidden types are being -/// inferred, this method adds constraints to the regions -/// appearing in those underlying hidden types to ensure that they -/// at least do not refer to random scopes within the current -/// function. These constraints are not (quite) sufficient to -/// guarantee that the regions are actually legal values; that -/// final condition is imposed after region inference is done. -/// -/// # The Problem -/// -/// Let's work through an example to explain how it works. Assume -/// the current function is as follows: -/// -/// ```text -/// fn foo<'a, 'b>(..) -> (impl Bar<'a>, impl Bar<'b>) -/// ``` -/// -/// Here, we have two `impl Trait` types whose values are being -/// inferred (the `impl Bar<'a>` and the `impl -/// Bar<'b>`). Conceptually, this is sugar for a setup where we -/// define underlying opaque types (`Foo1`, `Foo2`) and then, in -/// the return type of `foo`, we *reference* those definitions: -/// -/// ```text -/// type Foo1<'x> = impl Bar<'x>; -/// type Foo2<'x> = impl Bar<'x>; -/// fn foo<'a, 'b>(..) -> (Foo1<'a>, Foo2<'b>) { .. } -/// // ^^^^ ^^ -/// // | | -/// // | args -/// // def_id -/// ``` -/// -/// As indicating in the comments above, each of those references -/// is (in the compiler) basically generic parameters (`args`) -/// applied to the type of a suitable `def_id` (which identifies -/// `Foo1` or `Foo2`). -/// -/// Now, at this point in compilation, what we have done is to -/// replace each of the references (`Foo1<'a>`, `Foo2<'b>`) with -/// fresh inference variables C1 and C2. We wish to use the values -/// of these variables to infer the underlying types of `Foo1` and -/// `Foo2`. That is, this gives rise to higher-order (pattern) unification -/// constraints like: -/// -/// ```text -/// for<'a> (Foo1<'a> = C1) -/// for<'b> (Foo1<'b> = C2) -/// ``` -/// -/// For these equation to be satisfiable, the types `C1` and `C2` -/// can only refer to a limited set of regions. For example, `C1` -/// can only refer to `'static` and `'a`, and `C2` can only refer -/// to `'static` and `'b`. The job of this function is to impose that -/// constraint. -/// -/// Up to this point, C1 and C2 are basically just random type -/// inference variables, and hence they may contain arbitrary -/// regions. In fact, it is fairly likely that they do! Consider -/// this possible definition of `foo`: -/// -/// ```text -/// fn foo<'a, 'b>(x: &'a i32, y: &'b i32) -> (impl Bar<'a>, impl Bar<'b>) { -/// (&*x, &*y) -/// } -/// ``` -/// -/// Here, the values for the concrete types of the two impl -/// traits will include inference variables: -/// -/// ```text -/// &'0 i32 -/// &'1 i32 -/// ``` -/// -/// Ordinarily, the subtyping rules would ensure that these are -/// sufficiently large. But since `impl Bar<'a>` isn't a specific -/// type per se, we don't get such constraints by default. This -/// is where this function comes into play. It adds extra -/// constraints to ensure that all the regions which appear in the -/// inferred type are regions that could validly appear. -/// -/// This is actually a bit of a tricky constraint in general. We -/// want to say that each variable (e.g., `'0`) can only take on -/// values that were supplied as arguments to the opaque type -/// (e.g., `'a` for `Foo1<'a>`) or `'static`, which is always in -/// scope. We don't have a constraint quite of this kind in the current -/// region checker. -/// -/// # The Solution -/// -/// We generally prefer to make `<=` constraints, since they -/// integrate best into the region solver. To do that, we find the -/// "minimum" of all the arguments that appear in the args: that -/// is, some region which is less than all the others. In the case -/// of `Foo1<'a>`, that would be `'a` (it's the only choice, after -/// all). Then we apply that as a least bound to the variables -/// (e.g., `'a <= '0`). -/// -/// In some cases, there is no minimum. Consider this example: -/// -/// ```text -/// fn baz<'a, 'b>() -> impl Trait<'a, 'b> { ... } -/// ``` -/// -/// Here we would report a more complex "in constraint", like `'r -/// in ['a, 'b, 'static]` (where `'r` is some region appearing in -/// the hidden type). -/// -/// # Constrain regions, not the hidden concrete type -/// -/// Note that generating constraints on each region `Rc` is *not* -/// the same as generating an outlives constraint on `Tc` itself. -/// For example, if we had a function like this: -/// -/// ``` -/// # #![feature(type_alias_impl_trait)] -/// # fn main() {} -/// # trait Foo<'a> {} -/// # impl<'a, T> Foo<'a> for (&'a u32, T) {} -/// fn foo<'a, T>(x: &'a u32, y: T) -> impl Foo<'a> { -/// (x, y) -/// } -/// -/// // Equivalent to: -/// # mod dummy { use super::*; -/// type FooReturn<'a, T> = impl Foo<'a>; -/// #[define_opaque(FooReturn)] -/// fn foo<'a, T>(x: &'a u32, y: T) -> FooReturn<'a, T> { -/// (x, y) -/// } -/// # } -/// ``` -/// -/// then the hidden type `Tc` would be `(&'0 u32, T)` (where `'0` -/// is an inference variable). If we generated a constraint that -/// `Tc: 'a`, then this would incorrectly require that `T: 'a` -- -/// but this is not necessary, because the opaque type we -/// create will be allowed to reference `T`. So we only generate a -/// constraint that `'0: 'a`. -fn register_member_constraints<'tcx>( - typeck: &mut TypeChecker<'_, 'tcx>, - member_constraints: &mut MemberConstraintSet<'tcx, ty::RegionVid>, - opaque_type_key: OpaqueTypeKey<'tcx>, - OpaqueHiddenType { span, ty: hidden_ty }: OpaqueHiddenType<'tcx>, -) { - let tcx = typeck.tcx(); - let hidden_ty = typeck.infcx.resolve_vars_if_possible(hidden_ty); - debug!(?hidden_ty); - - let variances = tcx.variances_of(opaque_type_key.def_id); - debug!(?variances); - - // For a case like `impl Foo<'a, 'b>`, we would generate a constraint - // `'r in ['a, 'b, 'static]` for each region `'r` that appears in the - // hidden type (i.e., it must be equal to `'a`, `'b`, or `'static`). - // - // `conflict1` and `conflict2` are the two region bounds that we - // detected which were unrelated. They are used for diagnostics. - - // Create the set of choice regions: each region in the hidden - // type can be equal to any of the region parameters of the - // opaque type definition. - let fr_static = typeck.universal_regions.fr_static; - let choice_regions: Vec<_> = opaque_type_key - .args - .iter() - .enumerate() - .filter(|(i, _)| variances[*i] == ty::Invariant) - .filter_map(|(_, arg)| match arg.kind() { - GenericArgKind::Lifetime(r) => Some(typeck.to_region_vid(r)), - GenericArgKind::Type(_) | GenericArgKind::Const(_) => None, - }) - .chain(iter::once(fr_static)) - .collect(); - - // FIXME(#42940): This should use the `FreeRegionsVisitor`, but that's - // not currently sound until we have existential regions. - hidden_ty.visit_with(&mut ConstrainOpaqueTypeRegionVisitor { - tcx, - op: |r| { - member_constraints.add_member_constraint( - opaque_type_key, - hidden_ty, - span, - typeck.to_region_vid(r), - &choice_regions, - ) - }, - }); -} - -/// Visitor that requires that (almost) all regions in the type visited outlive -/// `least_region`. We cannot use `push_outlives_components` because regions in -/// closure signatures are not included in their outlives components. We need to -/// ensure all regions outlive the given bound so that we don't end up with, -/// say, `ReVar` appearing in a return type and causing ICEs when other -/// functions end up with region constraints involving regions from other -/// functions. -/// -/// We also cannot use `for_each_free_region` because for closures it includes -/// the regions parameters from the enclosing item. -/// -/// We ignore any type parameters because impl trait values are assumed to -/// capture all the in-scope type parameters. -struct ConstrainOpaqueTypeRegionVisitor<'tcx, OP: FnMut(ty::Region<'tcx>)> { - tcx: TyCtxt<'tcx>, - op: OP, -} - -impl<'tcx, OP> TypeVisitor> for ConstrainOpaqueTypeRegionVisitor<'tcx, OP> -where - OP: FnMut(ty::Region<'tcx>), -{ - fn visit_region(&mut self, r: ty::Region<'tcx>) { - match r.kind() { - // ignore bound regions, keep visiting - ty::ReBound(_, _) => {} - _ => (self.op)(r), - } - } - - fn visit_ty(&mut self, ty: Ty<'tcx>) { - // We're only interested in types involving regions - if !ty.flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS) { - return; - } - - match *ty.kind() { - ty::Closure(_, args) => { - // Skip lifetime parameters of the enclosing item(s) - - for upvar in args.as_closure().upvar_tys() { - upvar.visit_with(self); - } - args.as_closure().sig_as_fn_ptr_ty().visit_with(self); - } - - ty::CoroutineClosure(_, args) => { - // Skip lifetime parameters of the enclosing item(s) - - for upvar in args.as_coroutine_closure().upvar_tys() { - upvar.visit_with(self); - } - - args.as_coroutine_closure().signature_parts_ty().visit_with(self); - } - - ty::Coroutine(_, args) => { - // Skip lifetime parameters of the enclosing item(s) - // Also skip the witness type, because that has no free regions. - - for upvar in args.as_coroutine().upvar_tys() { - upvar.visit_with(self); - } - args.as_coroutine().return_ty().visit_with(self); - args.as_coroutine().yield_ty().visit_with(self); - args.as_coroutine().resume_ty().visit_with(self); - } - - ty::Alias(kind, ty::AliasTy { def_id, args, .. }) - if let Some(variances) = self.tcx.opt_alias_variances(kind, def_id) => - { - // Skip lifetime parameters that are not captured, since they do - // not need member constraints registered for them; we'll erase - // them (and hopefully in the future replace them with placeholders). - for (v, s) in std::iter::zip(variances, args.iter()) { - if *v != ty::Bivariant { - s.visit_with(self); - } - } - } - - _ => { - ty.super_visit_with(self); - } - } - } -} diff --git a/compiler/rustc_borrowck/src/type_check/relate_tys.rs b/compiler/rustc_borrowck/src/type_check/relate_tys.rs index e023300f1c28..7ac2dff12f75 100644 --- a/compiler/rustc_borrowck/src/type_check/relate_tys.rs +++ b/compiler/rustc_borrowck/src/type_check/relate_tys.rs @@ -124,8 +124,13 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> { // by using `ty_vid rel B` and then finally and end by equating `ty_vid` to // the opaque. let mut enable_subtyping = |ty, opaque_is_expected| { - let ty_vid = infcx.next_ty_var_id_in_universe(self.span(), ty::UniverseIndex::ROOT); - + // We create the fresh inference variable in the highest universe. + // In theory we could limit it to the highest universe in the args of + // the opaque but that isn't really worth the effort. + // + // We'll make sure that the opaque type can actually name everything + // in its hidden type later on. + let ty_vid = infcx.next_ty_vid(self.span()); let variance = if opaque_is_expected { self.ambient_variance } else { @@ -187,7 +192,7 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> { types: &mut |_bound_ty: ty::BoundTy| { unreachable!("we only replace regions in nll_relate, not types") }, - consts: &mut |_bound_var: ty::BoundVar| { + consts: &mut |_bound_const: ty::BoundConst| { unreachable!("we only replace regions in nll_relate, not consts") }, }; @@ -216,7 +221,7 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> { *ex_reg_var } else { let ex_reg_var = - self.next_existential_region_var(true, br.kind.get_name(infcx.infcx.tcx)); + self.next_existential_region_var(br.kind.get_name(infcx.infcx.tcx)); debug!(?ex_reg_var); reg_map.insert(br, ex_reg_var); @@ -226,7 +231,7 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> { types: &mut |_bound_ty: ty::BoundTy| { unreachable!("we only replace regions in nll_relate, not types") }, - consts: &mut |_bound_var: ty::BoundVar| { + consts: &mut |_bound_const: ty::BoundConst| { unreachable!("we only replace regions in nll_relate, not consts") }, }; @@ -244,17 +249,9 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> { } #[instrument(skip(self), level = "debug")] - fn next_existential_region_var( - &mut self, - from_forall: bool, - name: Option, - ) -> ty::Region<'tcx> { - let origin = NllRegionVariableOrigin::Existential { from_forall }; - - let reg_var = - self.type_checker.infcx.next_nll_region_var(origin, || RegionCtxt::Existential(name)); - - reg_var + fn next_existential_region_var(&mut self, name: Option) -> ty::Region<'tcx> { + let origin = NllRegionVariableOrigin::Existential { name }; + self.type_checker.infcx.next_nll_region_var(origin, || RegionCtxt::Existential(name)) } #[instrument(skip(self), level = "debug")] diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs index 240c9a5223be..64a7b4084349 100644 --- a/compiler/rustc_borrowck/src/universal_regions.rs +++ b/compiler/rustc_borrowck/src/universal_regions.rs @@ -40,6 +40,7 @@ use crate::BorrowckInferCtxt; use crate::renumber::RegionCtxt; #[derive(Debug)] +#[derive(Clone)] // FIXME(#146079) pub(crate) struct UniversalRegions<'tcx> { indices: UniversalRegionIndices<'tcx>, @@ -200,6 +201,7 @@ impl<'tcx> DefiningTy<'tcx> { } #[derive(Debug)] +#[derive(Clone)] // FIXME(#146079) struct UniversalRegionIndices<'tcx> { /// For those regions that may appear in the parameter environment /// ('static and early-bound regions), we maintain a map from the @@ -217,7 +219,7 @@ struct UniversalRegionIndices<'tcx> { /// Whether we've encountered an error region. If we have, cancel all /// outlives errors, as they are likely bogus. - pub tainted_by_errors: Cell>, + pub encountered_re_error: Cell>, } #[derive(Debug, PartialEq)] @@ -442,8 +444,8 @@ impl<'tcx> UniversalRegions<'tcx> { self.fr_fn_body } - pub(crate) fn tainted_by_errors(&self) -> Option { - self.indices.tainted_by_errors.get() + pub(crate) fn encountered_re_error(&self) -> Option { + self.indices.encountered_re_error.get() } } @@ -706,7 +708,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { UniversalRegionIndices { indices: global_mapping.chain(arg_mapping).collect(), fr_static, - tainted_by_errors: Cell::new(None), + encountered_re_error: Cell::new(None), } } @@ -916,7 +918,7 @@ impl<'tcx> UniversalRegionIndices<'tcx> { match r.kind() { ty::ReVar(..) => r.as_var(), ty::ReError(guar) => { - self.tainted_by_errors.set(Some(guar)); + self.encountered_re_error.set(Some(guar)); // We use the `'static` `RegionVid` because `ReError` doesn't actually exist in the // `UniversalRegionIndices`. This is fine because 1) it is a fallback only used if // errors are being emitted and 2) it leaves the happy path unaffected. diff --git a/compiler/rustc_builtin_macros/Cargo.toml b/compiler/rustc_builtin_macros/Cargo.toml index 4c1264c6f1ce..ce9a3ce3f248 100644 --- a/compiler/rustc_builtin_macros/Cargo.toml +++ b/compiler/rustc_builtin_macros/Cargo.toml @@ -10,7 +10,6 @@ doctest = false # tidy-alphabetical-start rustc_ast = { path = "../rustc_ast" } rustc_ast_pretty = { path = "../rustc_ast_pretty" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_attr_parsing = { path = "../rustc_attr_parsing" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } @@ -34,3 +33,8 @@ smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } thin-vec = "0.2.12" tracing = "0.1" # tidy-alphabetical-end + +[features] +# tidy-alphabetical-start +llvm_enzyme = [] +# tidy-alphabetical-end diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl index ae186d744c40..5e498bf98fb5 100644 --- a/compiler/rustc_builtin_macros/messages.ftl +++ b/compiler/rustc_builtin_macros/messages.ftl @@ -64,6 +64,11 @@ builtin_macros_autodiff_ty_activity = {$act} can not be used for this type builtin_macros_autodiff_unknown_activity = did not recognize Activity: `{$act}` builtin_macros_autodiff_width = autodiff width must fit u32, but is {$width} + +builtin_macros_avoid_att_syntax = avoid using `.att_syntax`, prefer using `options(att_syntax)` instead + +builtin_macros_avoid_intel_syntax = avoid using `.intel_syntax`, Intel syntax is the default + builtin_macros_bad_derive_target = `derive` may only be applied to `struct`s, `enum`s and `union`s .label = not applicable here .label2 = not a `struct`, `enum` or `union` @@ -130,6 +135,15 @@ builtin_macros_concat_missing_literal = expected a literal builtin_macros_default_arg = `#[default]` attribute does not accept a value .suggestion = try using `#[default]` +builtin_macros_derive_from_usage_note = `#[derive(From)]` can only be used on structs with exactly one field + +builtin_macros_derive_from_wrong_field_count = `#[derive(From)]` used on a struct with {$multiple_fields -> + [true] multiple fields + *[false] no fields +} + +builtin_macros_derive_from_wrong_target = `#[derive(From)]` used on {$kind} + builtin_macros_derive_macro_call = `derive` cannot be used on items with type macros builtin_macros_derive_path_args_list = traits in `#[derive(...)]` don't accept arguments @@ -138,6 +152,8 @@ builtin_macros_derive_path_args_list = traits in `#[derive(...)]` don't accept a builtin_macros_derive_path_args_value = traits in `#[derive(...)]` don't accept values .suggestion = remove the value +builtin_macros_duplicate_macro_attribute = duplicated attribute + builtin_macros_env_not_defined = environment variable `{$var}` not defined at compile time .cargo = Cargo sets build script variables at run time. Use `std::env::var({$var_expr})` instead .custom = use `std::env::var({$var_expr})` to read the variable at run time @@ -222,6 +238,8 @@ builtin_macros_format_unused_args = multiple unused formatting arguments builtin_macros_format_use_positional = consider using a positional formatting argument instead +builtin_macros_incomplete_include = include macro expected single expression in source + builtin_macros_multiple_default_attrs = multiple `#[default]` attributes .note = only one `#[default]` attribute is needed .label = `#[default]` used here @@ -259,15 +277,13 @@ builtin_macros_only_one_argument = {$name} takes 1 argument builtin_macros_proc_macro = `proc-macro` crate types currently cannot export any items other than functions tagged with `#[proc_macro]`, `#[proc_macro_derive]`, or `#[proc_macro_attribute]` -builtin_macros_proc_macro_attribute_only_be_used_on_bare_functions = the `#[{$path}]` attribute may only be used on bare functions - builtin_macros_proc_macro_attribute_only_usable_with_crate_type = the `#[{$path}]` attribute is only usable with crates of the `proc-macro` crate type builtin_macros_requires_cfg_pattern = macro requires a cfg-pattern as an argument .label = cfg-pattern required -builtin_macros_source_uitls_expected_item = expected item, found `{$token}` +builtin_macros_source_utils_expected_item = expected item, found `{$token}` builtin_macros_takes_no_arguments = {$name} takes no arguments @@ -287,3 +303,5 @@ builtin_macros_unexpected_lit = expected path to a trait, found literal .label = not a trait .str_lit = try using `#[derive({$sym})]` .other = for example, write `#[derive(Debug)]` for `Debug` + +builtin_macros_unnameable_test_items = cannot test inner items diff --git a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs index e75bc944d7ec..40946f3b2791 100644 --- a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs +++ b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs @@ -1,4 +1,4 @@ -use rustc_ast::ptr::P; +use rustc_ast::expand::allocator::{ALLOC_ERROR_HANDLER, global_fn_name}; use rustc_ast::{ self as ast, Fn, FnHeader, FnSig, Generics, ItemKind, Safety, Stmt, StmtKind, TyKind, }; @@ -46,7 +46,7 @@ pub(crate) fn expand( let const_body = ecx.expr_block(ecx.block(span, stmts)); let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body); let const_item = if is_stmt { - Annotatable::Stmt(P(ecx.stmt_item(span, const_item))) + Annotatable::Stmt(Box::new(ecx.stmt_item(span, const_item))) } else { Annotatable::Item(const_item) }; @@ -56,7 +56,7 @@ pub(crate) fn expand( } // #[rustc_std_internal_symbol] -// unsafe fn __rg_oom(size: usize, align: usize) -> ! { +// unsafe fn __rust_alloc_error_handler(size: usize, align: usize) -> ! { // handler(core::alloc::Layout::from_size_align_unchecked(size, align)) // } fn generate_handler(cx: &ExtCtxt<'_>, handler: Ident, span: Span, sig_span: Span) -> Stmt { @@ -85,7 +85,7 @@ fn generate_handler(cx: &ExtCtxt<'_>, handler: Ident, span: Span, sig_span: Span let kind = ItemKind::Fn(Box::new(Fn { defaultness: ast::Defaultness::Final, sig, - ident: Ident::from_str_and_span("__rg_oom", span), + ident: Ident::from_str_and_span(&global_fn_name(ALLOC_ERROR_HANDLER), span), generics: Generics::default(), contract: None, body, diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index 1fb998172226..971b68d14757 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -1,5 +1,3 @@ -use lint::BuiltinLintDiag; -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{AsmMacro, token}; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; @@ -19,7 +17,7 @@ use crate::{errors, fluent_generated as fluent}; /// Validated assembly arguments, ready for macro expansion. struct ValidatedAsmArgs { - pub templates: Vec>, + pub templates: Vec>, pub operands: Vec<(ast::InlineAsmOperand, Span)>, named_args: FxIndexMap, reg_args: GrowableBitSet, @@ -353,7 +351,7 @@ fn expand_preparsed_asm( lint::builtin::BAD_ASM_STYLE, find_span(".intel_syntax"), ecx.current_expansion.lint_node_id, - BuiltinLintDiag::AvoidUsingIntelSyntax, + errors::AvoidIntelSyntax, ); } if template_str.contains(".att_syntax") { @@ -361,7 +359,7 @@ fn expand_preparsed_asm( lint::builtin::BAD_ASM_STYLE, find_span(".att_syntax"), ecx.current_expansion.lint_node_id, - BuiltinLintDiag::AvoidUsingAttSyntax, + errors::AvoidAttSyntax, ); } } @@ -370,7 +368,7 @@ fn expand_preparsed_asm( if args.options.contains(ast::InlineAsmOptions::RAW) { template.push(ast::InlineAsmTemplatePiece::String(template_str.to_string().into())); let template_num_lines = 1 + template_str.matches('\n').count(); - line_spans.extend(std::iter::repeat(template_sp).take(template_num_lines)); + line_spans.extend(std::iter::repeat_n(template_sp, template_num_lines)); continue; } @@ -525,7 +523,7 @@ fn expand_preparsed_asm( if parser.line_spans.is_empty() { let template_num_lines = 1 + template_str.matches('\n').count(); - line_spans.extend(std::iter::repeat(template_sp).take(template_num_lines)); + line_spans.extend(std::iter::repeat_n(template_sp, template_num_lines)); } else { line_spans.extend( parser @@ -600,9 +598,9 @@ pub(super) fn expand_asm<'cx>( return ExpandResult::Retry(()); }; let expr = match mac { - Ok(inline_asm) => P(ast::Expr { + Ok(inline_asm) => Box::new(ast::Expr { id: ast::DUMMY_NODE_ID, - kind: ast::ExprKind::InlineAsm(P(inline_asm)), + kind: ast::ExprKind::InlineAsm(Box::new(inline_asm)), span: sp, attrs: ast::AttrVec::new(), tokens: None, @@ -630,9 +628,9 @@ pub(super) fn expand_naked_asm<'cx>( return ExpandResult::Retry(()); }; let expr = match mac { - Ok(inline_asm) => P(ast::Expr { + Ok(inline_asm) => Box::new(ast::Expr { id: ast::DUMMY_NODE_ID, - kind: ast::ExprKind::InlineAsm(P(inline_asm)), + kind: ast::ExprKind::InlineAsm(Box::new(inline_asm)), span: sp, attrs: ast::AttrVec::new(), tokens: None, @@ -660,7 +658,7 @@ pub(super) fn expand_global_asm<'cx>( return ExpandResult::Retry(()); }; match mac { - Ok(inline_asm) => MacEager::items(smallvec![P(ast::Item { + Ok(inline_asm) => MacEager::items(smallvec![Box::new(ast::Item { attrs: ast::AttrVec::new(), id: ast::DUMMY_NODE_ID, kind: ast::ItemKind::GlobalAsm(Box::new(inline_asm)), diff --git a/compiler/rustc_builtin_macros/src/assert.rs b/compiler/rustc_builtin_macros/src/assert.rs index c659b1cff59b..855da5caa312 100644 --- a/compiler/rustc_builtin_macros/src/assert.rs +++ b/compiler/rustc_builtin_macros/src/assert.rs @@ -1,6 +1,5 @@ mod context; -use rustc_ast::ptr::P; use rustc_ast::token::Delimiter; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, Path, PathSegment, UnOp, token}; @@ -55,9 +54,9 @@ pub(crate) fn expand_assert<'cx>( let expr = if let Some(tokens) = custom_message { let then = cx.expr( call_site_span, - ExprKind::MacCall(P(MacCall { + ExprKind::MacCall(Box::new(MacCall { path: panic_path(), - args: P(DelimArgs { + args: Box::new(DelimArgs { dspan: DelimSpan::from_single(call_site_span), delim: Delimiter::Parenthesis, tokens, @@ -96,7 +95,7 @@ pub(crate) fn expand_assert<'cx>( } struct Assert { - cond_expr: P, + cond_expr: Box, custom_message: Option, } @@ -104,10 +103,10 @@ struct Assert { fn expr_if_not( cx: &ExtCtxt<'_>, span: Span, - cond: P, - then: P, - els: Option>, -) -> P { + cond: Box, + then: Box, + els: Option>, +) -> Box { cx.expr_if(span, cx.expr(span, ExprKind::Unary(UnOp::Not, cond)), then, els) } diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs index ea7248ca5393..31855cbd4e6c 100644 --- a/compiler/rustc_builtin_macros/src/assert/context.rs +++ b/compiler/rustc_builtin_macros/src/assert/context.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::token::{self, Delimiter, IdentIsRaw}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_ast::{ @@ -70,7 +69,7 @@ impl<'cx, 'a> Context<'cx, 'a> { /// } /// } /// ``` - pub(super) fn build(mut self, mut cond_expr: P, panic_path: Path) -> P { + pub(super) fn build(mut self, mut cond_expr: Box, panic_path: Path) -> Box { let expr_str = pprust::expr_to_string(&cond_expr); self.manage_cond_expr(&mut cond_expr); let initial_imports = self.build_initial_imports(); @@ -129,7 +128,7 @@ impl<'cx, 'a> Context<'cx, 'a> { } /// Takes the conditional expression of `assert!` and then wraps it inside `unlikely` - fn build_unlikely(&self, cond_expr: P) -> P { + fn build_unlikely(&self, cond_expr: Box) -> Box { let unlikely_path = self.cx.std_path(&[sym::intrinsics, sym::unlikely]); self.cx.expr_call( self.span, @@ -145,7 +144,7 @@ impl<'cx, 'a> Context<'cx, 'a> { /// __capture0, /// ... /// ); - fn build_panic(&self, expr_str: &str, panic_path: Path) -> P { + fn build_panic(&self, expr_str: &str, panic_path: Path) -> Box { let escaped_expr_str = escape_to_fmt(expr_str); let initial = [ TokenTree::token_joint( @@ -176,9 +175,9 @@ impl<'cx, 'a> Context<'cx, 'a> { }); self.cx.expr( self.span, - ExprKind::MacCall(P(MacCall { + ExprKind::MacCall(Box::new(MacCall { path: panic_path, - args: P(DelimArgs { + args: Box::new(DelimArgs { dspan: DelimSpan::from_single(self.span), delim: Delimiter::Parenthesis, tokens: initial.into_iter().chain(captures).collect::(), @@ -190,7 +189,7 @@ impl<'cx, 'a> Context<'cx, 'a> { /// Recursive function called until `cond_expr` and `fmt_str` are fully modified. /// /// See [Self::manage_initial_capture] and [Self::manage_try_capture] - fn manage_cond_expr(&mut self, expr: &mut P) { + fn manage_cond_expr(&mut self, expr: &mut Box) { match &mut expr.kind { ExprKind::AddrOf(_, mutability, local_expr) => { self.with_is_consumed_management(matches!(mutability, Mutability::Mut), |this| { @@ -331,7 +330,7 @@ impl<'cx, 'a> Context<'cx, 'a> { /// /// `fmt_str`, the formatting string used for debugging, is constructed to show possible /// captured variables. - fn manage_initial_capture(&mut self, expr: &mut P, path_ident: Ident) { + fn manage_initial_capture(&mut self, expr: &mut Box, path_ident: Ident) { if self.paths.contains(&path_ident) { return; } else { @@ -360,7 +359,12 @@ impl<'cx, 'a> Context<'cx, 'a> { /// (&Wrapper(__local_bindN)).try_capture(&mut __captureN); /// __local_bindN /// } - fn manage_try_capture(&mut self, capture: Ident, curr_capture_idx: usize, expr: &mut P) { + fn manage_try_capture( + &mut self, + capture: Ident, + curr_capture_idx: usize, + expr: &mut Box, + ) { let local_bind_string = format!("__local_bind{curr_capture_idx}"); let local_bind = Ident::new(Symbol::intern(&local_bind_string), self.span); self.local_bind_decls.push(self.cx.stmt_let( @@ -441,20 +445,20 @@ fn escape_to_fmt(s: &str) -> String { rslt } -fn expr_addr_of_mut(cx: &ExtCtxt<'_>, sp: Span, e: P) -> P { +fn expr_addr_of_mut(cx: &ExtCtxt<'_>, sp: Span, e: Box) -> Box { cx.expr(sp, ExprKind::AddrOf(BorrowKind::Ref, Mutability::Mut, e)) } fn expr_method_call( cx: &ExtCtxt<'_>, seg: PathSegment, - receiver: P, - args: ThinVec>, + receiver: Box, + args: ThinVec>, span: Span, -) -> P { +) -> Box { cx.expr(span, ExprKind::MethodCall(Box::new(MethodCall { seg, receiver, args, span }))) } -fn expr_paren(cx: &ExtCtxt<'_>, sp: Span, e: P) -> P { +fn expr_paren(cx: &ExtCtxt<'_>, sp: Span, e: Box) -> Box { cx.expr(sp, ExprKind::Paren(e)) } diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index c78447783327..ddc59bfe1414 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -11,23 +11,23 @@ mod llvm_enzyme { AutoDiffAttrs, DiffActivity, DiffMode, valid_input_activity, valid_ret_activity, valid_ty_for_activity, }; - use rustc_ast::ptr::P; use rustc_ast::token::{Lit, LitKind, Token, TokenKind}; use rustc_ast::tokenstream::*; use rustc_ast::visit::AssocCtxt::*; use rustc_ast::{ - self as ast, AssocItemKind, BindingMode, ExprKind, FnRetTy, FnSig, Generics, ItemKind, - MetaItemInner, PatKind, QSelf, TyKind, Visibility, + self as ast, AngleBracketedArg, AngleBracketedArgs, AnonConst, AssocItemKind, BindingMode, + FnRetTy, FnSig, GenericArg, GenericArgs, GenericParamKind, Generics, ItemKind, + MetaItemInner, PatKind, Path, PathSegment, TyKind, Visibility, }; use rustc_expand::base::{Annotatable, ExtCtxt}; - use rustc_span::{Ident, Span, Symbol, kw, sym}; + use rustc_span::{Ident, Span, Symbol, sym}; use thin_vec::{ThinVec, thin_vec}; use tracing::{debug, trace}; use crate::errors; pub(crate) fn outer_normal_attr( - kind: &P, + kind: &Box, id: rustc_ast::AttrId, span: Span, ) -> rustc_ast::Attribute { @@ -73,7 +73,7 @@ mod llvm_enzyme { } // Get information about the function the macro is applied to - fn extract_item_info(iitem: &P) -> Option<(Visibility, FnSig, Ident, Generics)> { + fn extract_item_info(iitem: &Box) -> Option<(Visibility, FnSig, Ident, Generics)> { match &iitem.kind { ItemKind::Fn(box ast::Fn { sig, ident, generics, .. }) => { Some((iitem.vis.clone(), sig.clone(), ident.clone(), generics.clone())) @@ -180,11 +180,8 @@ mod llvm_enzyme { } /// We expand the autodiff macro to generate a new placeholder function which passes - /// type-checking and can be called by users. The function body of the placeholder function will - /// later be replaced on LLVM-IR level, so the design of the body is less important and for now - /// should just prevent early inlining and optimizations which alter the function signature. - /// The exact signature of the generated function depends on the configuration provided by the - /// user, but here is an example: + /// type-checking and can be called by users. The exact signature of the generated function + /// depends on the configuration provided by the user, but here is an example: /// /// ``` /// #[autodiff(cos_box, Reverse, Duplicated, Active)] @@ -195,19 +192,12 @@ mod llvm_enzyme { /// which becomes expanded to: /// ``` /// #[rustc_autodiff] - /// #[inline(never)] /// fn sin(x: &Box) -> f32 { /// f32::sin(**x) /// } /// #[rustc_autodiff(Reverse, Duplicated, Active)] - /// #[inline(never)] /// fn cos_box(x: &Box, dx: &mut Box, dret: f32) -> f32 { - /// unsafe { - /// asm!("NOP"); - /// }; - /// ::core::hint::black_box(sin(x)); - /// ::core::hint::black_box((dx, dret)); - /// ::core::hint::black_box(sin(x)) + /// std::intrinsics::autodiff(sin::<>, cos_box::<>, (x, dx, dret)) /// } /// ``` /// FIXME(ZuseZ4): Once autodiff is enabled by default, make this a doc comment which is checked @@ -219,7 +209,8 @@ mod llvm_enzyme { mut item: Annotatable, mode: DiffMode, ) -> Vec { - if cfg!(not(llvm_enzyme)) { + // FIXME(bjorn3) maybe have the backend directly tell if autodiff is supported? + if cfg!(not(feature = "llvm_enzyme")) { ecx.sess.dcx().emit_err(errors::AutoDiffSupportNotBuild { span: meta_item.span }); return vec![item]; } @@ -228,16 +219,24 @@ mod llvm_enzyme { // first get information about the annotable item: visibility, signature, name and generic // parameters. // these will be used to generate the differentiated version of the function - let Some((vis, sig, primal, generics)) = (match &item { - Annotatable::Item(iitem) => extract_item_info(iitem), + let Some((vis, sig, primal, generics, impl_of_trait)) = (match &item { + Annotatable::Item(iitem) => { + extract_item_info(iitem).map(|(v, s, p, g)| (v, s, p, g, false)) + } Annotatable::Stmt(stmt) => match &stmt.kind { - ast::StmtKind::Item(iitem) => extract_item_info(iitem), + ast::StmtKind::Item(iitem) => { + extract_item_info(iitem).map(|(v, s, p, g)| (v, s, p, g, false)) + } _ => None, }, - Annotatable::AssocItem(assoc_item, Impl { .. }) => match &assoc_item.kind { - ast::AssocItemKind::Fn(box ast::Fn { sig, ident, generics, .. }) => { - Some((assoc_item.vis.clone(), sig.clone(), ident.clone(), generics.clone())) - } + Annotatable::AssocItem(assoc_item, Impl { of_trait }) => match &assoc_item.kind { + ast::AssocItemKind::Fn(box ast::Fn { sig, ident, generics, .. }) => Some(( + assoc_item.vis.clone(), + sig.clone(), + ident.clone(), + generics.clone(), + *of_trait, + )), _ => None, }, _ => None, @@ -255,7 +254,6 @@ mod llvm_enzyme { }; let has_ret = has_ret(&sig.decl.output); - let sig_span = ecx.with_call_site_ctxt(sig.span); // create TokenStream from vec elemtents: // meta_item doesn't have a .tokens field @@ -324,19 +322,23 @@ mod llvm_enzyme { } let span = ecx.with_def_site_ctxt(expand_span); - let n_active: u32 = x - .input_activity - .iter() - .filter(|a| **a == DiffActivity::Active || **a == DiffActivity::ActiveOnly) - .count() as u32; - let (d_sig, new_args, idents, errored) = gen_enzyme_decl(ecx, &sig, &x, span); - let d_body = gen_enzyme_body( - ecx, &x, n_active, &sig, &d_sig, primal, &new_args, span, sig_span, idents, errored, - &generics, + let d_sig = gen_enzyme_decl(ecx, &sig, &x, span); + + let d_body = ecx.block( + span, + thin_vec![call_autodiff( + ecx, + primal, + first_ident(&meta_item_vec[0]), + span, + &d_sig, + &generics, + impl_of_trait, + )], ); // The first element of it is the name of the function to be generated - let asdf = Box::new(ast::Fn { + let d_fn = Box::new(ast::Fn { defaultness: ast::Defaultness::Final, sig: d_sig, ident: first_ident(&meta_item_vec[0]), @@ -346,7 +348,7 @@ mod llvm_enzyme { define_opaque: None, }); let mut rustc_ad_attr = - P(ast::NormalAttr::from_ident(Ident::with_dummy_span(sym::rustc_autodiff))); + Box::new(ast::NormalAttr::from_ident(Ident::with_dummy_span(sym::rustc_autodiff))); let ts2: Vec = vec![TokenTree::Token( Token::new(TokenKind::Ident(sym::never, false.into()), span), @@ -363,33 +365,34 @@ mod llvm_enzyme { args: ast::AttrArgs::Delimited(never_arg), tokens: None, }; - let inline_never_attr = P(ast::NormalAttr { item: inline_item, tokens: None }); + let inline_never_attr = Box::new(ast::NormalAttr { item: inline_item, tokens: None }); let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let attr = outer_normal_attr(&rustc_ad_attr, new_id, span); let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let inline_never = outer_normal_attr(&inline_never_attr, new_id, span); - // We're avoid duplicating the attributes `#[rustc_autodiff]` and `#[inline(never)]`. + // We're avoid duplicating the attribute `#[rustc_autodiff]`. fn same_attribute(attr: &ast::AttrKind, item: &ast::AttrKind) -> bool { match (attr, item) { (ast::AttrKind::Normal(a), ast::AttrKind::Normal(b)) => { let a = &a.item.path; let b = &b.item.path; - a.segments.len() == b.segments.len() - && a.segments.iter().zip(b.segments.iter()).all(|(a, b)| a.ident == b.ident) + a.segments.iter().eq_by(&b.segments, |a, b| a.ident == b.ident) } _ => false, } } + let mut has_inline_never = false; + // Don't add it multiple times: let orig_annotatable: Annotatable = match item { Annotatable::Item(ref mut iitem) => { if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { iitem.attrs.push(attr); } - if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { - iitem.attrs.push(inline_never.clone()); + if iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } Annotatable::Item(iitem.clone()) } @@ -397,8 +400,8 @@ mod llvm_enzyme { if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { assoc_item.attrs.push(attr); } - if !assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { - assoc_item.attrs.push(inline_never.clone()); + if assoc_item.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } Annotatable::AssocItem(assoc_item.clone(), i) } @@ -408,9 +411,8 @@ mod llvm_enzyme { if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &attr.kind)) { iitem.attrs.push(attr); } - if !iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) - { - iitem.attrs.push(inline_never.clone()); + if iitem.attrs.iter().any(|a| same_attribute(&a.kind, &inline_never.kind)) { + has_inline_never = true; } } _ => unreachable!("stmt kind checked previously"), @@ -429,12 +431,21 @@ mod llvm_enzyme { tokens: ts, }); + let new_id = ecx.sess.psess.attr_id_generator.mk_attr_id(); let d_attr = outer_normal_attr(&rustc_ad_attr, new_id, span); + + // If the source function has the `#[inline(never)]` attribute, we'll also add it to the diff function + let mut d_attrs = thin_vec![d_attr]; + + if has_inline_never { + d_attrs.push(inline_never); + } + let d_annotatable = match &item { Annotatable::AssocItem(_, _) => { - let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(asdf); - let d_fn = P(ast::AssocItem { - attrs: thin_vec![d_attr, inline_never], + let assoc_item: AssocItemKind = ast::AssocItemKind::Fn(d_fn); + let d_fn = Box::new(ast::AssocItem { + attrs: d_attrs, id: ast::DUMMY_NODE_ID, span, vis, @@ -444,16 +455,16 @@ mod llvm_enzyme { Annotatable::AssocItem(d_fn, Impl { of_trait: false }) } Annotatable::Item(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr, inline_never], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, d_attrs, ItemKind::Fn(d_fn)); d_fn.vis = vis; Annotatable::Item(d_fn) } Annotatable::Stmt(_) => { - let mut d_fn = ecx.item(span, thin_vec![d_attr, inline_never], ItemKind::Fn(asdf)); + let mut d_fn = ecx.item(span, d_attrs, ItemKind::Fn(d_fn)); d_fn.vis = vis; - Annotatable::Stmt(P(ast::Stmt { + Annotatable::Stmt(Box::new(ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(d_fn), span, @@ -485,282 +496,95 @@ mod llvm_enzyme { ty } - // Will generate a body of the type: + // Generate `autodiff` intrinsic call // ``` - // { - // unsafe { - // asm!("NOP"); - // } - // ::core::hint::black_box(primal(args)); - // ::core::hint::black_box((args, ret)); - // - // } + // std::intrinsics::autodiff(source, diff, (args)) // ``` - fn init_body_helper( + fn call_autodiff( ecx: &ExtCtxt<'_>, - span: Span, primal: Ident, - new_names: &[String], - sig_span: Span, - new_decl_span: Span, - idents: &[Ident], - errored: bool, + diff: Ident, + span: Span, + d_sig: &FnSig, generics: &Generics, - ) -> (P, P, P, P) { - let blackbox_path = ecx.std_path(&[sym::hint, sym::black_box]); - let noop = ast::InlineAsm { - asm_macro: ast::AsmMacro::Asm, - template: vec![ast::InlineAsmTemplatePiece::String("NOP".into())], - template_strs: Box::new([]), - operands: vec![], - clobber_abis: vec![], - options: ast::InlineAsmOptions::PURE | ast::InlineAsmOptions::NOMEM, - line_spans: vec![], - }; - let noop_expr = ecx.expr_asm(span, P(noop)); - let unsf = ast::BlockCheckMode::Unsafe(ast::UnsafeSource::CompilerGenerated); - let unsf_block = ast::Block { - stmts: thin_vec![ecx.stmt_semi(noop_expr)], - id: ast::DUMMY_NODE_ID, - tokens: None, - rules: unsf, + is_impl: bool, + ) -> rustc_ast::Stmt { + let primal_path_expr = gen_turbofish_expr(ecx, primal, generics, span, is_impl); + let diff_path_expr = gen_turbofish_expr(ecx, diff, generics, span, is_impl); + + let tuple_expr = ecx.expr_tuple( span, - }; - let unsf_expr = ecx.expr_block(P(unsf_block)); - let blackbox_call_expr = ecx.expr_path(ecx.path(span, blackbox_path)); - let primal_call = gen_primal_call(ecx, span, primal, idents, generics); - let black_box_primal_call = ecx.expr_call( - new_decl_span, - blackbox_call_expr.clone(), - thin_vec![primal_call.clone()], + d_sig + .decl + .inputs + .iter() + .map(|arg| match arg.pat.kind { + PatKind::Ident(_, ident, _) => ecx.expr_path(ecx.path_ident(span, ident)), + _ => todo!(), + }) + .collect::>() + .into(), ); - let tup_args = new_names + + let enzyme_path_idents = ecx.std_path(&[sym::intrinsics, sym::autodiff]); + let enzyme_path = ecx.path(span, enzyme_path_idents); + let call_expr = ecx.expr_call( + span, + ecx.expr_path(enzyme_path), + vec![primal_path_expr, diff_path_expr, tuple_expr].into(), + ); + + ecx.stmt_expr(call_expr) + } + + // Generate turbofish expression from fn name and generics + // Given `foo` and `` params, gen `foo::` + // We use this expression when passing primal and diff function to the autodiff intrinsic + fn gen_turbofish_expr( + ecx: &ExtCtxt<'_>, + ident: Ident, + generics: &Generics, + span: Span, + is_impl: bool, + ) -> Box { + let generic_args = generics + .params .iter() - .map(|arg| ecx.expr_path(ecx.path_ident(span, Ident::from_str(arg)))) - .collect(); - - let black_box_remaining_args = ecx.expr_call( - sig_span, - blackbox_call_expr.clone(), - thin_vec![ecx.expr_tuple(sig_span, tup_args)], - ); - - let mut body = ecx.block(span, ThinVec::new()); - body.stmts.push(ecx.stmt_semi(unsf_expr)); - - // This uses primal args which won't be available if we errored before - if !errored { - body.stmts.push(ecx.stmt_semi(black_box_primal_call.clone())); - } - body.stmts.push(ecx.stmt_semi(black_box_remaining_args)); - - (body, primal_call, black_box_primal_call, blackbox_call_expr) - } - - /// We only want this function to type-check, since we will replace the body - /// later on llvm level. Using `loop {}` does not cover all return types anymore, - /// so instead we manually build something that should pass the type checker. - /// We also add a inline_asm line, as one more barrier for rustc to prevent inlining - /// or const propagation. inline_asm will also triggers an Enzyme crash if due to another - /// bug would ever try to accidentally differentiate this placeholder function body. - /// Finally, we also add back_box usages of all input arguments, to prevent rustc - /// from optimizing any arguments away. - fn gen_enzyme_body( - ecx: &ExtCtxt<'_>, - x: &AutoDiffAttrs, - n_active: u32, - sig: &ast::FnSig, - d_sig: &ast::FnSig, - primal: Ident, - new_names: &[String], - span: Span, - sig_span: Span, - idents: Vec, - errored: bool, - generics: &Generics, - ) -> P { - let new_decl_span = d_sig.span; - - // Just adding some default inline-asm and black_box usages to prevent early inlining - // and optimizations which alter the function signature. - // - // The bb_primal_call is the black_box call of the primal function. We keep it around, - // since it has the convenient property of returning the type of the primal function, - // Remember, we only care to match types here. - // No matter which return we pick, we always wrap it into a std::hint::black_box call, - // to prevent rustc from propagating it into the caller. - let (mut body, primal_call, bb_primal_call, bb_call_expr) = init_body_helper( - ecx, - span, - primal, - new_names, - sig_span, - new_decl_span, - &idents, - errored, - generics, - ); - - if !has_ret(&d_sig.decl.output) { - // there is no return type that we have to match, () works fine. - return body; - } - - // Everything from here onwards just tries to fulfil the return type. Fun! - - // having an active-only return means we'll drop the original return type. - // So that can be treated identical to not having one in the first place. - let primal_ret = has_ret(&sig.decl.output) && !x.has_active_only_ret(); - - if primal_ret && n_active == 0 && x.mode.is_rev() { - // We only have the primal ret. - body.stmts.push(ecx.stmt_expr(bb_primal_call)); - return body; - } - - if !primal_ret && n_active == 1 { - // Again no tuple return, so return default float val. - let ty = match d_sig.decl.output { - FnRetTy::Ty(ref ty) => ty.clone(), - FnRetTy::Default(span) => { - panic!("Did not expect Default ret ty: {:?}", span); + .filter_map(|p| match &p.kind { + GenericParamKind::Type { .. } => { + let path = ast::Path::from_ident(p.ident); + let ty = ecx.ty_path(path); + Some(AngleBracketedArg::Arg(GenericArg::Type(ty))) } - }; - let arg = ty.kind.is_simple_path().unwrap(); - let tmp = ecx.def_site_path(&[arg, kw::Default]); - let default_call_expr = ecx.expr_path(ecx.path(span, tmp)); - let default_call_expr = ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - body.stmts.push(ecx.stmt_expr(default_call_expr)); - return body; - } + GenericParamKind::Const { .. } => { + let expr = ecx.expr_path(ast::Path::from_ident(p.ident)); + let anon_const = AnonConst { id: ast::DUMMY_NODE_ID, value: expr }; + Some(AngleBracketedArg::Arg(GenericArg::Const(anon_const))) + } + GenericParamKind::Lifetime { .. } => None, + }) + .collect::>(); - let mut exprs: P = primal_call; - let d_ret_ty = match d_sig.decl.output { - FnRetTy::Ty(ref ty) => ty.clone(), - FnRetTy::Default(span) => { - panic!("Did not expect Default ret ty: {:?}", span); - } + let args: AngleBracketedArgs = AngleBracketedArgs { span, args: generic_args }; + + let segment = PathSegment { + ident, + id: ast::DUMMY_NODE_ID, + args: Some(Box::new(GenericArgs::AngleBracketed(args))), }; - if x.mode.is_fwd() { - // Fwd mode is easy. If the return activity is Const, we support arbitrary types. - // Otherwise, we only support a scalar, a pair of scalars, or an array of scalars. - // We checked that (on a best-effort base) in the preceding gen_enzyme_decl function. - // In all three cases, we can return `std::hint::black_box(::default())`. - if x.ret_activity == DiffActivity::Const { - // Here we call the primal function, since our dummy function has the same return - // type due to the Const return activity. - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]); - } else { - let q = QSelf { ty: d_ret_ty, path_span: span, position: 0 }; - let y = ExprKind::Path( - Some(P(q)), - ecx.path_ident(span, Ident::with_dummy_span(kw::Default)), - ); - let default_call_expr = ecx.expr(span, y); - let default_call_expr = - ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![default_call_expr]); - } - } else if x.mode.is_rev() { - if x.width == 1 { - // We either have `-> ArbitraryType` or `-> (ArbitraryType, repeated_float_scalars)`. - match d_ret_ty.kind { - TyKind::Tup(ref args) => { - // We have a tuple return type. We need to create a tuple of the same size - // and fill it with default values. - let mut exprs2 = thin_vec![exprs]; - for arg in args.iter().skip(1) { - let arg = arg.kind.is_simple_path().unwrap(); - let tmp = ecx.def_site_path(&[arg, kw::Default]); - let default_call_expr = ecx.expr_path(ecx.path(span, tmp)); - let default_call_expr = - ecx.expr_call(new_decl_span, default_call_expr, thin_vec![]); - exprs2.push(default_call_expr); - } - exprs = ecx.expr_tuple(new_decl_span, exprs2); - } - _ => { - // Interestingly, even the `-> ArbitraryType` case - // ends up getting matched and handled correctly above, - // so we don't have to handle any other case for now. - panic!("Unsupported return type: {:?}", d_ret_ty); - } - } - } - exprs = ecx.expr_call(new_decl_span, bb_call_expr, thin_vec![exprs]); + + let segments = if is_impl { + thin_vec![ + PathSegment { ident: Ident::from_str("Self"), id: ast::DUMMY_NODE_ID, args: None }, + segment, + ] } else { - unreachable!("Unsupported mode: {:?}", x.mode); - } + thin_vec![segment] + }; - body.stmts.push(ecx.stmt_expr(exprs)); + let path = Path { span, segments, tokens: None }; - body - } - - fn gen_primal_call( - ecx: &ExtCtxt<'_>, - span: Span, - primal: Ident, - idents: &[Ident], - generics: &Generics, - ) -> P { - let has_self = idents.len() > 0 && idents[0].name == kw::SelfLower; - - if has_self { - let args: ThinVec<_> = - idents[1..].iter().map(|arg| ecx.expr_path(ecx.path_ident(span, *arg))).collect(); - let self_expr = ecx.expr_self(span); - ecx.expr_method_call(span, self_expr, primal, args) - } else { - let args: ThinVec<_> = - idents.iter().map(|arg| ecx.expr_path(ecx.path_ident(span, *arg))).collect(); - let mut primal_path = ecx.path_ident(span, primal); - - let is_generic = !generics.params.is_empty(); - - match (is_generic, primal_path.segments.last_mut()) { - (true, Some(function_path)) => { - let primal_generic_types = generics - .params - .iter() - .filter(|param| matches!(param.kind, ast::GenericParamKind::Type { .. })); - - let generated_generic_types = primal_generic_types - .map(|type_param| { - let generic_param = TyKind::Path( - None, - ast::Path { - span, - segments: thin_vec![ast::PathSegment { - ident: type_param.ident, - args: None, - id: ast::DUMMY_NODE_ID, - }], - tokens: None, - }, - ); - - ast::AngleBracketedArg::Arg(ast::GenericArg::Type(P(ast::Ty { - id: type_param.id, - span, - kind: generic_param, - tokens: None, - }))) - }) - .collect(); - - function_path.args = - Some(P(ast::GenericArgs::AngleBracketed(ast::AngleBracketedArgs { - span, - args: generated_generic_types, - }))); - } - _ => {} - } - - let primal_call_expr = ecx.expr_path(primal_path); - ecx.expr_call(span, primal_call_expr, args) - } + ecx.expr_path(path) } // Generate the new function declaration. Const arguments are kept as is. Duplicated arguments must @@ -779,7 +603,7 @@ mod llvm_enzyme { sig: &ast::FnSig, x: &AutoDiffAttrs, span: Span, - ) -> (ast::FnSig, Vec, Vec, bool) { + ) -> ast::FnSig { let dcx = ecx.sess.dcx(); let has_ret = has_ret(&sig.decl.output); let sig_args = sig.decl.inputs.len() + if has_ret { 1 } else { 0 }; @@ -791,7 +615,7 @@ mod llvm_enzyme { found: num_activities, }); // This is not the right signature, but we can continue parsing. - return (sig.clone(), vec![], vec![], true); + return sig.clone(); } assert!(sig.decl.inputs.len() == x.input_activity.len()); assert!(has_ret == x.has_ret_activity()); @@ -834,7 +658,7 @@ mod llvm_enzyme { if errors { // This is not the right signature, but we can continue parsing. - return (sig.clone(), new_inputs, idents, true); + return sig.clone(); } let unsafe_activities = x @@ -856,7 +680,7 @@ mod llvm_enzyme { for i in 0..x.width { let mut shadow_arg = arg.clone(); // We += into the shadow in reverse mode. - shadow_arg.ty = P(assure_mut_ref(&arg.ty)); + shadow_arg.ty = Box::new(assure_mut_ref(&arg.ty)); let old_name = if let PatKind::Ident(_, ident, _) = arg.pat.kind { ident.name } else { @@ -866,7 +690,7 @@ mod llvm_enzyme { let name: String = format!("d{}_{}", old_name, i); new_inputs.push(name.clone()); let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span); - shadow_arg.pat = P(ast::Pat { + shadow_arg.pat = Box::new(ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::NONE, ident, None), span: shadow_arg.pat.span, @@ -898,7 +722,7 @@ mod llvm_enzyme { let name: String = format!("b{}_{}", old_name, i); new_inputs.push(name.clone()); let ident = Ident::from_str_and_span(&name, shadow_arg.pat.span); - shadow_arg.pat = P(ast::Pat { + shadow_arg.pat = Box::new(ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::NONE, ident, None), span: shadow_arg.pat.span, @@ -942,7 +766,7 @@ mod llvm_enzyme { let shadow_arg = ast::Param { attrs: ThinVec::new(), ty: ty.clone(), - pat: P(ast::Pat { + pat: Box::new(ast::Pat { id: ast::DUMMY_NODE_ID, kind: PatKind::Ident(BindingMode::NONE, ident, None), span: ty.span, @@ -966,7 +790,12 @@ mod llvm_enzyme { FnRetTy::Default(span) => { // We want to return std::hint::black_box(()). let kind = TyKind::Tup(ThinVec::new()); - let ty = P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None }); + let ty = Box::new(rustc_ast::Ty { + kind, + id: ast::DUMMY_NODE_ID, + span, + tokens: None, + }); d_decl.output = FnRetTy::Ty(ty.clone()); assert!(matches!(x.ret_activity, DiffActivity::None)); // this won't be used below, so any type would be fine. @@ -987,7 +816,7 @@ mod llvm_enzyme { }; TyKind::Array(ty.clone(), anon_const) }; - let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }); + let ty = Box::new(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }); d_decl.output = FnRetTy::Ty(ty); } if matches!(x.ret_activity, DiffActivity::DualOnly | DiffActivity::DualvOnly) { @@ -1000,7 +829,8 @@ mod llvm_enzyme { value: ecx.expr_usize(span, x.width as usize), }; let kind = TyKind::Array(ty.clone(), anon_const); - let ty = P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }); + let ty = + Box::new(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }); d_decl.output = FnRetTy::Ty(ty); } } @@ -1022,14 +852,14 @@ mod llvm_enzyme { act_ret.insert(0, ty.clone()); } let kind = TyKind::Tup(act_ret); - P(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }) + Box::new(rustc_ast::Ty { kind, id: ty.id, span: ty.span, tokens: None }) } FnRetTy::Default(span) => { if act_ret.len() == 1 { act_ret[0].clone() } else { let kind = TyKind::Tup(act_ret.iter().map(|arg| arg.clone()).collect()); - P(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None }) + Box::new(rustc_ast::Ty { kind, id: ast::DUMMY_NODE_ID, span, tokens: None }) } } }; @@ -1042,7 +872,7 @@ mod llvm_enzyme { } let d_sig = FnSig { header: d_header, decl: d_decl, span }; trace!("Generated signature: {:?}", d_sig); - (d_sig, new_inputs, idents, false) + d_sig } } diff --git a/compiler/rustc_builtin_macros/src/cfg_accessible.rs b/compiler/rustc_builtin_macros/src/cfg_accessible.rs index 5f203dd5d113..48d80004cdd6 100644 --- a/compiler/rustc_builtin_macros/src/cfg_accessible.rs +++ b/compiler/rustc_builtin_macros/src/cfg_accessible.rs @@ -1,9 +1,9 @@ //! Implementation of the `#[cfg_accessible(path)]` attribute macro. use rustc_ast as ast; +use rustc_attr_parsing::validate_attr; use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier}; use rustc_feature::AttributeTemplate; -use rustc_parse::validate_attr; use rustc_span::{Span, sym}; use crate::errors; @@ -44,7 +44,7 @@ impl MultiItemModifier for Expander { item: Annotatable, _is_derive_const: bool, ) -> ExpandResult, Annotatable> { - let template = AttributeTemplate { list: Some("path"), ..Default::default() }; + let template = AttributeTemplate { list: Some(&["path"]), ..Default::default() }; validate_attr::check_builtin_meta_item( &ecx.sess.psess, meta_item, diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index ec3b87467a9b..dd770fe5f1a1 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -2,7 +2,6 @@ use core::ops::ControlFlow; use rustc_ast as ast; use rustc_ast::mut_visit::MutVisitor; -use rustc_ast::ptr::P; use rustc_ast::visit::{AssocCtxt, Visitor}; use rustc_ast::{Attribute, HasAttrs, HasTokens, NodeId, mut_visit, visit}; use rustc_errors::PResult; @@ -132,7 +131,7 @@ impl CfgEval<'_> { let stmt = parser .parse_stmt_without_recovery(false, ForceCollect::Yes, false)? .unwrap(); - Annotatable::Stmt(P(self.flat_map_stmt(stmt).pop().unwrap())) + Annotatable::Stmt(Box::new(self.flat_map_stmt(stmt).pop().unwrap())) } Annotatable::Expr(_) => { let mut expr = parser.parse_expr_force_collect()?; @@ -166,7 +165,7 @@ impl MutVisitor for CfgEval<'_> { mut_visit::walk_expr(self, expr); } - fn filter_map_expr(&mut self, expr: P) -> Option> { + fn filter_map_expr(&mut self, expr: Box) -> Option> { let mut expr = configure!(self, expr); mut_visit::walk_expr(self, &mut expr); Some(expr) @@ -185,24 +184,24 @@ impl MutVisitor for CfgEval<'_> { mut_visit::walk_flat_map_stmt(self, stmt) } - fn flat_map_item(&mut self, item: P) -> SmallVec<[P; 1]> { + fn flat_map_item(&mut self, item: Box) -> SmallVec<[Box; 1]> { let item = configure!(self, item); mut_visit::walk_flat_map_item(self, item) } fn flat_map_assoc_item( &mut self, - item: P, + item: Box, ctxt: AssocCtxt, - ) -> SmallVec<[P; 1]> { + ) -> SmallVec<[Box; 1]> { let item = configure!(self, item); mut_visit::walk_flat_map_assoc_item(self, item, ctxt) } fn flat_map_foreign_item( &mut self, - foreign_item: P, - ) -> SmallVec<[P; 1]> { + foreign_item: Box, + ) -> SmallVec<[Box; 1]> { let foreign_item = configure!(self, foreign_item); mut_visit::walk_flat_map_foreign_item(self, foreign_item) } diff --git a/compiler/rustc_builtin_macros/src/concat_bytes.rs b/compiler/rustc_builtin_macros/src/concat_bytes.rs index fd2d740c0203..8885017b930d 100644 --- a/compiler/rustc_builtin_macros/src/concat_bytes.rs +++ b/compiler/rustc_builtin_macros/src/concat_bytes.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ExprKind, LitIntType, LitKind, StrStyle, UintTy, token}; use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacEager, MacroExpanderResult}; @@ -90,7 +89,7 @@ fn handle_array_element( cx: &ExtCtxt<'_>, guar: &mut Option, missing_literals: &mut Vec, - expr: &P, + expr: &Box, ) -> Option { let dcx = cx.dcx(); diff --git a/compiler/rustc_builtin_macros/src/contracts.rs b/compiler/rustc_builtin_macros/src/contracts.rs index 6a24af361fe7..52ddfbaa99f9 100644 --- a/compiler/rustc_builtin_macros/src/contracts.rs +++ b/compiler/rustc_builtin_macros/src/contracts.rs @@ -17,7 +17,7 @@ impl AttrProcMacro for ExpandRequires { annotation: TokenStream, annotated: TokenStream, ) -> Result { - expand_requires_tts(ecx, span, annotation, annotated) + expand_contract_clause_tts(ecx, span, annotation, annotated, kw::ContractRequires) } } @@ -29,7 +29,7 @@ impl AttrProcMacro for ExpandEnsures { annotation: TokenStream, annotated: TokenStream, ) -> Result { - expand_ensures_tts(ecx, span, annotation, annotated) + expand_contract_clause_tts(ecx, span, annotation, annotated, kw::ContractEnsures) } } @@ -71,6 +71,14 @@ fn expand_contract_clause( .span_err(attr_span, "contract annotations can only be used on functions")); } + // Contracts are not yet supported on async/gen functions + if new_tts.iter().any(|tt| is_kw(tt, kw::Async) || is_kw(tt, kw::Gen)) { + return Err(ecx.sess.dcx().span_err( + attr_span, + "contract annotations are not yet supported on async or gen functions", + )); + } + // Found the `fn` keyword, now find either the `where` token or the function body. let next_tt = loop { let Some(tt) = cursor.next() else { @@ -122,48 +130,23 @@ fn expand_contract_clause( Ok(new_tts) } -fn expand_requires_tts( +fn expand_contract_clause_tts( ecx: &mut ExtCtxt<'_>, attr_span: Span, annotation: TokenStream, annotated: TokenStream, + clause_keyword: rustc_span::Symbol, ) -> Result { let feature_span = ecx.with_def_site_ctxt(attr_span); expand_contract_clause(ecx, attr_span, annotated, |new_tts| { new_tts.push_tree(TokenTree::Token( - token::Token::from_ast_ident(Ident::new(kw::ContractRequires, feature_span)), - Spacing::Joint, - )); - new_tts.push_tree(TokenTree::Token( - token::Token::new(token::TokenKind::OrOr, attr_span), - Spacing::Alone, - )); - new_tts.push_tree(TokenTree::Delimited( - DelimSpan::from_single(attr_span), - DelimSpacing::new(Spacing::JointHidden, Spacing::JointHidden), - token::Delimiter::Parenthesis, - annotation, - )); - Ok(()) - }) -} - -fn expand_ensures_tts( - ecx: &mut ExtCtxt<'_>, - attr_span: Span, - annotation: TokenStream, - annotated: TokenStream, -) -> Result { - let feature_span = ecx.with_def_site_ctxt(attr_span); - expand_contract_clause(ecx, attr_span, annotated, |new_tts| { - new_tts.push_tree(TokenTree::Token( - token::Token::from_ast_ident(Ident::new(kw::ContractEnsures, feature_span)), + token::Token::from_ast_ident(Ident::new(clause_keyword, feature_span)), Spacing::Joint, )); new_tts.push_tree(TokenTree::Delimited( DelimSpan::from_single(attr_span), DelimSpacing::new(Spacing::JointHidden, Spacing::JointHidden), - token::Delimiter::Parenthesis, + token::Delimiter::Brace, annotation, )); Ok(()) diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs index e259f5b3955b..09d827b0635e 100644 --- a/compiler/rustc_builtin_macros/src/derive.rs +++ b/compiler/rustc_builtin_macros/src/derive.rs @@ -1,10 +1,10 @@ use rustc_ast as ast; use rustc_ast::{GenericParamKind, ItemKind, MetaItemInner, MetaItemKind, StmtKind}; +use rustc_attr_parsing::validate_attr; use rustc_expand::base::{ Annotatable, DeriveResolution, ExpandResult, ExtCtxt, Indeterminate, MultiItemModifier, }; use rustc_feature::AttributeTemplate; -use rustc_parse::validate_attr; use rustc_session::Session; use rustc_span::{ErrorGuaranteed, Ident, Span, sym}; @@ -34,8 +34,10 @@ impl MultiItemModifier for Expander { let (sess, features) = (ecx.sess, ecx.ecfg.features); let result = ecx.resolver.resolve_derives(ecx.current_expansion.id, ecx.force_mode, &|| { - let template = - AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() }; + let template = AttributeTemplate { + list: Some(&["Trait1, Trait2, ..."]), + ..Default::default() + }; validate_attr::check_builtin_meta_item( &sess.psess, meta_item, diff --git a/compiler/rustc_builtin_macros/src/deriving/bounds.rs b/compiler/rustc_builtin_macros/src/deriving/bounds.rs index dd8f0e46a0e0..63342880b094 100644 --- a/compiler/rustc_builtin_macros/src/deriving/bounds.rs +++ b/compiler/rustc_builtin_macros/src/deriving/bounds.rs @@ -51,43 +51,4 @@ pub(crate) fn expand_deriving_const_param_ty( }; trait_def.expand(cx, mitem, item, push); - - let trait_def = TraitDef { - span, - path: path_std!(marker::UnsizedConstParamTy), - skip_path_as_bound: false, - needs_copy_as_bound_if_packed: false, - additional_bounds: vec![ty::Ty::Path(path_std!(cmp::Eq))], - supports_unions: false, - methods: Vec::new(), - associated_types: Vec::new(), - is_const, - is_staged_api_crate: cx.ecfg.features.staged_api(), - }; - - trait_def.expand(cx, mitem, item, push); -} - -pub(crate) fn expand_deriving_unsized_const_param_ty( - cx: &ExtCtxt<'_>, - span: Span, - mitem: &MetaItem, - item: &Annotatable, - push: &mut dyn FnMut(Annotatable), - is_const: bool, -) { - let trait_def = TraitDef { - span, - path: path_std!(marker::UnsizedConstParamTy), - skip_path_as_bound: false, - needs_copy_as_bound_if_packed: false, - additional_bounds: vec![ty::Ty::Path(path_std!(cmp::Eq))], - supports_unions: false, - methods: Vec::new(), - associated_types: Vec::new(), - is_const, - is_staged_api_crate: cx.ecfg.features.staged_api(), - }; - - trait_def.expand(cx, mitem, item, push); } diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs index 990835fa2773..a0f71a1868b5 100644 --- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs +++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::{BinOpKind, BorrowKind, Expr, ExprKind, MetaItem, Mutability}; use rustc_expand::base::{Annotatable, ExtCtxt}; use rustc_span::{Span, sym}; @@ -119,7 +118,7 @@ fn get_substructure_equality_expr( cx: &ExtCtxt<'_>, span: Span, substructure: &Substructure<'_>, -) -> P { +) -> Box { use SubstructureFields::*; match substructure.fields { @@ -180,7 +179,7 @@ fn get_substructure_equality_expr( /// /// Panics if there are not exactly two arguments to compare (should be `self` /// and `other`). -fn get_field_equality_expr(cx: &ExtCtxt<'_>, field: &FieldInfo) -> P { +fn get_field_equality_expr(cx: &ExtCtxt<'_>, field: &FieldInfo) -> Box { let [rhs] = &field.other_selflike_exprs[..] else { cx.dcx().span_bug(field.span, "not exactly 2 arguments in `derive(PartialEq)`"); }; @@ -198,7 +197,7 @@ fn get_field_equality_expr(cx: &ExtCtxt<'_>, field: &FieldInfo) -> P { /// This is used to strip away any number of leading `&` from an expression /// (e.g., `&&&T` becomes `T`). Only removes immutable references; mutable /// references are preserved. -fn peel_refs(mut expr: &P) -> P { +fn peel_refs(mut expr: &Box) -> Box { while let ExprKind::AddrOf(BorrowKind::Ref, Mutability::Not, inner) = &expr.kind { expr = &inner; } @@ -210,7 +209,7 @@ fn peel_refs(mut expr: &P) -> P { /// /// If the given expression is a block, it is wrapped in parentheses; otherwise, /// it is returned unchanged. -fn wrap_block_expr(cx: &ExtCtxt<'_>, expr: P) -> P { +fn wrap_block_expr(cx: &ExtCtxt<'_>, expr: Box) -> Box { if matches!(&expr.kind, ExprKind::Block(..)) { return cx.expr_paren(expr.span, expr); } diff --git a/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs b/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs index 6082e376435a..5b378de8bbdd 100644 --- a/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs +++ b/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs @@ -108,11 +108,7 @@ pub(crate) fn expand_deriving_coerce_pointee( cx.item( span, attrs.clone(), - ast::ItemKind::Impl(Box::new(ast::Impl { - safety: ast::Safety::Default, - polarity: ast::ImplPolarity::Positive, - defaultness: ast::Defaultness::Final, - constness: ast::Const::No, + ast::ItemKind::Impl(ast::Impl { generics: Generics { params: generics .params @@ -137,10 +133,16 @@ pub(crate) fn expand_deriving_coerce_pointee( where_clause: generics.where_clause.clone(), span: generics.span, }, - of_trait: Some(trait_ref), + of_trait: Some(Box::new(ast::TraitImplHeader { + safety: ast::Safety::Default, + polarity: ast::ImplPolarity::Positive, + defaultness: ast::Defaultness::Final, + constness: ast::Const::No, + trait_ref, + })), self_ty: self_type.clone(), items: ThinVec::new(), - })), + }), ), )); } @@ -152,16 +154,18 @@ pub(crate) fn expand_deriving_coerce_pointee( let item = cx.item( span, attrs.clone(), - ast::ItemKind::Impl(Box::new(ast::Impl { - safety: ast::Safety::Default, - polarity: ast::ImplPolarity::Positive, - defaultness: ast::Defaultness::Final, - constness: ast::Const::No, + ast::ItemKind::Impl(ast::Impl { generics, - of_trait: Some(trait_ref), + of_trait: Some(Box::new(ast::TraitImplHeader { + safety: ast::Safety::Default, + polarity: ast::ImplPolarity::Positive, + defaultness: ast::Defaultness::Final, + constness: ast::Const::No, + trait_ref, + })), self_ty: self_type.clone(), items: ThinVec::new(), - })), + }), ); push(Annotatable::Item(item)); }; @@ -352,21 +356,14 @@ fn contains_maybe_sized_bound(bounds: &[GenericBound]) -> bool { bounds.iter().any(is_maybe_sized_bound) } -fn path_segment_is_exact_match(path_segments: &[ast::PathSegment], syms: &[Symbol]) -> bool { - path_segments.iter().zip(syms).all(|(segment, &symbol)| segment.ident.name == symbol) -} - fn is_sized_marker(path: &ast::Path) -> bool { const CORE_UNSIZE: [Symbol; 3] = [sym::core, sym::marker, sym::Sized]; const STD_UNSIZE: [Symbol; 3] = [sym::std, sym::marker, sym::Sized]; - if path.segments.len() == 4 && path.is_global() { - path_segment_is_exact_match(&path.segments[1..], &CORE_UNSIZE) - || path_segment_is_exact_match(&path.segments[1..], &STD_UNSIZE) - } else if path.segments.len() == 3 { - path_segment_is_exact_match(&path.segments, &CORE_UNSIZE) - || path_segment_is_exact_match(&path.segments, &STD_UNSIZE) + let segments = || path.segments.iter().map(|segment| segment.ident.name); + if path.is_global() { + segments().skip(1).eq(CORE_UNSIZE) || segments().skip(1).eq(STD_UNSIZE) } else { - *path == sym::Sized + segments().eq(CORE_UNSIZE) || segments().eq(STD_UNSIZE) || *path == sym::Sized } } diff --git a/compiler/rustc_builtin_macros/src/deriving/debug.rs b/compiler/rustc_builtin_macros/src/deriving/debug.rs index 1d63ce7d5fd2..597af0e09c06 100644 --- a/compiler/rustc_builtin_macros/src/deriving/debug.rs +++ b/compiler/rustc_builtin_macros/src/deriving/debug.rs @@ -94,7 +94,7 @@ fn show_substructure(cx: &ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> field: &FieldInfo, index: usize, len: usize, - ) -> ast::ptr::P { + ) -> Box { if index < len - 1 { field.self_expr.clone() } else { diff --git a/compiler/rustc_builtin_macros/src/deriving/default.rs b/compiler/rustc_builtin_macros/src/deriving/default.rs index b4e2d27fed33..2462114ec24a 100644 --- a/compiler/rustc_builtin_macros/src/deriving/default.rs +++ b/compiler/rustc_builtin_macros/src/deriving/default.rs @@ -56,7 +56,7 @@ pub(crate) fn expand_deriving_default( trait_def.expand(cx, mitem, item, push) } -fn default_call(cx: &ExtCtxt<'_>, span: Span) -> ast::ptr::P { +fn default_call(cx: &ExtCtxt<'_>, span: Span) -> Box { // Note that `kw::Default` is "default" and `sym::Default` is "Default"! let default_ident = cx.std_path(&[kw::Default, sym::Default, kw::Default]); cx.expr_call_global(span, default_ident, ThinVec::new()) diff --git a/compiler/rustc_builtin_macros/src/deriving/from.rs b/compiler/rustc_builtin_macros/src/deriving/from.rs new file mode 100644 index 000000000000..ab25de7c9175 --- /dev/null +++ b/compiler/rustc_builtin_macros/src/deriving/from.rs @@ -0,0 +1,133 @@ +use rustc_ast as ast; +use rustc_ast::{ItemKind, VariantData}; +use rustc_errors::MultiSpan; +use rustc_expand::base::{Annotatable, DummyResult, ExtCtxt}; +use rustc_span::{Ident, Span, kw, sym}; +use thin_vec::thin_vec; + +use crate::deriving::generic::ty::{Bounds, Path, PathKind, Ty}; +use crate::deriving::generic::{ + BlockOrExpr, FieldlessVariantsStrategy, MethodDef, SubstructureFields, TraitDef, + combine_substructure, +}; +use crate::deriving::pathvec_std; +use crate::errors; + +/// Generate an implementation of the `From` trait, provided that `item` +/// is a struct or a tuple struct with exactly one field. +pub(crate) fn expand_deriving_from( + cx: &ExtCtxt<'_>, + span: Span, + mitem: &ast::MetaItem, + annotatable: &Annotatable, + push: &mut dyn FnMut(Annotatable), + is_const: bool, +) { + let Annotatable::Item(item) = &annotatable else { + cx.dcx().bug("derive(From) used on something else than an item"); + }; + + let err_span = || { + let item_span = item.kind.ident().map(|ident| ident.span).unwrap_or(item.span); + MultiSpan::from_spans(vec![span, item_span]) + }; + + // `#[derive(From)]` is currently usable only on structs with exactly one field. + let field = match &item.kind { + ItemKind::Struct(_, _, data) => { + if let [field] = data.fields() { + Ok(field.clone()) + } else { + let guar = cx.dcx().emit_err(errors::DeriveFromWrongFieldCount { + span: err_span(), + multiple_fields: data.fields().len() > 1, + }); + Err(guar) + } + } + ItemKind::Enum(_, _, _) | ItemKind::Union(_, _, _) => { + let guar = cx.dcx().emit_err(errors::DeriveFromWrongTarget { + span: err_span(), + kind: &format!("{} {}", item.kind.article(), item.kind.descr()), + }); + Err(guar) + } + _ => cx.dcx().bug("Invalid derive(From) ADT input"), + }; + + let from_type = Ty::AstTy(match field { + Ok(ref field) => field.ty.clone(), + Err(guar) => cx.ty(span, ast::TyKind::Err(guar)), + }); + + let path = + Path::new_(pathvec_std!(convert::From), vec![Box::new(from_type.clone())], PathKind::Std); + + // Generate code like this: + // + // struct S(u32); + // #[automatically_derived] + // impl ::core::convert::From for S { + // #[inline] + // fn from(value: u32) -> S { + // Self(value) + // } + // } + let from_trait_def = TraitDef { + span, + path, + skip_path_as_bound: true, + needs_copy_as_bound_if_packed: false, + additional_bounds: Vec::new(), + supports_unions: false, + methods: vec![MethodDef { + name: sym::from, + generics: Bounds { bounds: vec![] }, + explicit_self: false, + nonself_args: vec![(from_type, sym::value)], + ret_ty: Ty::Self_, + attributes: thin_vec![cx.attr_word(sym::inline, span)], + fieldless_variants_strategy: FieldlessVariantsStrategy::Default, + combine_substructure: combine_substructure(Box::new(|cx, span, substructure| { + let field = match field { + Ok(ref field) => field, + Err(guar) => { + return BlockOrExpr::new_expr(DummyResult::raw_expr(span, Some(guar))); + } + }; + + let self_kw = Ident::new(kw::SelfUpper, span); + let expr: Box = match substructure.fields { + SubstructureFields::StaticStruct(variant, _) => match variant { + // Self { field: value } + VariantData::Struct { .. } => cx.expr_struct_ident( + span, + self_kw, + thin_vec![cx.field_imm( + span, + field.ident.unwrap(), + cx.expr_ident(span, Ident::new(sym::value, span)) + )], + ), + // Self(value) + VariantData::Tuple(_, _) => cx.expr_call_ident( + span, + self_kw, + thin_vec![cx.expr_ident(span, Ident::new(sym::value, span))], + ), + variant => { + cx.dcx().bug(format!("Invalid derive(From) ADT variant: {variant:?}")); + } + }, + _ => cx.dcx().bug("Invalid derive(From) ADT input"), + }; + BlockOrExpr::new_expr(expr) + })), + }], + associated_types: Vec::new(), + is_const, + is_staged_api_crate: cx.ecfg.features.staged_api(), + }; + + from_trait_def.expand(cx, mitem, annotatable, push); +} diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs index b24e55637613..24a71ae94389 100644 --- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs +++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs @@ -180,17 +180,16 @@ use std::{iter, vec}; pub(crate) use StaticFields::*; pub(crate) use SubstructureFields::*; -use rustc_ast::ptr::P; use rustc_ast::token::{IdentIsRaw, LitKind, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenTree}; use rustc_ast::{ self as ast, AnonConst, AttrArgs, BindingMode, ByRef, DelimArgs, EnumDef, Expr, GenericArg, GenericParamKind, Generics, Mutability, PatKind, Safety, VariantData, }; -use rustc_attr_data_structures::{AttributeKind, ReprPacked}; use rustc_attr_parsing::AttributeParser; use rustc_expand::base::{Annotatable, ExtCtxt}; use rustc_hir::Attribute; +use rustc_hir::attrs::{AttributeKind, ReprPacked}; use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym}; use thin_vec::{ThinVec, thin_vec}; use ty::{Bounds, Path, Ref, Self_, Ty}; @@ -272,7 +271,7 @@ pub(crate) struct Substructure<'a> { pub type_ident: Ident, /// Verbatim access to any non-selflike arguments, i.e. arguments that /// don't have type `&Self`. - pub nonselflike_args: &'a [P], + pub nonselflike_args: &'a [Box], pub fields: &'a SubstructureFields<'a>, } @@ -284,10 +283,10 @@ pub(crate) struct FieldInfo { pub name: Option, /// The expression corresponding to this field of `self` /// (specifically, a reference to it). - pub self_expr: P, + pub self_expr: Box, /// The expressions corresponding to references to this field in /// the other selflike arguments. - pub other_selflike_exprs: Vec>, + pub other_selflike_exprs: Vec>, pub maybe_scalar: bool, } @@ -323,7 +322,7 @@ pub(crate) enum SubstructureFields<'a> { /// The discriminant of an enum. The first field is a `FieldInfo` for the discriminants, as /// if they were fields. The second field is the expression to combine the /// discriminant expression with; it will be `None` if no match is necessary. - EnumDiscr(FieldInfo, Option>), + EnumDiscr(FieldInfo, Option>), /// A static method where `Self` is a struct. StaticStruct(&'a ast::VariantData, StaticFields), @@ -345,7 +344,7 @@ pub(crate) fn combine_substructure( struct TypeParameter { bound_generic_params: ThinVec, - ty: P, + ty: Box, } /// The code snippets built up for derived code are sometimes used as blocks @@ -354,23 +353,23 @@ struct TypeParameter { /// avoiding the insertion of any unnecessary blocks. /// /// The statements come before the expression. -pub(crate) struct BlockOrExpr(ThinVec, Option>); +pub(crate) struct BlockOrExpr(ThinVec, Option>); impl BlockOrExpr { pub(crate) fn new_stmts(stmts: ThinVec) -> BlockOrExpr { BlockOrExpr(stmts, None) } - pub(crate) fn new_expr(expr: P) -> BlockOrExpr { + pub(crate) fn new_expr(expr: Box) -> BlockOrExpr { BlockOrExpr(ThinVec::new(), Some(expr)) } - pub(crate) fn new_mixed(stmts: ThinVec, expr: Option>) -> BlockOrExpr { + pub(crate) fn new_mixed(stmts: ThinVec, expr: Option>) -> BlockOrExpr { BlockOrExpr(stmts, expr) } // Converts it into a block. - fn into_block(mut self, cx: &ExtCtxt<'_>, span: Span) -> P { + fn into_block(mut self, cx: &ExtCtxt<'_>, span: Span) -> Box { if let Some(expr) = self.1 { self.0.push(cx.stmt_expr(expr)); } @@ -378,7 +377,7 @@ impl BlockOrExpr { } // Converts it into an expression. - fn into_expr(self, cx: &ExtCtxt<'_>, span: Span) -> P { + fn into_expr(self, cx: &ExtCtxt<'_>, span: Span) -> Box { if self.0.is_empty() { match self.1 { None => cx.expr_block(cx.block(span, ThinVec::new())), @@ -432,7 +431,7 @@ fn find_type_parameters( { self.type_params.push(TypeParameter { bound_generic_params: self.bound_generic_params_stack.clone(), - ty: P(ty.clone()), + ty: Box::new(ty.clone()), }); } @@ -544,7 +543,7 @@ impl<'a> TraitDef<'a> { }) .cloned(), ); - push(Annotatable::Item(P(ast::Item { attrs, ..(*newitem).clone() }))) + push(Annotatable::Item(Box::new(ast::Item { attrs, ..(*newitem).clone() }))) } _ => unreachable!(), } @@ -590,15 +589,15 @@ impl<'a> TraitDef<'a> { cx: &ExtCtxt<'_>, type_ident: Ident, generics: &Generics, - field_tys: Vec>, - methods: Vec>, + field_tys: Vec>, + methods: Vec>, is_packed: bool, - ) -> P { + ) -> Box { let trait_path = self.path.to_path(cx, self.span, type_ident, generics); // Transform associated types from `deriving::ty::Ty` into `ast::AssocItem` let associated_types = self.associated_types.iter().map(|&(ident, ref type_def)| { - P(ast::AssocItem { + Box::new(ast::AssocItem { id: ast::DUMMY_NODE_ID, span: self.span, vis: ast::Visibility { @@ -611,7 +610,7 @@ impl<'a> TraitDef<'a> { defaultness: ast::Defaultness::Final, ident, generics: Generics::default(), - where_clauses: ast::TyAliasWhereClauses::default(), + after_where_clause: ast::WhereClause::default(), bounds: Vec::new(), ty: Some(type_def.to_ty(cx, self.span, type_ident, generics)), })), @@ -827,21 +826,25 @@ impl<'a> TraitDef<'a> { ) } - let opt_trait_ref = Some(trait_ref); - cx.item( self.span, attrs, - ast::ItemKind::Impl(Box::new(ast::Impl { - safety: ast::Safety::Default, - polarity: ast::ImplPolarity::Positive, - defaultness: ast::Defaultness::Final, - constness: if self.is_const { ast::Const::Yes(DUMMY_SP) } else { ast::Const::No }, + ast::ItemKind::Impl(ast::Impl { generics: trait_generics, - of_trait: opt_trait_ref, + of_trait: Some(Box::new(ast::TraitImplHeader { + safety: ast::Safety::Default, + polarity: ast::ImplPolarity::Positive, + defaultness: ast::Defaultness::Final, + constness: if self.is_const { + ast::Const::Yes(DUMMY_SP) + } else { + ast::Const::No + }, + trait_ref, + })), self_ty: self_type, items: methods.into_iter().chain(associated_types).collect(), - })), + }), ) } @@ -853,8 +856,8 @@ impl<'a> TraitDef<'a> { generics: &Generics, from_scratch: bool, is_packed: bool, - ) -> P { - let field_tys: Vec> = + ) -> Box { + let field_tys: Vec> = struct_def.fields().iter().map(|field| field.ty.clone()).collect(); let methods = self @@ -906,7 +909,7 @@ impl<'a> TraitDef<'a> { type_ident: Ident, generics: &Generics, from_scratch: bool, - ) -> P { + ) -> Box { let mut field_tys = Vec::new(); for variant in &enum_def.variants { @@ -962,7 +965,7 @@ impl<'a> MethodDef<'a> { cx: &ExtCtxt<'_>, trait_: &TraitDef<'_>, type_ident: Ident, - nonselflike_args: &[P], + nonselflike_args: &[Box], fields: &SubstructureFields<'_>, ) -> BlockOrExpr { let span = trait_.span; @@ -978,7 +981,7 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'_>, generics: &Generics, type_ident: Ident, - ) -> P { + ) -> Box { self.ret_ty.to_ty(cx, trait_.span, type_ident, generics) } @@ -999,7 +1002,8 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'_>, type_ident: Ident, generics: &Generics, - ) -> (Option, ThinVec>, Vec>, Vec<(Ident, P)>) { + ) -> (Option, ThinVec>, Vec>, Vec<(Ident, Box)>) + { let mut selflike_args = ThinVec::new(); let mut nonselflike_args = Vec::new(); let mut nonself_arg_tys = Vec::new(); @@ -1036,9 +1040,9 @@ impl<'a> MethodDef<'a> { type_ident: Ident, generics: &Generics, explicit_self: Option, - nonself_arg_tys: Vec<(Ident, P)>, + nonself_arg_tys: Vec<(Ident, Box)>, body: BlockOrExpr, - ) -> P { + ) -> Box { let span = trait_.span; // Create the generics that aren't for `Self`. let fn_generics = self.generics.to_generics(cx, span, type_ident, generics); @@ -1065,7 +1069,7 @@ impl<'a> MethodDef<'a> { let defaultness = ast::Defaultness::Final; // Create the method. - P(ast::AssocItem { + Box::new(ast::AssocItem { id: ast::DUMMY_NODE_ID, attrs: self.attributes.clone(), span, @@ -1128,8 +1132,8 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'b>, struct_def: &'b VariantData, type_ident: Ident, - selflike_args: &[P], - nonselflike_args: &[P], + selflike_args: &[Box], + nonselflike_args: &[Box], is_packed: bool, ) -> BlockOrExpr { assert!(selflike_args.len() == 1 || selflike_args.len() == 2); @@ -1151,7 +1155,7 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'_>, struct_def: &VariantData, type_ident: Ident, - nonselflike_args: &[P], + nonselflike_args: &[Box], ) -> BlockOrExpr { let summary = trait_.summarise_struct(cx, struct_def); @@ -1205,8 +1209,8 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'b>, enum_def: &'b EnumDef, type_ident: Ident, - mut selflike_args: ThinVec>, - nonselflike_args: &[P], + mut selflike_args: ThinVec>, + nonselflike_args: &[Box], ) -> BlockOrExpr { assert!( !selflike_args.is_empty(), @@ -1418,7 +1422,7 @@ impl<'a> MethodDef<'a> { // ... // _ => ::core::intrinsics::unreachable(), // } - let get_match_expr = |mut selflike_args: ThinVec>| { + let get_match_expr = |mut selflike_args: ThinVec>| { let match_arg = if selflike_args.len() == 1 { selflike_args.pop().unwrap() } else { @@ -1454,7 +1458,7 @@ impl<'a> MethodDef<'a> { trait_: &TraitDef<'_>, enum_def: &EnumDef, type_ident: Ident, - nonselflike_args: &[P], + nonselflike_args: &[Box], ) -> BlockOrExpr { self.call_substructure_method( cx, @@ -1503,7 +1507,7 @@ impl<'a> TraitDef<'a> { struct_def: &'a VariantData, prefixes: &[String], by_ref: ByRef, - ) -> ThinVec> { + ) -> ThinVec { prefixes .iter() .map(|prefix| { @@ -1539,7 +1543,7 @@ impl<'a> TraitDef<'a> { attrs: ast::AttrVec::new(), id: ast::DUMMY_NODE_ID, span: pat.span.with_ctxt(self.span.ctxt()), - pat, + pat: Box::new(pat), is_placeholder: false, } }) @@ -1558,7 +1562,7 @@ impl<'a> TraitDef<'a> { fn create_fields(&self, struct_def: &'a VariantData, mk_exprs: F) -> Vec where - F: Fn(usize, &ast::FieldDef, Span) -> Vec>, + F: Fn(usize, &ast::FieldDef, Span) -> Vec>, { struct_def .fields() @@ -1606,7 +1610,7 @@ impl<'a> TraitDef<'a> { fn create_struct_field_access_fields( &self, cx: &ExtCtxt<'_>, - selflike_args: &[P], + selflike_args: &[Box], struct_def: &'a VariantData, is_packed: bool, ) -> Vec { @@ -1651,7 +1655,7 @@ pub(crate) enum CsFold<'a> { /// The combination of two field expressions. E.g. for `PartialEq::eq` this /// is something like ` && `. - Combine(Span, P, P), + Combine(Span, Box, Box), // The fallback case for a struct or enum variant with no fields. Fieldless, @@ -1665,9 +1669,9 @@ pub(crate) fn cs_fold( trait_span: Span, substructure: &Substructure<'_>, mut f: F, -) -> P +) -> Box where - F: FnMut(&ExtCtxt<'_>, CsFold<'_>) -> P, + F: FnMut(&ExtCtxt<'_>, CsFold<'_>) -> Box, { match substructure.fields { EnumMatching(.., all_fields) | Struct(_, all_fields) => { diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs index f34a6ae1d982..1458553d4925 100644 --- a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs +++ b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs @@ -2,8 +2,7 @@ //! when specifying impls to be derived. pub(crate) use Ty::*; -use rustc_ast::ptr::P; -use rustc_ast::{self as ast, Expr, GenericArg, GenericParamKind, Generics, SelfKind}; +use rustc_ast::{self as ast, Expr, GenericArg, GenericParamKind, Generics, SelfKind, TyKind}; use rustc_expand::base::ExtCtxt; use rustc_span::source_map::respan; use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw}; @@ -41,7 +40,7 @@ impl Path { span: Span, self_ty: Ident, self_generics: &Generics, - ) -> P { + ) -> Box { cx.ty_path(self.to_path(cx, span, self_ty, self_generics)) } pub(crate) fn to_path( @@ -66,7 +65,7 @@ impl Path { } } -/// A type. Supports pointers, Self, and literals. +/// A type. Supports pointers, Self, literals, unit or an arbitrary AST path. #[derive(Clone)] pub(crate) enum Ty { Self_, @@ -77,6 +76,8 @@ pub(crate) enum Ty { Path(Path), /// For () return types. Unit, + /// An arbitrary type. + AstTy(Box), } pub(crate) fn self_ref() -> Ty { @@ -90,7 +91,7 @@ impl Ty { span: Span, self_ty: Ident, self_generics: &Generics, - ) -> P { + ) -> Box { match self { Ref(ty, mutbl) => { let raw_ty = ty.to_ty(cx, span, self_ty, self_generics); @@ -102,6 +103,7 @@ impl Ty { let ty = ast::TyKind::Tup(ThinVec::new()); cx.ty(span, ty) } + AstTy(ty) => ty.clone(), } } @@ -133,6 +135,10 @@ impl Ty { cx.path_all(span, false, vec![self_ty], params) } Path(p) => p.to_path(cx, span, self_ty, generics), + AstTy(ty) => match &ty.kind { + TyKind::Path(_, path) => path.clone(), + _ => cx.dcx().span_bug(span, "non-path in a path in generic `derive`"), + }, Ref(..) => cx.dcx().span_bug(span, "ref in a path in generic `derive`"), Unit => cx.dcx().span_bug(span, "unit in a path in generic `derive`"), } @@ -192,7 +198,7 @@ impl Bounds { } } -pub(crate) fn get_explicit_self(cx: &ExtCtxt<'_>, span: Span) -> (P, ast::ExplicitSelf) { +pub(crate) fn get_explicit_self(cx: &ExtCtxt<'_>, span: Span) -> (Box, ast::ExplicitSelf) { // This constructs a fresh `self` path. let self_path = cx.expr_self(span); let self_ty = respan(span, SelfKind::Region(None, ast::Mutability::Not)); diff --git a/compiler/rustc_builtin_macros/src/deriving/mod.rs b/compiler/rustc_builtin_macros/src/deriving/mod.rs index e45d09b57960..cee6952fa346 100644 --- a/compiler/rustc_builtin_macros/src/deriving/mod.rs +++ b/compiler/rustc_builtin_macros/src/deriving/mod.rs @@ -1,7 +1,6 @@ //! The compiler code necessary to implement the `#[derive]` extensions. use rustc_ast as ast; -use rustc_ast::ptr::P; use rustc_ast::{GenericArg, MetaItem}; use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt, MultiItemModifier}; use rustc_span::{Span, Symbol, sym}; @@ -24,6 +23,7 @@ pub(crate) mod clone; pub(crate) mod coerce_pointee; pub(crate) mod debug; pub(crate) mod default; +pub(crate) mod from; pub(crate) mod hash; #[path = "cmp/eq.rs"] @@ -66,7 +66,7 @@ impl MultiItemModifier for BuiltinDerive { &mut |a| { // Cannot use 'ecx.stmt_item' here, because we need to pass 'ecx' // to the function - items.push(Annotatable::Stmt(P(ast::Stmt { + items.push(Annotatable::Stmt(Box::new(ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Item(a.expect_item()), span, @@ -91,20 +91,20 @@ fn call_intrinsic( cx: &ExtCtxt<'_>, span: Span, intrinsic: Symbol, - args: ThinVec>, -) -> P { + args: ThinVec>, +) -> Box { let span = cx.with_def_site_ctxt(span); let path = cx.std_path(&[sym::intrinsics, intrinsic]); cx.expr_call_global(span, path, args) } /// Constructs an expression that calls the `unreachable` intrinsic. -fn call_unreachable(cx: &ExtCtxt<'_>, span: Span) -> P { +fn call_unreachable(cx: &ExtCtxt<'_>, span: Span) -> Box { let span = cx.with_def_site_ctxt(span); let path = cx.std_path(&[sym::intrinsics, sym::unreachable]); let call = cx.expr_call_global(span, path, ThinVec::new()); - cx.expr_block(P(ast::Block { + cx.expr_block(Box::new(ast::Block { stmts: thin_vec![cx.stmt_expr(call)], id: ast::DUMMY_NODE_ID, rules: ast::BlockCheckMode::Unsafe(ast::CompilerGenerated), @@ -116,7 +116,7 @@ fn call_unreachable(cx: &ExtCtxt<'_>, span: Span) -> P { fn assert_ty_bounds( cx: &ExtCtxt<'_>, stmts: &mut ThinVec, - ty: P, + ty: Box, span: Span, assert_path: &[Symbol], ) { diff --git a/compiler/rustc_builtin_macros/src/edition_panic.rs b/compiler/rustc_builtin_macros/src/edition_panic.rs index ccfcc3079ebf..08f555b9e52f 100644 --- a/compiler/rustc_builtin_macros/src/edition_panic.rs +++ b/compiler/rustc_builtin_macros/src/edition_panic.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::token::Delimiter; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::*; @@ -48,7 +47,7 @@ fn expand<'cx>( ExpandResult::Ready(MacEager::expr( cx.expr( sp, - ExprKind::MacCall(P(MacCall { + ExprKind::MacCall(Box::new(MacCall { path: Path { span: sp, segments: cx @@ -58,7 +57,7 @@ fn expand<'cx>( .collect(), tokens: None, }, - args: P(DelimArgs { + args: Box::new(DelimArgs { dspan: DelimSpan::from_single(sp), delim: Delimiter::Parenthesis, tokens: tts, diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs index 6bcf4d3e0a2e..d6ffbb5a4101 100644 --- a/compiler/rustc_builtin_macros/src/errors.rs +++ b/compiler/rustc_builtin_macros/src/errors.rs @@ -3,9 +3,29 @@ use rustc_errors::{ Diag, DiagCtxtHandle, Diagnostic, EmissionGuarantee, Level, MultiSpan, SingleLabelManySpans, Subdiagnostic, }; -use rustc_macros::{Diagnostic, Subdiagnostic}; +use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_span::{Ident, Span, Symbol}; +#[derive(LintDiagnostic)] +#[diag(builtin_macros_avoid_intel_syntax)] +pub(crate) struct AvoidIntelSyntax; + +#[derive(LintDiagnostic)] +#[diag(builtin_macros_avoid_att_syntax)] +pub(crate) struct AvoidAttSyntax; + +#[derive(LintDiagnostic)] +#[diag(builtin_macros_incomplete_include)] +pub(crate) struct IncompleteInclude; + +#[derive(LintDiagnostic)] +#[diag(builtin_macros_unnameable_test_items)] +pub(crate) struct UnnameableTestItems; + +#[derive(LintDiagnostic)] +#[diag(builtin_macros_duplicate_macro_attribute)] +pub(crate) struct DuplicateMacroAttribute; + #[derive(Diagnostic)] #[diag(builtin_macros_requires_cfg_pattern)] pub(crate) struct RequiresCfgPattern { @@ -446,6 +466,24 @@ pub(crate) struct DefaultHasArg { pub(crate) span: Span, } +#[derive(Diagnostic)] +#[diag(builtin_macros_derive_from_wrong_target)] +#[note(builtin_macros_derive_from_usage_note)] +pub(crate) struct DeriveFromWrongTarget<'a> { + #[primary_span] + pub(crate) span: MultiSpan, + pub(crate) kind: &'a str, +} + +#[derive(Diagnostic)] +#[diag(builtin_macros_derive_from_wrong_field_count)] +#[note(builtin_macros_derive_from_usage_note)] +pub(crate) struct DeriveFromWrongFieldCount { + #[primary_span] + pub(crate) span: MultiSpan, + pub(crate) multiple_fields: bool, +} + #[derive(Diagnostic)] #[diag(builtin_macros_derive_macro_call)] pub(crate) struct DeriveMacroCall { @@ -905,14 +943,6 @@ pub(crate) struct TakesNoArguments<'a> { pub name: &'a str, } -#[derive(Diagnostic)] -#[diag(builtin_macros_proc_macro_attribute_only_be_used_on_bare_functions)] -pub(crate) struct AttributeOnlyBeUsedOnBareFunctions<'a> { - #[primary_span] - pub span: Span, - pub path: &'a str, -} - #[derive(Diagnostic)] #[diag(builtin_macros_proc_macro_attribute_only_usable_with_crate_type)] pub(crate) struct AttributeOnlyUsableWithCrateType<'a> { @@ -922,7 +952,7 @@ pub(crate) struct AttributeOnlyUsableWithCrateType<'a> { } #[derive(Diagnostic)] -#[diag(builtin_macros_source_uitls_expected_item)] +#[diag(builtin_macros_source_utils_expected_item)] pub(crate) struct ExpectedItem<'a> { #[primary_span] pub span: Span, diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs index 6785cb6aef5a..a0ee7ac19899 100644 --- a/compiler/rustc_builtin_macros/src/format.rs +++ b/compiler/rustc_builtin_macros/src/format.rs @@ -1,7 +1,6 @@ use std::ops::Range; use parse::Position::ArgumentNamed; -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ Expr, ExprKind, FormatAlignment, FormatArgPosition, FormatArgPositionKind, FormatArgs, @@ -11,11 +10,12 @@ use rustc_ast::{ }; use rustc_data_structures::fx::FxHashSet; use rustc_errors::{ - Applicability, Diag, MultiSpan, PResult, SingleLabelManySpans, listify, pluralize, + Applicability, BufferedEarlyLint, Diag, MultiSpan, PResult, SingleLabelManySpans, listify, + pluralize, }; use rustc_expand::base::*; use rustc_lint_defs::builtin::NAMED_ARGUMENTS_USED_POSITIONALLY; -use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, LintId}; +use rustc_lint_defs::{BuiltinLintDiag, LintId}; use rustc_parse::exp; use rustc_parse_format as parse; use rustc_span::{BytePos, ErrorGuaranteed, Ident, InnerSpan, Span, Symbol}; @@ -45,7 +45,7 @@ use PositionUsedAs::*; #[derive(Debug)] struct MacroInput { - fmtstr: P, + fmtstr: Box, args: FormatArguments, /// Whether the first argument was a string literal or a result from eager macro expansion. /// If it's not a string literal, we disallow implicit argument capturing. @@ -69,35 +69,26 @@ struct MacroInput { /// Ok((fmtstr, parsed arguments)) /// ``` fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, MacroInput> { - let mut args = FormatArguments::new(); - let mut p = ecx.new_parser_from_tts(tts); - if p.token == token::Eof { - return Err(ecx.dcx().create_err(errors::FormatRequiresString { span: sp })); - } - - let first_token = &p.token; - - let fmtstr = if let token::Literal(lit) = first_token.kind - && matches!(lit.kind, token::Str | token::StrRaw(_)) - { + // parse the format string + let fmtstr = match p.token.kind { + token::Eof => return Err(ecx.dcx().create_err(errors::FormatRequiresString { span: sp })), // This allows us to properly handle cases when the first comma // after the format string is mistakenly replaced with any operator, // which cause the expression parser to eat too much tokens. - p.parse_literal_maybe_minus()? - } else { + token::Literal(token::Lit { kind: token::Str | token::StrRaw(_), .. }) => { + p.parse_literal_maybe_minus()? + } // Otherwise, we fall back to the expression parser. - p.parse_expr()? + _ => p.parse_expr()?, }; - // Only allow implicit captures to be used when the argument is a direct literal - // instead of a macro expanding to one. - let is_direct_literal = matches!(fmtstr.kind, ExprKind::Lit(_)); - + // parse comma FormatArgument pairs + let mut args = FormatArguments::new(); let mut first = true; - while p.token != token::Eof { + // parse a comma, or else report an error if !p.eat(exp!(Comma)) { if first { p.clear_expected_token_types(); @@ -120,9 +111,11 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, } } first = false; + // accept a trailing comma if p.token == token::Eof { break; - } // accept trailing commas + } + // parse a FormatArgument match p.token.ident() { Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => { p.bump(); @@ -156,6 +149,10 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, } } } + + // Only allow implicit captures for direct literals + let is_direct_literal = matches!(fmtstr.kind, ExprKind::Lit(_)); + Ok(MacroInput { fmtstr, args, is_direct_literal }) } @@ -565,9 +562,11 @@ fn make_format_args( &used, &args, &pieces, + &invalid_refs, detect_foreign_fmt, str_style, fmt_str, + uncooked_fmt_str.1.as_str(), fmt_span, ); } @@ -596,7 +595,8 @@ fn make_format_args( named_arg_sp: arg_name.span, named_arg_name: arg_name.name.to_string(), is_formatting_arg: matches!(used_as, Width | Precision), - }, + } + .into(), }); } } @@ -644,9 +644,11 @@ fn report_missing_placeholders( used: &[bool], args: &FormatArguments, pieces: &[parse::Piece<'_>], + invalid_refs: &[(usize, Option, PositionUsedAs, FormatArgPositionKind)], detect_foreign_fmt: bool, str_style: Option, fmt_str: &str, + uncooked_fmt_str: &str, fmt_span: Span, ) { let mut diag = if let &[(span, named)] = &unused[..] { @@ -761,6 +763,35 @@ fn report_missing_placeholders( diag.span_label(fmt_span, "formatting specifier missing"); } + if !found_foreign && invalid_refs.is_empty() { + // Show example if user didn't use any format specifiers + let show_example = !used.contains(&true); + + if !show_example { + if unused.len() > 1 { + diag.note(format!("consider adding {} format specifiers", unused.len())); + } + } else { + let msg = if unused.len() == 1 { + "a format specifier".to_string() + } else { + format!("{} format specifiers", unused.len()) + }; + + let sugg = match str_style { + None => format!("\"{}{}\"", uncooked_fmt_str, "{}".repeat(unused.len())), + Some(n_hashes) => format!( + "r{hashes}\"{uncooked_fmt_str}{fmt_specifiers}\"{hashes}", + hashes = "#".repeat(n_hashes), + fmt_specifiers = "{}".repeat(unused.len()) + ), + }; + let msg = format!("format specifiers use curly braces, consider adding {msg}"); + + diag.span_suggestion_verbose(fmt_span, msg, sugg, Applicability::MaybeIncorrect); + } + } + diag.emit(); } @@ -1018,7 +1049,7 @@ fn expand_format_args_impl<'cx>( }; match mac { Ok(format_args) => { - MacEager::expr(ecx.expr(sp, ExprKind::FormatArgs(P(format_args)))) + MacEager::expr(ecx.expr(sp, ExprKind::FormatArgs(Box::new(format_args)))) } Err(guar) => MacEager::expr(DummyResult::raw_expr(sp, Some(guar))), } diff --git a/compiler/rustc_builtin_macros/src/format_foreign.rs b/compiler/rustc_builtin_macros/src/format_foreign.rs index 13d5b42942ac..cf563a53973c 100644 --- a/compiler/rustc_builtin_macros/src/format_foreign.rs +++ b/compiler/rustc_builtin_macros/src/format_foreign.rs @@ -346,18 +346,18 @@ pub(crate) mod printf { // ```regex // (?x) // ^ % - // (?: (?P \d+) \$ )? - // (?P [-+ 0\#']* ) - // (?P \d+ | \* (?: (?P \d+) \$ )? )? - // (?: \. (?P \d+ | \* (?: (?P \d+) \$ )? ) )? - // (?P + // (?: (?Box \d+) \$ )? + // (?Box [-+ 0\#']* ) + // (?Box \d+ | \* (?: (?Box \d+) \$ )? )? + // (?: \. (?Box \d+ | \* (?: (?Box \d+) \$ )? ) )? + // (?Box // # Standard // hh | h | ll | l | L | z | j | t // // # Other // | I32 | I64 | I | q // )? - // (?P . ) + // (?Box . ) // ``` // Used to establish the full span at the end. @@ -416,7 +416,7 @@ pub(crate) mod printf { // Yes, this *is* the parameter. Some(('$', end2)) => { state = Flags; - parameter = Some(at.slice_between(end).unwrap().parse().unwrap()); + parameter = at.slice_between(end).unwrap().parse().ok(); move_to!(end2); } // Wait, no, actually, it's the width. diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs index 4b1958bce322..c968353504e1 100644 --- a/compiler/rustc_builtin_macros/src/global_allocator.rs +++ b/compiler/rustc_builtin_macros/src/global_allocator.rs @@ -1,7 +1,6 @@ use rustc_ast::expand::allocator::{ ALLOCATOR_METHODS, AllocatorMethod, AllocatorMethodInput, AllocatorTy, global_fn_name, }; -use rustc_ast::ptr::P; use rustc_ast::{ self as ast, AttrVec, Expr, Fn, FnHeader, FnSig, Generics, ItemKind, Mutability, Param, Safety, Stmt, StmtKind, Ty, TyKind, @@ -51,7 +50,7 @@ pub(crate) fn expand( let const_body = ecx.expr_block(ecx.block(span, stmts)); let const_item = ecx.item_const(span, Ident::new(kw::Underscore, span), const_ty, const_body); let const_item = if is_stmt { - Annotatable::Stmt(P(ecx.stmt_item(span, const_item))) + Annotatable::Stmt(Box::new(ecx.stmt_item(span, const_item))) } else { Annotatable::Item(const_item) }; @@ -86,11 +85,11 @@ impl AllocFnFactory<'_, '_> { body, define_opaque: None, })); - let item = self.cx.item(self.span, self.attrs(), kind); + let item = self.cx.item(self.span, self.attrs(method), kind); self.cx.stmt_item(self.ty_span, item) } - fn call_allocator(&self, method: Symbol, mut args: ThinVec>) -> P { + fn call_allocator(&self, method: Symbol, mut args: ThinVec>) -> Box { let method = self.cx.std_path(&[sym::alloc, sym::GlobalAlloc, method]); let method = self.cx.expr_path(self.cx.path(self.ty_span, method)); let allocator = self.cx.path_ident(self.ty_span, self.global); @@ -101,11 +100,21 @@ impl AllocFnFactory<'_, '_> { self.cx.expr_call(self.ty_span, method, args) } - fn attrs(&self) -> AttrVec { - thin_vec![self.cx.attr_word(sym::rustc_std_internal_symbol, self.span)] + fn attrs(&self, method: &AllocatorMethod) -> AttrVec { + let alloc_attr = match method.name { + sym::alloc => sym::rustc_allocator, + sym::dealloc => sym::rustc_deallocator, + sym::realloc => sym::rustc_reallocator, + sym::alloc_zeroed => sym::rustc_allocator_zeroed, + _ => unreachable!("Unknown allocator method!"), + }; + thin_vec![ + self.cx.attr_word(sym::rustc_std_internal_symbol, self.span), + self.cx.attr_word(alloc_attr, self.span) + ] } - fn arg_ty(&self, input: &AllocatorMethodInput, args: &mut ThinVec) -> P { + fn arg_ty(&self, input: &AllocatorMethodInput, args: &mut ThinVec) -> Box { match input.ty { AllocatorTy::Layout => { // If an allocator method is ever introduced having multiple @@ -142,30 +151,30 @@ impl AllocFnFactory<'_, '_> { self.cx.expr_ident(self.span, ident) } - AllocatorTy::ResultPtr | AllocatorTy::Unit => { + AllocatorTy::Never | AllocatorTy::ResultPtr | AllocatorTy::Unit => { panic!("can't convert AllocatorTy to an argument") } } } - fn ret_ty(&self, ty: &AllocatorTy) -> P { + fn ret_ty(&self, ty: &AllocatorTy) -> Box { match *ty { AllocatorTy::ResultPtr => self.ptr_u8(), AllocatorTy::Unit => self.cx.ty(self.span, TyKind::Tup(ThinVec::new())), - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + AllocatorTy::Layout | AllocatorTy::Never | AllocatorTy::Usize | AllocatorTy::Ptr => { panic!("can't convert `AllocatorTy` to an output") } } } - fn usize(&self) -> P { + fn usize(&self) -> Box { let usize = self.cx.path_ident(self.span, Ident::new(sym::usize, self.span)); self.cx.ty_path(usize) } - fn ptr_u8(&self) -> P { + fn ptr_u8(&self) -> Box { let u8 = self.cx.path_ident(self.span, Ident::new(sym::u8, self.span)); let ty_u8 = self.cx.ty_path(u8); self.cx.ty_ptr(self.span, ty_u8, Mutability::Mut) diff --git a/compiler/rustc_builtin_macros/src/iter.rs b/compiler/rustc_builtin_macros/src/iter.rs index 7ad83903a1be..e9f340ef1197 100644 --- a/compiler/rustc_builtin_macros/src/iter.rs +++ b/compiler/rustc_builtin_macros/src/iter.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{CoroutineKind, DUMMY_NODE_ID, Expr, ast, token}; use rustc_errors::PResult; @@ -24,7 +23,7 @@ fn parse_closure<'a>( cx: &mut ExtCtxt<'a>, span: Span, stream: TokenStream, -) -> PResult<'a, P> { +) -> PResult<'a, Box> { let mut closure_parser = cx.new_parser_from_tts(stream); let coroutine_kind = Some(CoroutineKind::Gen { diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs index 7bc448a9acb8..57cf62ea6121 100644 --- a/compiler/rustc_builtin_macros/src/lib.rs +++ b/compiler/rustc_builtin_macros/src/lib.rs @@ -8,10 +8,10 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] #![feature(assert_matches)] -#![feature(autodiff)] #![feature(box_patterns)] #![feature(decl_macro)] #![feature(if_let_guard)] +#![feature(iter_order_by)] #![feature(proc_macro_internals)] #![feature(proc_macro_quote)] #![feature(rustdoc_internals)] @@ -130,7 +130,6 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { Clone: clone::expand_deriving_clone, Copy: bounds::expand_deriving_copy, ConstParamTy: bounds::expand_deriving_const_param_ty, - UnsizedConstParamTy: bounds::expand_deriving_unsized_const_param_ty, Debug: debug::expand_deriving_debug, Default: default::expand_deriving_default, Eq: eq::expand_deriving_eq, @@ -139,6 +138,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) { PartialEq: partial_eq::expand_deriving_partial_eq, PartialOrd: partial_ord::expand_deriving_partial_ord, CoercePointee: coerce_pointee::expand_deriving_coerce_pointee, + From: from::expand_deriving_from, } let client = rustc_proc_macro::bridge::client::Client::expand1(rustc_proc_macro::quote); diff --git a/compiler/rustc_builtin_macros/src/pattern_type.rs b/compiler/rustc_builtin_macros/src/pattern_type.rs index b61af0b0aaa1..87a5a440140e 100644 --- a/compiler/rustc_builtin_macros/src/pattern_type.rs +++ b/compiler/rustc_builtin_macros/src/pattern_type.rs @@ -1,4 +1,3 @@ -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{AnonConst, DUMMY_NODE_ID, Ty, TyPat, TyPatKind, ast, token}; use rustc_errors::PResult; @@ -22,42 +21,51 @@ pub(crate) fn expand<'cx>( ExpandResult::Ready(base::MacEager::ty(cx.ty(sp, ast::TyKind::Pat(ty, pat)))) } -fn parse_pat_ty<'a>(cx: &mut ExtCtxt<'a>, stream: TokenStream) -> PResult<'a, (P, P)> { +fn parse_pat_ty<'a>( + cx: &mut ExtCtxt<'a>, + stream: TokenStream, +) -> PResult<'a, (Box, Box)> { let mut parser = cx.new_parser_from_tts(stream); let ty = parser.parse_ty()?; parser.expect_keyword(exp!(Is))?; - let pat = pat_to_ty_pat( - cx, - *parser.parse_pat_no_top_guard( - None, - RecoverComma::No, - RecoverColon::No, - CommaRecoveryMode::EitherTupleOrPipe, - )?, - ); + let start = parser.token.span; + let pat = if parser.eat(exp!(Bang)) { + parser.expect_keyword(exp!(Null))?; + ty_pat(TyPatKind::NotNull, start.to(parser.token.span)) + } else { + pat_to_ty_pat( + cx, + parser.parse_pat_no_top_guard( + None, + RecoverComma::No, + RecoverColon::No, + CommaRecoveryMode::EitherTupleOrPipe, + )?, + ) + }; if parser.token != token::Eof { parser.unexpected()?; } - Ok((ty, pat)) + Ok((ty, Box::new(pat))) } -fn ty_pat(kind: TyPatKind, span: Span) -> P { - P(TyPat { id: DUMMY_NODE_ID, kind, span, tokens: None }) +fn ty_pat(kind: TyPatKind, span: Span) -> TyPat { + TyPat { id: DUMMY_NODE_ID, kind, span, tokens: None } } -fn pat_to_ty_pat(cx: &mut ExtCtxt<'_>, pat: ast::Pat) -> P { +fn pat_to_ty_pat(cx: &mut ExtCtxt<'_>, pat: ast::Pat) -> TyPat { let kind = match pat.kind { ast::PatKind::Range(start, end, include_end) => TyPatKind::Range( - start.map(|value| P(AnonConst { id: DUMMY_NODE_ID, value })), - end.map(|value| P(AnonConst { id: DUMMY_NODE_ID, value })), + start.map(|value| Box::new(AnonConst { id: DUMMY_NODE_ID, value })), + end.map(|value| Box::new(AnonConst { id: DUMMY_NODE_ID, value })), include_end, ), ast::PatKind::Or(variants) => { - TyPatKind::Or(variants.into_iter().map(|pat| pat_to_ty_pat(cx, *pat)).collect()) + TyPatKind::Or(variants.into_iter().map(|pat| pat_to_ty_pat(cx, pat)).collect()) } ast::PatKind::Err(guar) => TyPatKind::Err(guar), _ => TyPatKind::Err(cx.dcx().span_err(pat.span, "pattern not supported in pattern types")), diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs index 09f5e6f6efc1..6ac3e17503d0 100644 --- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs +++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs @@ -1,15 +1,14 @@ use std::{mem, slice}; -use rustc_ast::ptr::P; use rustc_ast::visit::{self, Visitor}; use rustc_ast::{self as ast, HasNodeId, NodeId, attr}; use rustc_ast_pretty::pprust; -use rustc_attr_data_structures::AttributeKind; use rustc_attr_parsing::AttributeParser; use rustc_errors::DiagCtxtHandle; use rustc_expand::base::{ExtCtxt, ResolverExpand}; use rustc_expand::expand::{AstFragment, ExpansionConfig}; use rustc_feature::Features; +use rustc_hir::attrs::AttributeKind; use rustc_session::Session; use rustc_span::hygiene::AstPass; use rustc_span::source_map::SourceMap; @@ -232,12 +231,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> { let fn_ident = if let ast::ItemKind::Fn(fn_) = &item.kind { fn_.ident } else { - self.dcx - .create_err(errors::AttributeOnlyBeUsedOnBareFunctions { - span: attr.span, - path: &pprust::path_to_string(&attr.get_normal_item().path), - }) - .emit(); + // Error handled by general target checking logic return; }; @@ -286,7 +280,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> { // // ... // ]; // } -fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P { +fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box { let expn_id = cx.resolver.expansion_for_ast_pass( DUMMY_SP, AstPass::ProcMacroHarness, diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs index ecfd46a84ec3..16adaab15c52 100644 --- a/compiler/rustc_builtin_macros/src/source_util.rs +++ b/compiler/rustc_builtin_macros/src/source_util.rs @@ -1,9 +1,10 @@ +//! The implementation of built-in macros which relate to the file system. + use std::path::{Path, PathBuf}; use std::rc::Rc; use std::sync::Arc; use rustc_ast as ast; -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{join_path_idents, token}; use rustc_ast_pretty::pprust; @@ -11,10 +12,11 @@ use rustc_expand::base::{ DummyResult, ExpandResult, ExtCtxt, MacEager, MacResult, MacroExpanderResult, resolve_path, }; use rustc_expand::module::DirOwnership; -use rustc_lint_defs::BuiltinLintDiag; -use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_parse::lexer::StripTokens; +use rustc_parse::parser::ForceCollect; use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal, utf8_error}; use rustc_session::lint::builtin::INCOMPLETE_INCLUDE; +use rustc_session::parse::ParseSess; use rustc_span::source_map::SourceMap; use rustc_span::{ByteSymbol, Pos, Span, Symbol}; use smallvec::SmallVec; @@ -24,11 +26,7 @@ use crate::util::{ check_zero_tts, get_single_str_from_tts, get_single_str_spanned_from_tts, parse_expr, }; -// These macros all relate to the file system; they either return -// the column/row/filename of the expression, or they include -// a given file into the current one. - -/// line!(): expands to the current line number +/// Expand `line!()` to the current line number. pub(crate) fn expand_line( cx: &mut ExtCtxt<'_>, sp: Span, @@ -43,7 +41,7 @@ pub(crate) fn expand_line( ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.line as u32))) } -/* column!(): expands to the current column number */ +/// Expand `column!()` to the current column number. pub(crate) fn expand_column( cx: &mut ExtCtxt<'_>, sp: Span, @@ -58,9 +56,7 @@ pub(crate) fn expand_column( ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1))) } -/// file!(): expands to the current filename */ -/// The source_file (`loc.file`) contains a bunch more information we could spit -/// out if we wanted. +/// Expand `file!()` to the current filename. pub(crate) fn expand_file( cx: &mut ExtCtxt<'_>, sp: Span, @@ -82,6 +78,7 @@ pub(crate) fn expand_file( ))) } +/// Expand `stringify!($input)`. pub(crate) fn expand_stringify( cx: &mut ExtCtxt<'_>, sp: Span, @@ -92,6 +89,7 @@ pub(crate) fn expand_stringify( ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&s)))) } +/// Expand `module_path!()` to (a textual representation of) the current module path. pub(crate) fn expand_mod( cx: &mut ExtCtxt<'_>, sp: Span, @@ -105,9 +103,9 @@ pub(crate) fn expand_mod( ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&string)))) } -/// include! : parse the given file as an expr -/// This is generally a bad idea because it's going to behave -/// unhygienically. +/// Expand `include!($input)`. +/// +/// This works in item and expression position. Notably, it doesn't work in pattern position. pub(crate) fn expand_include<'cx>( cx: &'cx mut ExtCtxt<'_>, sp: Span, @@ -117,64 +115,76 @@ pub(crate) fn expand_include<'cx>( let ExpandResult::Ready(mac) = get_single_str_from_tts(cx, sp, tts, "include!") else { return ExpandResult::Retry(()); }; - let file = match mac { - Ok(file) => file, + let path = match mac { + Ok(path) => path, Err(guar) => return ExpandResult::Ready(DummyResult::any(sp, guar)), }; // The file will be added to the code map by the parser - let file = match resolve_path(&cx.sess, file.as_str(), sp) { - Ok(f) => f, + let path = match resolve_path(&cx.sess, path.as_str(), sp) { + Ok(path) => path, Err(err) => { let guar = err.emit(); return ExpandResult::Ready(DummyResult::any(sp, guar)); } }; - let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(), &file, Some(sp))); // If in the included file we have e.g., `mod bar;`, - // then the path of `bar.rs` should be relative to the directory of `file`. + // then the path of `bar.rs` should be relative to the directory of `path`. // See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion. // `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained. - let dir_path = file.parent().unwrap_or(&file).to_owned(); + let dir_path = path.parent().unwrap_or(&path).to_owned(); cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path)); cx.current_expansion.dir_ownership = DirOwnership::Owned { relative: None }; struct ExpandInclude<'a> { - p: Parser<'a>, + psess: &'a ParseSess, + path: PathBuf, node_id: ast::NodeId, + span: Span, } impl<'a> MacResult for ExpandInclude<'a> { - fn make_expr(mut self: Box>) -> Option> { - let expr = parse_expr(&mut self.p).ok()?; - if self.p.token != token::Eof { - self.p.psess.buffer_lint( + fn make_expr(self: Box>) -> Option> { + let mut p = unwrap_or_emit_fatal(new_parser_from_file( + self.psess, + &self.path, + // Don't strip frontmatter for backward compatibility, `---` may be the start of a + // manifold negation. FIXME: Ideally, we wouldn't strip shebangs here either. + StripTokens::Shebang, + Some(self.span), + )); + let expr = parse_expr(&mut p).ok()?; + if p.token != token::Eof { + p.psess.buffer_lint( INCOMPLETE_INCLUDE, - self.p.token.span, + p.token.span, self.node_id, - BuiltinLintDiag::IncompleteInclude, + errors::IncompleteInclude, ); } Some(expr) } - fn make_items(mut self: Box>) -> Option; 1]>> { + fn make_items(self: Box>) -> Option; 1]>> { + let mut p = unwrap_or_emit_fatal(new_parser_from_file( + self.psess, + &self.path, + StripTokens::ShebangAndFrontmatter, + Some(self.span), + )); let mut ret = SmallVec::new(); loop { - match self.p.parse_item(ForceCollect::No) { + match p.parse_item(ForceCollect::No) { Err(err) => { err.emit(); break; } Ok(Some(item)) => ret.push(item), Ok(None) => { - if self.p.token != token::Eof { - self.p - .dcx() - .create_err(errors::ExpectedItem { - span: self.p.token.span, - token: &pprust::token_to_string(&self.p.token), - }) - .emit(); + if p.token != token::Eof { + p.dcx().emit_err(errors::ExpectedItem { + span: p.token.span, + token: &pprust::token_to_string(&p.token), + }); } break; @@ -185,10 +195,17 @@ pub(crate) fn expand_include<'cx>( } } - ExpandResult::Ready(Box::new(ExpandInclude { p, node_id: cx.current_expansion.lint_node_id })) + ExpandResult::Ready(Box::new(ExpandInclude { + psess: cx.psess(), + path, + node_id: cx.current_expansion.lint_node_id, + span: sp, + })) } -/// `include_str!`: read the given file, insert it as a literal string expr +/// Expand `include_str!($input)` to the content of the UTF-8-encoded file given by path `$input` as a string literal. +/// +/// This works in expression, pattern and statement position. pub(crate) fn expand_include_str( cx: &mut ExtCtxt<'_>, sp: Span, @@ -207,6 +224,7 @@ pub(crate) fn expand_include_str( Ok((bytes, bsp)) => match std::str::from_utf8(&bytes) { Ok(src) => { let interned_src = Symbol::intern(src); + // MacEager converts the expr into a pat if need be. MacEager::expr(cx.expr_str(cx.with_def_site_ctxt(bsp), interned_src)) } Err(utf8err) => { @@ -219,6 +237,9 @@ pub(crate) fn expand_include_str( }) } +/// Expand `include_bytes!($input)` to the content of the file given by path `$input`. +/// +/// This works in expression, pattern and statement position. pub(crate) fn expand_include_bytes( cx: &mut ExtCtxt<'_>, sp: Span, @@ -238,6 +259,7 @@ pub(crate) fn expand_include_bytes( // Don't care about getting the span for the raw bytes, // because the console can't really show them anyway. let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(ByteSymbol::intern(&bytes))); + // MacEager converts the expr into a pat if need be. MacEager::expr(expr) } Err(dummy) => dummy, diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs index ba3d8368b2a0..7a189ee1f4d0 100644 --- a/compiler/rustc_builtin_macros/src/test.rs +++ b/compiler/rustc_builtin_macros/src/test.rs @@ -4,11 +4,13 @@ use std::assert_matches::assert_matches; use std::iter; -use rustc_ast::ptr::P; -use rustc_ast::{self as ast, GenericParamKind, attr, join_path_idents}; +use rustc_ast::{self as ast, GenericParamKind, HasNodeId, attr, join_path_idents}; use rustc_ast_pretty::pprust; +use rustc_attr_parsing::AttributeParser; use rustc_errors::{Applicability, Diag, Level}; use rustc_expand::base::*; +use rustc_hir::Attribute; +use rustc_hir::attrs::AttributeKind; use rustc_span::{ErrorGuaranteed, FileNameDisplayPreference, Ident, Span, Symbol, sym}; use thin_vec::{ThinVec, thin_vec}; use tracing::debug; @@ -75,7 +77,7 @@ pub(crate) fn expand_test_case( } let ret = if is_stmt { - Annotatable::Stmt(P(ecx.stmt_item(item.span, item))) + Annotatable::Stmt(Box::new(ecx.stmt_item(item.span, item))) } else { Annotatable::Item(item) }; @@ -128,7 +130,7 @@ pub(crate) fn expand_test_or_bench( let ast::ItemKind::Fn(fn_) = &item.kind else { not_testable_error(cx, attr_sp, Some(&item)); return if is_stmt { - vec![Annotatable::Stmt(P(cx.stmt_item(item.span, item)))] + vec![Annotatable::Stmt(Box::new(cx.stmt_item(item.span, item)))] } else { vec![Annotatable::Item(item)] }; @@ -152,7 +154,7 @@ pub(crate) fn expand_test_or_bench( }; if check_result.is_err() { return if is_stmt { - vec![Annotatable::Stmt(P(cx.stmt_item(item.span, item)))] + vec![Annotatable::Stmt(Box::new(cx.stmt_item(item.span, item)))] } else { vec![Annotatable::Item(item)] }; @@ -198,7 +200,7 @@ pub(crate) fn expand_test_or_bench( // `-Cinstrument-coverage` builds. // This requires `#[allow_internal_unstable(coverage_attribute)]` on the // corresponding macro declaration in `core::macros`. - let coverage_off = |mut expr: P| { + let coverage_off = |mut expr: Box| { assert_matches!(expr.kind, ast::ExprKind::Closure(_)); expr.attrs.push(cx.attr_nested_word(sym::coverage, sym::off, sp)); expr @@ -385,11 +387,11 @@ pub(crate) fn expand_test_or_bench( if is_stmt { vec![ // Access to libtest under a hygienic name - Annotatable::Stmt(P(cx.stmt_item(sp, test_extern))), + Annotatable::Stmt(Box::new(cx.stmt_item(sp, test_extern))), // The generated test case - Annotatable::Stmt(P(cx.stmt_item(sp, test_const))), + Annotatable::Stmt(Box::new(cx.stmt_item(sp, test_const))), // The original item - Annotatable::Stmt(P(cx.stmt_item(sp, item))), + Annotatable::Stmt(Box::new(cx.stmt_item(sp, item))), ] } else { vec![ @@ -473,39 +475,19 @@ fn should_ignore_message(i: &ast::Item) -> Option { } fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic { - match attr::find_by_name(&i.attrs, sym::should_panic) { - Some(attr) => { - match attr.meta_item_list() { - // Handle #[should_panic(expected = "foo")] - Some(list) => { - let msg = list - .iter() - .find(|mi| mi.has_name(sym::expected)) - .and_then(|mi| mi.meta_item()) - .and_then(|mi| mi.value_str()); - if list.len() != 1 || msg.is_none() { - cx.dcx() - .struct_span_warn( - attr.span, - "argument must be of the form: \ - `expected = \"error message\"`", - ) - .with_note( - "errors in this attribute were erroneously \ - allowed and will become a hard error in a \ - future release", - ) - .emit(); - ShouldPanic::Yes(None) - } else { - ShouldPanic::Yes(msg) - } - } - // Handle #[should_panic] and #[should_panic = "expected"] - None => ShouldPanic::Yes(attr.value_str()), - } - } - None => ShouldPanic::No, + if let Some(Attribute::Parsed(AttributeKind::ShouldPanic { reason, .. })) = + AttributeParser::parse_limited( + cx.sess, + &i.attrs, + sym::should_panic, + i.span, + i.node_id(), + None, + ) + { + ShouldPanic::Yes(reason) + } else { + ShouldPanic::No } } diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs index 111c85d49eb0..2a6ac5754bfa 100644 --- a/compiler/rustc_builtin_macros/src/test_harness.rs +++ b/compiler/rustc_builtin_macros/src/test_harness.rs @@ -3,16 +3,15 @@ use std::mem; use rustc_ast as ast; +use rustc_ast::attr::contains_name; use rustc_ast::entry::EntryPointType; use rustc_ast::mut_visit::*; -use rustc_ast::ptr::P; use rustc_ast::visit::Visitor; use rustc_ast::{ModKind, attr}; use rustc_errors::DiagCtxtHandle; use rustc_expand::base::{ExtCtxt, ResolverExpand}; use rustc_expand::expand::{AstFragment, ExpansionConfig}; use rustc_feature::Features; -use rustc_lint_defs::BuiltinLintDiag; use rustc_session::Session; use rustc_session::lint::builtin::UNNAMEABLE_TEST_ITEMS; use rustc_span::hygiene::{AstPass, SyntaxContext, Transparency}; @@ -65,8 +64,8 @@ pub fn inject( if sess.is_test_crate() { let panic_strategy = match (panic_strategy, sess.opts.unstable_opts.panic_abort_tests) { - (PanicStrategy::Abort, true) => PanicStrategy::Abort, - (PanicStrategy::Abort, false) => { + (PanicStrategy::Abort | PanicStrategy::ImmediateAbort, true) => panic_strategy, + (PanicStrategy::Abort | PanicStrategy::ImmediateAbort, false) => { if panic_strategy == platform_panic_strategy { // Silently allow compiling with panic=abort on these platforms, // but with old behavior (abort if a test fails). @@ -142,7 +141,7 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> { if let ast::ItemKind::Mod( _, _, - ModKind::Loaded(.., ast::ModSpans { inner_span: span, .. }, _), + ModKind::Loaded(.., ast::ModSpans { inner_span: span, .. }), ) = item.kind { let prev_tests = mem::take(&mut self.tests); @@ -166,7 +165,7 @@ impl<'a> Visitor<'a> for InnerItemLinter<'_> { UNNAMEABLE_TEST_ITEMS, attr.span, i.id, - BuiltinLintDiag::UnnameableTestItems, + errors::UnnameableTestItems, ); } } @@ -174,9 +173,11 @@ impl<'a> Visitor<'a> for InnerItemLinter<'_> { fn entry_point_type(item: &ast::Item, at_root: bool) -> EntryPointType { match &item.kind { - ast::ItemKind::Fn(fn_) => { - rustc_ast::entry::entry_point_type(&item.attrs, at_root, Some(fn_.ident.name)) - } + ast::ItemKind::Fn(fn_) => rustc_ast::entry::entry_point_type( + contains_name(&item.attrs, sym::rustc_main), + at_root, + Some(fn_.ident.name), + ), _ => EntryPointType::None, } } @@ -284,15 +285,13 @@ fn generate_test_harness( /// [`TestCtxt::reexport_test_harness_main`] provides a different name for the `main` /// function and [`TestCtxt::test_runner`] provides a path that replaces /// `test::test_main_static`. -fn mk_main(cx: &mut TestCtxt<'_>) -> P { +fn mk_main(cx: &mut TestCtxt<'_>) -> Box { let sp = cx.def_site; let ecx = &cx.ext_cx; let test_ident = Ident::new(sym::test, sp); - let runner_name = match cx.panic_strategy { - PanicStrategy::Unwind => "test_main_static", - PanicStrategy::Abort => "test_main_static_abort", - }; + let runner_name = + if cx.panic_strategy.unwinds() { "test_main_static" } else { "test_main_static_abort" }; // test::test_main_static(...) let mut test_runner = cx.test_runner.clone().unwrap_or_else(|| { @@ -348,7 +347,7 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P { define_opaque: None, })); - let main = P(ast::Item { + let main = Box::new(ast::Item { attrs: thin_vec![main_attr, coverage_attr, doc_hidden_attr], id: ast::DUMMY_NODE_ID, kind: main, @@ -364,7 +363,7 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P { /// Creates a slice containing every test like so: /// &[&test1, &test2] -fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> P { +fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> Box { debug!("building test vector from {} tests", cx.test_cases.len()); let ecx = &cx.ext_cx; diff --git a/compiler/rustc_builtin_macros/src/util.rs b/compiler/rustc_builtin_macros/src/util.rs index 38fec2bff14c..e26f31dce67b 100644 --- a/compiler/rustc_builtin_macros/src/util.rs +++ b/compiler/rustc_builtin_macros/src/util.rs @@ -1,13 +1,12 @@ -use rustc_ast::ptr::P; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{self as ast, AttrStyle, Attribute, MetaItem, attr, token}; +use rustc_attr_parsing::validate_attr; use rustc_errors::{Applicability, Diag, ErrorGuaranteed}; use rustc_expand::base::{Annotatable, ExpandResult, ExtCtxt}; use rustc_expand::expand::AstFragment; use rustc_feature::AttributeTemplate; -use rustc_lint_defs::BuiltinLintDiag; use rustc_lint_defs::builtin::DUPLICATE_MACRO_ATTRIBUTES; -use rustc_parse::{exp, parser, validate_attr}; +use rustc_parse::{exp, parser}; use rustc_session::errors::report_lit_error; use rustc_span::{BytePos, Span, Symbol}; @@ -49,7 +48,7 @@ pub(crate) fn warn_on_duplicate_attribute(ecx: &ExtCtxt<'_>, item: &Annotatable, DUPLICATE_MACRO_ATTRIBUTES, attr.span, ecx.current_expansion.lint_node_id, - BuiltinLintDiag::DuplicateMacroAttribute, + errors::DuplicateMacroAttribute, ); } } @@ -83,7 +82,7 @@ type UnexpectedExprKind<'a> = Result<(Diag<'a>, bool /* has_suggestions */), Err #[allow(rustc::untranslatable_diagnostic)] pub(crate) fn expr_to_spanned_string<'a>( cx: &'a mut ExtCtxt<'_>, - expr: P, + expr: Box, err_msg: &'static str, ) -> ExpandResult, ()> { if !cx.force_mode @@ -135,7 +134,7 @@ pub(crate) fn expr_to_spanned_string<'a>( /// compilation on error, merely emits a non-fatal error and returns `Err`. pub(crate) fn expr_to_string( cx: &mut ExtCtxt<'_>, - expr: P, + expr: Box, err_msg: &'static str, ) -> ExpandResult, ()> { expr_to_spanned_string(cx, expr, err_msg).map(|res| { @@ -158,7 +157,7 @@ pub(crate) fn check_zero_tts(cx: &ExtCtxt<'_>, span: Span, tts: TokenStream, nam } /// Parse an expression. On error, emit it, advancing to `Eof`, and return `Err`. -pub(crate) fn parse_expr(p: &mut parser::Parser<'_>) -> Result, ErrorGuaranteed> { +pub(crate) fn parse_expr(p: &mut parser::Parser<'_>) -> Result, ErrorGuaranteed> { let guar = match p.parse_expr() { Ok(expr) => return Ok(expr), Err(err) => err.emit(), @@ -209,7 +208,7 @@ pub(crate) fn get_single_expr_from_tts( span: Span, tts: TokenStream, name: &str, -) -> ExpandResult, ErrorGuaranteed>, ()> { +) -> ExpandResult, ErrorGuaranteed>, ()> { let mut p = cx.new_parser_from_tts(tts); if p.token == token::Eof { let guar = cx.dcx().emit_err(errors::OnlyOneArgument { span, name }); @@ -232,7 +231,7 @@ pub(crate) fn get_single_expr_from_tts( pub(crate) fn get_exprs_from_tts( cx: &mut ExtCtxt<'_>, tts: TokenStream, -) -> ExpandResult>, ErrorGuaranteed>, ()> { +) -> ExpandResult>, ErrorGuaranteed>, ()> { let mut p = cx.new_parser_from_tts(tts); let mut es = Vec::new(); while p.token != token::Eof { diff --git a/compiler/rustc_codegen_cranelift/example/example.rs b/compiler/rustc_codegen_cranelift/example/example.rs index aeb38331edb0..769d262b9ebb 100644 --- a/compiler/rustc_codegen_cranelift/example/example.rs +++ b/compiler/rustc_codegen_cranelift/example/example.rs @@ -72,10 +72,6 @@ pub fn debug_tuple() -> DebugTuple { DebugTuple(()) } -pub fn size_of() -> usize { - intrinsics::size_of::() -} - pub fn use_size_of() -> usize { size_of::() } diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs index 2f53bbf8b793..304d0d648561 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -6,6 +6,7 @@ extern_types, decl_macro, rustc_attrs, + rustc_private, transparent_unions, auto_traits, freeze_impls, @@ -594,7 +595,7 @@ impl, U: ?Sized> CoerceUnsized> for Box {} impl Box { pub fn new(val: T) -> Box { unsafe { - let size = intrinsics::size_of::(); + let size = size_of::(); let ptr = libc::malloc(size); intrinsics::copy(&val as *const T as *const u8, ptr, size); Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, Global) @@ -646,11 +647,11 @@ pub mod intrinsics { #[rustc_intrinsic] pub fn abort() -> !; #[rustc_intrinsic] - pub fn size_of() -> usize; + pub const fn size_of() -> usize; #[rustc_intrinsic] pub unsafe fn size_of_val(val: *const T) -> usize; #[rustc_intrinsic] - pub fn align_of() -> usize; + pub const fn align_of() -> usize; #[rustc_intrinsic] pub unsafe fn align_of_val(val: *const T) -> usize; #[rustc_intrinsic] @@ -715,6 +716,23 @@ impl Index for [T] { } } +pub const fn size_of() -> usize { + ::SIZE +} + +pub const fn align_of() -> usize { + ::ALIGN +} + +trait SizedTypeProperties: Sized { + #[lang = "mem_size_const"] + const SIZE: usize = intrinsics::size_of::(); + + #[lang = "mem_align_const"] + const ALIGN: usize = intrinsics::align_of::(); +} +impl SizedTypeProperties for T {} + extern "C" { type VaListImpl; } diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs index 86602c6b2a3f..a9388814a7f5 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs @@ -109,10 +109,10 @@ fn start( puts(*argv as *const i8); } unsafe { - puts(*((argv as usize + intrinsics::size_of::<*const u8>()) as *const *const i8)); + puts(*((argv as usize + size_of::<*const u8>()) as *const *const i8)); } unsafe { - puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); + puts(*((argv as usize + 2 * size_of::<*const u8>()) as *const *const i8)); } } @@ -213,8 +213,8 @@ fn main() { assert_eq!(intrinsics::size_of_val(a) as u8, 16); assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4); - assert_eq!(intrinsics::align_of::() as u8, 2); - assert_eq!(intrinsics::align_of_val(&a) as u8, intrinsics::align_of::<&str>() as u8); + assert_eq!(align_of::() as u8, 2); + assert_eq!(intrinsics::align_of_val(&a) as u8, align_of::<&str>() as u8); let u8_needs_drop = const { intrinsics::needs_drop::() }; assert!(!u8_needs_drop); diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index 52e02c857c7a..62f1cc6a8933 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -121,7 +121,7 @@ rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift # ============================================================ rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump rm -r tests/run-make/strip # same -rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source +rm -r tests/run-make-cargo/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source rm -r tests/run-make/translation # same rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features rm -r tests/run-make/const-trait-stable-toolchain # same @@ -166,5 +166,5 @@ index 073116933bd..c3e4578204d 100644 EOF echo "[TEST] rustc test suite" -./x.py test --stage 0 --test-args=--no-capture tests/{codegen-units,run-make,ui,incremental} +./x.py test --stage 0 --test-args=--no-capture tests/{codegen-units,run-make,run-make-cargo,ui,incremental} popd diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs index c74efeb59f3f..d1b2b9a502ac 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -89,7 +89,7 @@ pub(super) fn add_local_place_comments<'tcx>( format!("{:?}", local), format!("{:?}", ty), size.bytes(), - align.abi.bytes(), + align.bytes(), if extra.is_empty() { "" } else { " " }, extra, )); diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 7d0731c77bdc..9a9a1f4a2c0f 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -467,7 +467,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( true } else { instance.is_some_and(|inst| { - fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) + fx.tcx.codegen_instance_attrs(inst.def).flags.contains(CodegenFnAttrFlags::COLD) }) }; if is_cold { @@ -715,7 +715,7 @@ pub(crate) fn codegen_drop<'tcx>( fx.bcx.ins().jump(ret_block, &[]); } else { match ty.kind() { - ty::Dynamic(_, _, ty::Dyn) => { + ty::Dynamic(_, _) => { // IN THIS ARM, WE HAVE: // ty = *mut (dyn Trait) // which is: exists ( *mut T, Vtable ) diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index 2031842062d9..7a909a740b05 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -233,7 +233,7 @@ pub(super) fn from_casted_value<'tcx>( // It may also be smaller for example when the type is a wrapper around an integer with a // larger alignment than the integer. std::cmp::max(abi_param_size, layout_size), - u32::try_from(layout.align.abi.bytes()).unwrap(), + u32::try_from(layout.align.bytes()).unwrap(), ); let mut block_params_iter = block_params.iter().copied(); for (offset, _) in abi_params { diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index 04f1129d87c1..67b89114356b 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -3,10 +3,9 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use rustc_ast::expand::allocator::{ - ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, - alloc_error_handler_name, default_fn_name, global_fn_name, + AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; -use rustc_codegen_ssa::base::allocator_kind_for_codegen; +use rustc_codegen_ssa::base::{allocator_kind_for_codegen, allocator_shim_contents}; use rustc_session::config::OomStrategy; use rustc_symbol_mangling::mangle_internal_symbol; @@ -15,74 +14,56 @@ use crate::prelude::*; /// Returns whether an allocator shim was created pub(crate) fn codegen(tcx: TyCtxt<'_>, module: &mut dyn Module) -> bool { let Some(kind) = allocator_kind_for_codegen(tcx) else { return false }; - codegen_inner( - tcx, - module, - kind, - tcx.alloc_error_handler_kind(()).unwrap(), - tcx.sess.opts.unstable_opts.oom, - ); + let methods = allocator_shim_contents(tcx, kind); + codegen_inner(tcx, module, &methods, tcx.sess.opts.unstable_opts.oom); true } fn codegen_inner( tcx: TyCtxt<'_>, module: &mut dyn Module, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], oom_strategy: OomStrategy, ) { let usize_ty = module.target_config().pointer_type(); - if kind == AllocatorKind::Default { - for method in ALLOCATOR_METHODS { - let mut arg_tys = Vec::with_capacity(method.inputs.len()); - for input in method.inputs.iter() { - match input.ty { - AllocatorTy::Layout => { - arg_tys.push(usize_ty); // size - arg_tys.push(usize_ty); // align - } - AllocatorTy::Ptr => arg_tys.push(usize_ty), - AllocatorTy::Usize => arg_tys.push(usize_ty), + for method in methods { + let mut arg_tys = Vec::with_capacity(method.inputs.len()); + for input in method.inputs.iter() { + match input.ty { + AllocatorTy::Layout => { + arg_tys.push(usize_ty); // size + arg_tys.push(usize_ty); // align + } + AllocatorTy::Ptr => arg_tys.push(usize_ty), + AllocatorTy::Usize => arg_tys.push(usize_ty), - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + AllocatorTy::Never | AllocatorTy::ResultPtr | AllocatorTy::Unit => { + panic!("invalid allocator arg") } } - let output = match method.output { - AllocatorTy::ResultPtr => Some(usize_ty), - AllocatorTy::Unit => None, - - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; - - let sig = Signature { - call_conv: module.target_config().default_call_conv, - params: arg_tys.iter().cloned().map(AbiParam::new).collect(), - returns: output.into_iter().map(AbiParam::new).collect(), - }; - crate::common::create_wrapper_function( - module, - sig, - &mangle_internal_symbol(tcx, &global_fn_name(method.name)), - &mangle_internal_symbol(tcx, &default_fn_name(method.name)), - ); } - } + let output = match method.output { + AllocatorTy::ResultPtr => Some(usize_ty), + AllocatorTy::Never | AllocatorTy::Unit => None, - let sig = Signature { - call_conv: module.target_config().default_call_conv, - params: vec![AbiParam::new(usize_ty), AbiParam::new(usize_ty)], - returns: vec![], - }; - crate::common::create_wrapper_function( - module, - sig, - &mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - &mangle_internal_symbol(tcx, alloc_error_handler_name(alloc_error_handler_kind)), - ); + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + + let sig = Signature { + call_conv: module.target_config().default_call_conv, + params: arg_tys.iter().cloned().map(AbiParam::new).collect(), + returns: output.into_iter().map(AbiParam::new).collect(), + }; + crate::common::create_wrapper_function( + module, + sig, + &mangle_internal_symbol(tcx, &global_fn_name(method.name)), + &mangle_internal_symbol(tcx, &default_fn_name(method.name)), + ); + } { let sig = Signature { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index bc0a0f034b23..7d50548b4026 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -44,9 +44,8 @@ pub(crate) fn codegen_fn<'tcx>( let _mir_guard = crate::PrintOnPanic(|| { let mut buf = Vec::new(); with_no_trimmed_paths!({ - use rustc_middle::mir::pretty; - let options = pretty::PrettyPrintMirOptions::from_cli(tcx); - pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf, options).unwrap(); + let writer = pretty::MirWriter::new(tcx); + writer.write_mir_fn(mir, &mut buf).unwrap(); }); String::from_utf8_lossy(&buf).into_owned() }); @@ -601,11 +600,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let val = codegen_operand(fx, operand); lval.write_cvalue(fx, val); } - Rvalue::CopyForDeref(place) => { - let cplace = codegen_place(fx, place); - let val = cplace.to_cvalue(fx); - lval.write_cvalue(fx, val) - } Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) => { let place = codegen_place(fx, place); let ref_ = place.place_ref(fx, lval.layout()); @@ -790,7 +784,7 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let operand = codegen_operand(fx, operand); crate::unsize::coerce_unsized_into(fx, operand, lval); } - Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => { + Rvalue::Cast(CastKind::Transmute | CastKind::Subtype, ref operand, _to_ty) => { let operand = codegen_operand(fx, operand); lval.write_cvalue_transmute(fx, operand); } @@ -835,25 +829,10 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fx.bcx.ins().nop(); } } - Rvalue::Len(place) => { - let place = codegen_place(fx, place); - let usize_layout = fx.layout_of(fx.tcx.types.usize); - let len = codegen_array_len(fx, place); - lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); - } - Rvalue::ShallowInitBox(ref operand, content_ty) => { - let content_ty = fx.monomorphize(content_ty); - let box_layout = fx.layout_of(Ty::new_box(fx.tcx, content_ty)); - let operand = codegen_operand(fx, operand); - let operand = operand.load_scalar(fx); - lval.write_cvalue(fx, CValue::by_val(operand, box_layout)); - } Rvalue::NullaryOp(ref null_op, ty) => { assert!(lval.layout().ty.is_sized(fx.tcx, fx.typing_env())); let layout = fx.layout_of(fx.monomorphize(ty)); let val = match null_op { - NullOp::SizeOf => layout.size.bytes(), - NullOp::AlignOf => layout.align.abi.bytes(), NullOp::OffsetOf(fields) => fx .tcx .offset_of_subfield( @@ -935,11 +914,12 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: let operand = codegen_operand(fx, operand); lval.write_cvalue_transmute(fx, operand); } + Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"), + Rvalue::ShallowInitBox(..) => bug!("`ShallowInitBox` in codegen"), } } StatementKind::StorageLive(_) | StatementKind::StorageDead(_) - | StatementKind::Deinit(_) | StatementKind::ConstEvalCounter | StatementKind::Nop | StatementKind::FakeRead(..) @@ -1003,7 +983,7 @@ pub(crate) fn codegen_place<'tcx>( cplace = cplace.place_deref(fx); } PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"), - PlaceElem::Subtype(ty) | PlaceElem::UnwrapUnsafeBinder(ty) => { + PlaceElem::UnwrapUnsafeBinder(ty) => { cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty)); } PlaceElem::Field(field, _ty) => { diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 2fbe5c02802a..81b1814605a1 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -439,7 +439,10 @@ pub(crate) struct FullyMonomorphizedLayoutCx<'tcx>(pub(crate) TyCtxt<'tcx>); impl<'tcx> LayoutOfHelpers<'tcx> for FullyMonomorphizedLayoutCx<'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::InvalidSimd { .. } + | LayoutError::ReferencesError(_) = err + { self.0.sess.dcx().span_fatal(span, err.to_string()) } else { self.0 @@ -458,7 +461,9 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for FullyMonomorphizedLayoutCx<'tcx> { span: Span, fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::InvalidSimd { .. }) = + err + { self.0.sess.dcx().emit_fatal(Spanned { span, node: err }) } else { match fn_abi_request { diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index a04cfa272376..3243e12e6999 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -5,7 +5,9 @@ use std::cmp::Ordering; use cranelift_module::*; use rustc_data_structures::fx::FxHashSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::interpret::{AllocId, GlobalAlloc, Scalar, read_target_uint}; +use rustc_middle::mir::interpret::{ + AllocId, GlobalAlloc, PointerArithmetic, Scalar, read_target_uint, +}; use rustc_middle::ty::{ExistentialTraitRef, ScalarInt}; use crate::prelude::*; @@ -138,8 +140,11 @@ pub(crate) fn codegen_const_value<'tcx>( let base_addr = match fx.tcx.global_alloc(alloc_id) { GlobalAlloc::Memory(alloc) => { if alloc.inner().len() == 0 { - assert_eq!(offset, Size::ZERO); - fx.bcx.ins().iconst(fx.pointer_type, alloc.inner().align.bytes() as i64) + let val = alloc.inner().align.bytes().wrapping_add(offset.bytes()); + fx.bcx.ins().iconst( + fx.pointer_type, + fx.tcx.truncate_to_target_usize(val) as i64, + ) } else { let data_id = data_id_for_alloc_id( &mut fx.constants_cx, @@ -281,8 +286,8 @@ fn data_id_for_static( .abi .bytes(); - let linkage = if import_linkage == rustc_middle::mir::mono::Linkage::ExternalWeak - || import_linkage == rustc_middle::mir::mono::Linkage::WeakAny + let linkage = if import_linkage == rustc_hir::attrs::Linkage::ExternalWeak + || import_linkage == rustc_hir::attrs::Linkage::WeakAny { Linkage::Preemptible } else { @@ -310,12 +315,15 @@ fn data_id_for_static( // `extern_with_linkage_foo` will instead be initialized to // zero. - let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name); + let ref_name = format!( + "_rust_extern_with_linkage_{:016x}_{symbol_name}", + tcx.stable_crate_id(LOCAL_CRATE) + ); let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap(); let mut data = DataDescription::new(); data.set_align(align); let data_gv = module.declare_data_in_data(data_id, &mut data); - data.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect()); + data.define(std::iter::repeat_n(0, pointer_ty(tcx).bytes() as usize).collect()); data.write_data_addr(0, data_gv, 0); match module.define_data(ref_data_id, &data) { // Every time the static is referenced there will be another definition of this global, @@ -329,8 +337,8 @@ fn data_id_for_static( let linkage = if definition { crate::linkage::get_static_linkage(tcx, def_id) - } else if attrs.linkage == Some(rustc_middle::mir::mono::Linkage::ExternalWeak) - || attrs.linkage == Some(rustc_middle::mir::mono::Linkage::WeakAny) + } else if attrs.linkage == Some(rustc_hir::attrs::Linkage::ExternalWeak) + || attrs.linkage == Some(rustc_hir::attrs::Linkage::WeakAny) { Linkage::Preemptible } else { @@ -594,7 +602,6 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( StatementKind::Assign(_) | StatementKind::FakeRead(_) | StatementKind::SetDiscriminant { .. } - | StatementKind::Deinit(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Retag(_, _) diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index 286e02b986b3..4c438742f3d2 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -304,7 +304,7 @@ impl DebugContext { entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id))); entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line)); - entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(static_layout.align.abi.bytes())); + entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(static_layout.align.bytes())); let mut expr = Expression::new(); expr.op_addr(address_for_data(data_id)); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs index 25b922c8be4c..0d49f32373ca 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/types.rs @@ -166,7 +166,7 @@ impl DebugContext { let tuple_entry = self.dwarf.unit.get_mut(tuple_type_id); tuple_entry.set(gimli::DW_AT_name, AttributeValue::StringRef(self.dwarf.strings.add(name))); tuple_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes())); - tuple_entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(layout.align.abi.bytes())); + tuple_entry.set(gimli::DW_AT_alignment, AttributeValue::Udata(layout.align.bytes())); for (i, (ty, dw_ty)) in components.into_iter().enumerate() { let member_id = self.dwarf.unit.add(tuple_type_id, gimli::DW_TAG_member); @@ -178,9 +178,7 @@ impl DebugContext { member_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty)); member_entry.set( gimli::DW_AT_alignment, - AttributeValue::Udata( - FullyMonomorphizedLayoutCx(tcx).layout_of(ty).align.abi.bytes(), - ), + AttributeValue::Udata(FullyMonomorphizedLayoutCx(tcx).layout_of(ty).align.bytes()), ); member_entry.set( gimli::DW_AT_data_member_location, diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 8ec3599b63d8..7bf1efc10653 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -18,12 +18,11 @@ use rustc_codegen_ssa::{ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; +use rustc_hir::attrs::Linkage as RLinkage; use rustc_metadata::fs::copy_to_stdout; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{ - CodegenUnit, Linkage as RLinkage, MonoItem, MonoItemData, Visibility, -}; +use rustc_middle::mir::mono::{CodegenUnit, MonoItem, MonoItemData, Visibility}; use rustc_session::Session; use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType}; @@ -672,18 +671,7 @@ pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box { } .to_owned(); - let cgus = if tcx.sess.opts.output_types.should_codegen() { - tcx.collect_and_partition_mono_items(()).codegen_units - } else { - // If only `--emit metadata` is used, we shouldn't perform any codegen. - // Also `tcx.collect_and_partition_mono_items` may panic in that case. - return Box::new(OngoingCodegen { - modules: vec![], - allocator_module: None, - crate_info: CrateInfo::new(tcx, target_cpu), - concurrency_limiter: ConcurrencyLimiter::new(0), - }); - }; + let cgus = tcx.collect_and_partition_mono_items(()).codegen_units; if tcx.dep_graph.is_fully_enabled() { for cgu in cgus { diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index b3497503bf06..fec46bf26975 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -33,9 +33,7 @@ fn create_jit_module(tcx: TyCtxt<'_>) -> (UnwindModule, CodegenCx) { } pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec) -> ! { - if !tcx.sess.opts.output_types.should_codegen() { - tcx.dcx().fatal("JIT mode doesn't work with `cargo check`"); - } + // FIXME error on check mode or crate types other than bin in CodegenBackend::init() if !tcx.crate_types().contains(&rustc_session::config::CrateType::Executable) { tcx.dcx().fatal("can't jit non-executable crate"); diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index 203b443269fa..1306c6aa5179 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -42,7 +42,10 @@ impl<'tcx> AsmCodegenMethods<'tcx> for GlobalAsmContext<'_, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for GlobalAsmContext<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::InvalidSimd { .. } + | LayoutError::ReferencesError(_) = err + { self.tcx.sess.dcx().span_fatal(span, err.to_string()) } else { self.tcx diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 4ff5773a06cb..ed40901ac9b8 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -969,7 +969,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = amount.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -982,7 +982,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_xsub => { @@ -991,7 +991,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = amount.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1004,7 +1004,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_and => { @@ -1013,7 +1013,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1025,7 +1025,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_or => { @@ -1034,7 +1034,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1046,7 +1046,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_xor => { @@ -1055,7 +1055,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1067,7 +1067,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_nand => { @@ -1076,7 +1076,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let layout = src.layout(); match layout.ty.kind() { - ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {} + ty::Uint(_) | ty::Int(_) => {} _ => { report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty); return Ok(()); @@ -1088,7 +1088,7 @@ fn codegen_regular_intrinsic_call<'tcx>( let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src); - let old = CValue::by_val(old, layout); + let old = CValue::by_val(old, ret.layout()); ret.write_cvalue(fx, old); } sym::atomic_max => { diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 8e34436fb5e0..5d48d0c94c58 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -26,13 +26,12 @@ extern crate rustc_fs_util; extern crate rustc_hir; extern crate rustc_incremental; extern crate rustc_index; +extern crate rustc_log; extern crate rustc_metadata; extern crate rustc_session; extern crate rustc_span; extern crate rustc_symbol_mangling; extern crate rustc_target; -#[macro_use] -extern crate tracing; // This prevents duplicating functions and statics that are already part of the host rustc process. #[allow(unused_extern_crates)] @@ -46,6 +45,7 @@ use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::{CodegenResults, TargetConfig}; +use rustc_log::tracing::info; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_session::Session; use rustc_session::config::OutputFilenames; @@ -165,6 +165,10 @@ impl CodegenBackend for CraneliftCodegenBackend { "" } + fn name(&self) -> &'static str { + "cranelift" + } + fn init(&self, sess: &Session) { use rustc_session::config::{InstrumentCoverage, Lto}; match sess.lto() { diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs index ca853aac1589..d76ab9d0109f 100644 --- a/compiler/rustc_codegen_cranelift/src/linkage.rs +++ b/compiler/rustc_codegen_cranelift/src/linkage.rs @@ -1,4 +1,5 @@ -use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; +use rustc_hir::attrs::Linkage as RLinkage; +use rustc_middle::mir::mono::{MonoItem, Visibility}; use crate::prelude::*; diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs index 2aee0b2e9742..c97eb3874b02 100644 --- a/compiler/rustc_codegen_cranelift/src/unsize.rs +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -30,9 +30,7 @@ pub(crate) fn unsized_info<'tcx>( fx.pointer_type, len.try_to_target_usize(fx.tcx).expect("expected monomorphic const in codegen") as i64, ), - (&ty::Dynamic(data_a, _, src_dyn_kind), &ty::Dynamic(data_b, _, target_dyn_kind)) - if src_dyn_kind == target_dyn_kind => - { + (&ty::Dynamic(data_a, _), &ty::Dynamic(data_b, _)) => { let old_info = old_info.expect("unsized_info: missing old info for trait upcasting coercion"); let b_principal_def_id = data_b.principal_def_id(); @@ -133,6 +131,11 @@ pub(crate) fn coerce_unsized_into<'tcx>( dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout())); }; match (&src_ty.kind(), &dst_ty.kind()) { + (ty::Pat(a, _), ty::Pat(b, _)) => { + let src = src.cast_pat_ty_to_base(fx.layout_of(*a)); + let dst = dst.place_transmute_type(fx, *b); + return coerce_unsized_into(fx, src, dst); + } (&ty::Ref(..), &ty::Ref(..)) | (&ty::Ref(..), &ty::RawPtr(..)) | (&ty::RawPtr(..), &ty::RawPtr(..)) => coerce_ptr(), @@ -169,7 +172,7 @@ pub(crate) fn size_and_align_of<'tcx>( if layout.is_sized() { return ( fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64), - fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64), + fx.bcx.ins().iconst(fx.pointer_type, layout.align.bytes() as i64), ); } @@ -188,7 +191,7 @@ pub(crate) fn size_and_align_of<'tcx>( // times the unit size. ( fx.bcx.ins().imul_imm(info.unwrap(), unit.size.bytes() as i64), - fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64), + fx.bcx.ins().iconst(fx.pointer_type, unit.align.bytes() as i64), ) } ty::Foreign(_) => { @@ -226,7 +229,7 @@ pub(crate) fn size_and_align_of<'tcx>( let unsized_offset_unadjusted = layout.fields.offset(i).bytes(); let unsized_offset_unadjusted = fx.bcx.ins().iconst(fx.pointer_type, unsized_offset_unadjusted as i64); - let sized_align = layout.align.abi.bytes(); + let sized_align = layout.align.bytes(); let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64); // Recurse to get the size of the dynamically sized field (must be diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 9d73f200afe2..9dcd4a33d44f 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -342,6 +342,14 @@ impl<'tcx> CValue<'tcx> { assert_eq!(self.layout().backend_repr, layout.backend_repr); CValue(self.0, layout) } + + pub(crate) fn cast_pat_ty_to_base(self, layout: TyAndLayout<'tcx>) -> Self { + let ty::Pat(base, _) = *self.layout().ty.kind() else { + panic!("not a pattern type: {:#?}", self.layout()) + }; + assert_eq!(layout.ty, base); + CValue(self.0, layout) + } } /// A place where you can write a value to or read a value from @@ -383,7 +391,7 @@ impl<'tcx> CPlace<'tcx> { let stack_slot = fx.create_stack_slot( u32::try_from(layout.size.bytes()).unwrap(), - u32::try_from(layout.align.abi.bytes()).unwrap(), + u32::try_from(layout.align.bytes()).unwrap(), ); CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout } } @@ -641,8 +649,8 @@ impl<'tcx> CPlace<'tcx> { let size = dst_layout.size.bytes(); // `emit_small_memory_copy` uses `u8` for alignments, just use the maximum // alignment that fits in a `u8` if the actual alignment is larger. - let src_align = src_layout.align.abi.bytes().try_into().unwrap_or(128); - let dst_align = dst_layout.align.abi.bytes().try_into().unwrap_or(128); + let src_align = src_layout.align.bytes().try_into().unwrap_or(128); + let dst_align = dst_layout.align.bytes().try_into().unwrap_or(128); fx.bcx.emit_small_memory_copy( fx.target_config, to_addr, @@ -660,7 +668,7 @@ impl<'tcx> CPlace<'tcx> { } } - /// Used for `ProjectionElem::Subtype`, `ty` has to be monomorphized before + /// Used for `ProjectionElem::UnwrapUnsafeBinder`, `ty` has to be monomorphized before /// passed on. pub(crate) fn place_transmute_type( self, @@ -909,8 +917,7 @@ pub(crate) fn assert_assignable<'tcx>( ); // fn(&T) -> for<'l> fn(&'l T) is allowed } - (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => { - // FIXME(dyn-star): Do the right thing with DynKinds + (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { for (from, to) in from_traits.iter().zip(to_traits) { let from = fx.tcx.normalize_erasing_late_bound_regions(fx.typing_env(), from); let to = fx.tcx.normalize_erasing_late_bound_regions(fx.typing_env(), to); diff --git a/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml b/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml index 759d0d59e268..e49c62d6c931 100644 --- a/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml +++ b/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml @@ -82,20 +82,16 @@ jobs: - name: Build sample project with target defined as JSON spec run: | ./y.sh prepare --only-libcore --cross - ./y.sh build --sysroot --features compiler-builtins-no-f16-f128 --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json + ./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json CG_RUSTFLAGS="-Clinker=m68k-unknown-linux-gnu-gcc" ./y.sh cargo build --manifest-path=./tests/hello-world/Cargo.toml --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json ./y.sh clean all - name: Build run: | ./y.sh prepare --only-libcore --cross - ./y.sh build --sysroot --features compiler-builtins-no-f16-f128 --target-triple m68k-unknown-linux-gnu + ./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu ./y.sh test --mini-tests --target-triple m68k-unknown-linux-gnu - # FIXME: since https://github.com/rust-lang/rust/pull/140809, we cannot run programs for architectures not - # supported by the object crate, since this adds a dependency on symbols.o for the panic runtime. - # And as such, a wrong order of the object files in the linker command now fails with an undefined reference - # to some symbols like __rustc::rust_panic. - #CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu ./y.sh test --cargo-tests --target-triple m68k-unknown-linux-gnu + CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu ./y.sh test --cargo-tests --target-triple m68k-unknown-linux-gnu ./y.sh clean all - name: Prepare dependencies @@ -104,23 +100,21 @@ jobs: git config --global user.name "User" ./y.sh prepare --cross - # FIXME: We cannot run programs for architectures not supported by the object crate. See comment above. - #- name: Run tests - #run: | - #./y.sh test --target-triple m68k-unknown-linux-gnu --release --clean --build-sysroot --sysroot-features compiler-builtins-no-f16-f128 ${{ matrix.commands }} + - name: Run tests + run: | + ./y.sh test --target-triple m68k-unknown-linux-gnu --release --clean --build-sysroot ${{ matrix.commands }} - # FIXME: We cannot run programs for architectures not supported by the object crate. See comment above. - #- name: Run Hello World! - #run: | - #./y.sh build --target-triple m68k-unknown-linux-gnu + - name: Run Hello World! + run: | + ./y.sh build --target-triple m68k-unknown-linux-gnu - #vm_dir=$(pwd)/vm - #cd tests/hello-world - #CG_RUSTFLAGS="-Clinker=m68k-unknown-linux-gnu-gcc" ../../y.sh cargo build --target m68k-unknown-linux-gnu - #sudo cp target/m68k-unknown-linux-gnu/debug/hello_world $vm_dir/home/ - #sudo chroot $vm_dir qemu-m68k-static /home/hello_world > hello_world_stdout - #expected_output="40" - #test $(cat hello_world_stdout) == $expected_output || (echo "Output differs. Actual output: $(cat hello_world_stdout)"; exit 1) + vm_dir=$(pwd)/vm + cd tests/hello-world + CG_RUSTFLAGS="-Clinker=m68k-unknown-linux-gnu-gcc" ../../y.sh cargo build --target m68k-unknown-linux-gnu + sudo cp target/m68k-unknown-linux-gnu/debug/hello_world $vm_dir/home/ + sudo chroot $vm_dir qemu-m68k-static /home/hello_world > hello_world_stdout + expected_output="40" + test $(cat hello_world_stdout) == $expected_output || (echo "Output differs. Actual output: $(cat hello_world_stdout)"; exit 1) # Summary job for the merge queue. # ALL THE PREVIOUS JOBS NEED TO BE ADDED TO THE `needs` SECTION OF THIS JOB! diff --git a/compiler/rustc_codegen_gcc/Cargo.lock b/compiler/rustc_codegen_gcc/Cargo.lock index 7f35c1a80bda..a5b972baf98e 100644 --- a/compiler/rustc_codegen_gcc/Cargo.lock +++ b/compiler/rustc_codegen_gcc/Cargo.lock @@ -56,18 +56,18 @@ dependencies = [ [[package]] name = "gccjit" -version = "2.7.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae99a89184220d967dd300139f2d2ae7d52c1a69d632b24aacc57c54625254ce" +checksum = "4a0e310ef75f396cd11b2443b353d55376656ca92c13cba36f92b7aff346ac1a" dependencies = [ "gccjit_sys", ] [[package]] name = "gccjit_sys" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24edb7bfe2b7b27c6d09ed23eebfcab0b359c8fe978433f902943e6f127a0f1b" +checksum = "95ed7572b30cd32430294dde6fb70822d58e67c6846a548647e8739776a0125b" dependencies = [ "libc", ] diff --git a/compiler/rustc_codegen_gcc/Cargo.toml b/compiler/rustc_codegen_gcc/Cargo.toml index 193348d1ef60..6031933bd2d2 100644 --- a/compiler/rustc_codegen_gcc/Cargo.toml +++ b/compiler/rustc_codegen_gcc/Cargo.toml @@ -24,7 +24,7 @@ default = ["master"] [dependencies] object = { version = "0.37.0", default-features = false, features = ["std", "read"] } tempfile = "3.20" -gccjit = "2.7" +gccjit = "2.8" #gccjit = { git = "https://github.com/rust-lang/gccjit.rs" } # Local copy. diff --git a/compiler/rustc_codegen_gcc/build_system/src/abi_test.rs b/compiler/rustc_codegen_gcc/build_system/src/abi_test.rs index 3c1531be27a5..a85886d87f36 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/abi_test.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/abi_test.rs @@ -31,7 +31,7 @@ pub fn run() -> Result<(), String> { Some("clones/abi-cafe".as_ref()), true, ) - .map_err(|err| (format!("Git clone failed with message: {err:?}!")))?; + .map_err(|err| format!("Git clone failed with message: {err:?}!"))?; // Configure abi-cafe to use the exact same rustc version we use - this is crucial. // Otherwise, the concept of ABI compatibility becomes meanignless. std::fs::copy("rust-toolchain", "clones/abi-cafe/rust-toolchain") diff --git a/compiler/rustc_codegen_gcc/build_system/src/fmt.rs b/compiler/rustc_codegen_gcc/build_system/src/fmt.rs index 7e6594f50f93..91535f217e35 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/fmt.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/fmt.rs @@ -1,7 +1,7 @@ use std::ffi::OsStr; use std::path::Path; -use crate::utils::run_command_with_output; +use crate::utils::{run_command_with_output, walk_dir}; fn show_usage() { println!( @@ -32,5 +32,31 @@ pub fn run() -> Result<(), String> { if check { &[&"cargo", &"fmt", &"--check"] } else { &[&"cargo", &"fmt"] }; run_command_with_output(cmd, Some(Path::new(".")))?; - run_command_with_output(cmd, Some(Path::new("build_system"))) + run_command_with_output(cmd, Some(Path::new("build_system")))?; + + run_rustfmt_recursively("tests/run", check) +} + +fn run_rustfmt_recursively

(dir: P, check: bool) -> Result<(), String> +where + P: AsRef, +{ + walk_dir( + dir, + &mut |dir| run_rustfmt_recursively(dir, check), + &mut |file_path| { + if file_path.extension().filter(|ext| ext == &OsStr::new("rs")).is_some() { + let rustfmt_cmd: &[&dyn AsRef] = if check { + &[&"rustfmt", &"--check", &file_path] + } else { + &[&"rustfmt", &file_path] + }; + + run_command_with_output(rustfmt_cmd, Some(Path::new("."))) + } else { + Ok(()) + } + }, + true, + ) } diff --git a/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs b/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs index 453211366b31..9714ce29af90 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/fuzz.rs @@ -43,18 +43,18 @@ pub fn run() -> Result<(), String> { "--start" => { start = str::parse(&args.next().ok_or_else(|| "Fuzz start not provided!".to_string())?) - .map_err(|err| (format!("Fuzz start not a number {err:?}!")))?; + .map_err(|err| format!("Fuzz start not a number {err:?}!"))?; } "--count" => { count = str::parse(&args.next().ok_or_else(|| "Fuzz count not provided!".to_string())?) - .map_err(|err| (format!("Fuzz count not a number {err:?}!")))?; + .map_err(|err| format!("Fuzz count not a number {err:?}!"))?; } "-j" | "--jobs" => { threads = str::parse( &args.next().ok_or_else(|| "Fuzz thread count not provided!".to_string())?, ) - .map_err(|err| (format!("Fuzz thread count not a number {err:?}!")))?; + .map_err(|err| format!("Fuzz thread count not a number {err:?}!"))?; } _ => return Err(format!("Unknown option {arg}")), } @@ -66,7 +66,7 @@ pub fn run() -> Result<(), String> { Some("clones/rustlantis".as_ref()), true, ) - .map_err(|err| (format!("Git clone failed with message: {err:?}!")))?; + .map_err(|err| format!("Git clone failed with message: {err:?}!"))?; // Ensure that we are on the newest rustlantis commit. let cmd: &[&dyn AsRef] = &[&"git", &"pull", &"origin"]; diff --git a/compiler/rustc_codegen_gcc/build_system/src/test.rs b/compiler/rustc_codegen_gcc/build_system/src/test.rs index 2c8271c36a94..1823aa71f408 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/test.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/test.rs @@ -531,7 +531,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { r#"change-id = 115898 [rust] -codegen-backends = [] +codegen-backends = ["gcc"] deny-warnings = false verbose-tests = true @@ -1083,11 +1083,12 @@ where fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { test_rustc_inner(env, args, |_| Ok(false), false, "run-make")?; + test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo")?; test_rustc_inner(env, args, |_| Ok(false), false, "ui") } fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - let result1 = test_rustc_inner( + let run_make_result = test_rustc_inner( env, args, retain_files_callback("tests/failing-run-make-tests.txt", "run-make"), @@ -1095,7 +1096,15 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { "run-make", ); - let result2 = test_rustc_inner( + let run_make_cargo_result = test_rustc_inner( + env, + args, + retain_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"), + false, + "run-make", + ); + + let ui_result = test_rustc_inner( env, args, retain_files_callback("tests/failing-ui-tests.txt", "ui"), @@ -1103,7 +1112,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { "ui", ); - result1.and(result2) + run_make_result.and(run_make_cargo_result).and(ui_result) } fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { @@ -1120,6 +1129,13 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { remove_files_callback("tests/failing-run-make-tests.txt", "run-make"), false, "run-make", + )?; + test_rustc_inner( + env, + args, + remove_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"), + false, + "run-make-cargo", ) } diff --git a/compiler/rustc_codegen_gcc/libgccjit.version b/compiler/rustc_codegen_gcc/libgccjit.version index f62154968d3d..dc9a00128646 100644 --- a/compiler/rustc_codegen_gcc/libgccjit.version +++ b/compiler/rustc_codegen_gcc/libgccjit.version @@ -1 +1 @@ -04ce66d8c918de9273bd7101638ad8724edf5e21 +4e995bd73c4490edfe5080ec6014d63aa9abed5f diff --git a/compiler/rustc_codegen_gcc/messages.ftl b/compiler/rustc_codegen_gcc/messages.ftl index a70ac08f01ae..b9b77b7d18c6 100644 --- a/compiler/rustc_codegen_gcc/messages.ftl +++ b/compiler/rustc_codegen_gcc/messages.ftl @@ -4,3 +4,5 @@ codegen_gcc_unwinding_inline_asm = codegen_gcc_copy_bitcode = failed to copy bitcode to object file: {$err} codegen_gcc_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$gcc_err}) + +codegen_gcc_explicit_tail_calls_unsupported = explicit tail calls with the 'become' keyword are not implemented in the GCC backend diff --git a/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch b/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch index 9cc377850b9b..3a8c37a8b8d9 100644 --- a/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch +++ b/compiler/rustc_codegen_gcc/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch @@ -1,29 +1,28 @@ -From b8f3eed3053c9333b5dfbeaeb2a6a65a4b3156df Mon Sep 17 00:00:00 2001 -From: Antoni Boucher -Date: Tue, 29 Aug 2023 13:06:34 -0400 +From 190e26c9274b3c93a9ee3516b395590e6bd9213b Mon Sep 17 00:00:00 2001 +From: None +Date: Sun, 3 Aug 2025 19:54:56 -0400 Subject: [PATCH] Patch 0001-Add-stdarch-Cargo.toml-for-testing.patch --- - library/stdarch/Cargo.toml | 23 +++++++++++++++++++++++ - 1 file changed, 23 insertions(+) + library/stdarch/Cargo.toml | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) create mode 100644 library/stdarch/Cargo.toml diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml new file mode 100644 -index 0000000..4c63700 +index 0000000..bd6725c --- /dev/null +++ b/library/stdarch/Cargo.toml -@@ -0,0 +1,21 @@ +@@ -0,0 +1,20 @@ +[workspace] +resolver = "1" +members = [ -+ "crates/core_arch", -+ "crates/std_detect", -+ "crates/stdarch-gen-arm", ++ "crates/*", + #"examples/" +] +exclude = [ -+ "crates/wasm-assert-instr-tests" ++ "crates/wasm-assert-instr-tests", ++ "rust_programs", +] + +[profile.release] @@ -36,5 +35,5 @@ index 0000000..4c63700 +opt-level = 3 +incremental = true -- -2.42.0 +2.50.1 diff --git a/compiler/rustc_codegen_gcc/rust-toolchain b/compiler/rustc_codegen_gcc/rust-toolchain index 2fe8ec4647fa..04d33dfb116c 100644 --- a/compiler/rustc_codegen_gcc/rust-toolchain +++ b/compiler/rustc_codegen_gcc/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-07-04" +channel = "nightly-2025-08-25" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/compiler/rustc_codegen_gcc/src/allocator.rs b/compiler/rustc_codegen_gcc/src/allocator.rs index 0d8dc93274f9..647569694b04 100644 --- a/compiler/rustc_codegen_gcc/src/allocator.rs +++ b/compiler/rustc_codegen_gcc/src/allocator.rs @@ -2,8 +2,7 @@ use gccjit::FnAttribute; use gccjit::{Context, FunctionType, RValue, ToRValue, Type}; use rustc_ast::expand::allocator::{ - ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, - alloc_error_handler_name, default_fn_name, global_fn_name, + AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, default_fn_name, global_fn_name, }; use rustc_middle::bug; use rustc_middle::ty::TyCtxt; @@ -18,8 +17,7 @@ pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, mods: &mut GccContext, _module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) { let context = &mods.context; let usize = match tcx.sess.target.pointer_width { @@ -31,45 +29,35 @@ pub(crate) unsafe fn codegen( let i8 = context.new_type::(); let i8p = i8.make_pointer(); - if kind == AllocatorKind::Default { - for method in ALLOCATOR_METHODS { - let mut types = Vec::with_capacity(method.inputs.len()); - for input in method.inputs.iter() { - match input.ty { - AllocatorTy::Layout => { - types.push(usize); - types.push(usize); - } - AllocatorTy::Ptr => types.push(i8p), - AllocatorTy::Usize => types.push(usize), + for method in methods { + let mut types = Vec::with_capacity(method.inputs.len()); + for input in method.inputs.iter() { + match input.ty { + AllocatorTy::Layout => { + types.push(usize); + types.push(usize); + } + AllocatorTy::Ptr => types.push(i8p), + AllocatorTy::Usize => types.push(usize), - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + AllocatorTy::Never | AllocatorTy::ResultPtr | AllocatorTy::Unit => { + panic!("invalid allocator arg") } } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, - - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; - let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); - let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); - - create_wrapper_function(tcx, context, &from_name, Some(&to_name), &types, output); } - } + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Never | AllocatorTy::Unit => None, - // FIXME(bjorn3): Add noreturn attribute - create_wrapper_function( - tcx, - context, - &mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - Some(&mangle_internal_symbol(tcx, alloc_error_handler_name(alloc_error_handler_kind))), - &[usize, usize], - None, - ); + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); + let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); + + create_wrapper_function(tcx, context, &from_name, Some(&to_name), &types, output); + } create_const_value_function( tcx, @@ -99,11 +87,15 @@ fn create_const_value_function( let func = context.new_function(None, FunctionType::Exported, output, &[], name, false); #[cfg(feature = "master")] - func.add_attribute(FnAttribute::Visibility(symbol_visibility_to_gcc( - tcx.sess.default_visibility(), - ))); + { + func.add_attribute(FnAttribute::Visibility(symbol_visibility_to_gcc( + tcx.sess.default_visibility(), + ))); - func.add_attribute(FnAttribute::AlwaysInline); + // FIXME(antoyo): cg_llvm sets AlwaysInline, but AlwaysInline is different in GCC and using + // it here will causes linking errors when using LTO. + func.add_attribute(FnAttribute::Inline); + } if tcx.sess.must_emit_unwind_tables() { // TODO(antoyo): emit unwind tables. diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index 17e2e028b16f..f237861b1595 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -546,9 +546,16 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { } if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { - // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient - // on all architectures. For instance, what about FP stack? - extended_asm.add_clobber("cc"); + match asm_arch { + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => { + // "cc" is cr0 on powerpc. + } + // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient + // on all architectures. For instance, what about FP stack? + _ => { + extended_asm.add_clobber("cc"); + } + } } if !options.contains(InlineAsmOptions::NOMEM) { extended_asm.add_clobber("memory"); @@ -698,8 +705,13 @@ fn reg_class_to_gcc(reg_class: InlineAsmRegClass) -> &'static str { InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) - | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => "wa", + InlineAsmRegClass::PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -774,11 +786,15 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => { - cx.type_vector(cx.type_i32(), 4) - } - InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) - | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + InlineAsmRegClass::PowerPC( + PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg, + ) => cx.type_vector(cx.type_i32(), 4), + InlineAsmRegClass::PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -949,6 +965,13 @@ fn modifier_to_gcc( InlineAsmRegClass::LoongArch(_) => None, InlineAsmRegClass::Mips(_) => None, InlineAsmRegClass::Nvptx(_) => None, + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => { + if modifier.is_none() { + Some('x') + } else { + modifier + } + } InlineAsmRegClass::PowerPC(_) => None, InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, diff --git a/compiler/rustc_codegen_gcc/src/attributes.rs b/compiler/rustc_codegen_gcc/src/attributes.rs index 7a1ae6ca9c8b..04b43bb8bb7c 100644 --- a/compiler/rustc_codegen_gcc/src/attributes.rs +++ b/compiler/rustc_codegen_gcc/src/attributes.rs @@ -2,8 +2,8 @@ use gccjit::FnAttribute; use gccjit::Function; #[cfg(feature = "master")] -use rustc_attr_data_structures::InlineAttr; -use rustc_attr_data_structures::InstructionSetAttr; +use rustc_hir::attrs::InlineAttr; +use rustc_hir::attrs::InstructionSetAttr; #[cfg(feature = "master")] use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; #[cfg(feature = "master")] diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs index d558dfbc1c45..598bbe74007d 100644 --- a/compiler/rustc_codegen_gcc/src/back/lto.rs +++ b/compiler/rustc_codegen_gcc/src/back/lto.rs @@ -29,7 +29,8 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput}; use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; +use rustc_log::tracing::info; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::Lto; @@ -51,12 +52,11 @@ fn prepare_lto( cgcx: &CodegenContext, each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, -) -> Result { +) -> LtoData { let tmp_path = match tempdir() { Ok(tmp_path) => tmp_path, Err(error) => { - eprintln!("Cannot create temporary directory: {}", error); - return Err(FatalError); + dcx.fatal(format!("Cannot create temporary directory: {}", error)); } }; @@ -91,15 +91,14 @@ fn prepare_lto( upstream_modules.push((module, CString::new(name).unwrap())); } Err(e) => { - dcx.emit_err(e); - return Err(FatalError); + dcx.emit_fatal(e); } } } } } - Ok(LtoData { upstream_modules, tmp_path }) + LtoData { upstream_modules, tmp_path } } fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> { @@ -114,10 +113,10 @@ pub(crate) fn run_fat( cgcx: &CodegenContext, each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?; + let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); /*let symbols_below_threshold = lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>();*/ fat_lto( @@ -137,7 +136,7 @@ fn fat_lto( mut serialized_modules: Vec<(SerializedModule, CString)>, tmp_path: TempDir, //symbols_below_threshold: &[String], -) -> Result, FatalError> { +) -> ModuleCodegen { let _timer = cgcx.prof.generic_activity("GCC_fat_lto_build_monolithic_module"); info!("going for a fat lto"); @@ -206,7 +205,7 @@ fn fat_lto( let path = tmp_path.path().to_path_buf().join(&module.name); let path = path.to_str().expect("path"); let context = &module.module_llvm.context; - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // NOTE: we need to set the optimization level here in order for LTO to do its job. context.set_optimization_level(to_gcc_opt_level(config.opt_level)); context.add_command_line_option("-flto=auto"); @@ -261,7 +260,7 @@ fn fat_lto( // of now. module.module_llvm.temp_dir = Some(tmp_path); - Ok(module) + module } pub struct ModuleBuffer(PathBuf); @@ -286,10 +285,10 @@ pub(crate) fn run_thin( each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?; + let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx); if cgcx.opts.cg.linker_plugin_lto.enabled() { unreachable!( "We should never reach this case if the LTO step \ @@ -307,12 +306,9 @@ pub(crate) fn run_thin( ) } -pub(crate) fn prepare_thin( - module: ModuleCodegen, - _emit_summary: bool, -) -> (String, ThinBuffer) { +pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { let name = module.name; - //let buffer = ThinBuffer::new(module.module_llvm.context, true, emit_summary); + //let buffer = ThinBuffer::new(module.module_llvm.context, true); let buffer = ThinBuffer::new(&module.module_llvm.context); (name, buffer) } @@ -355,7 +351,7 @@ fn thin_lto( tmp_path: TempDir, cached_modules: Vec<(SerializedModule, WorkProduct)>, //_symbols_below_threshold: &[String], -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); info!("going for that thin, thin LTO"); @@ -518,13 +514,13 @@ fn thin_lto( // TODO: save the directory so that it gets deleted later. std::mem::forget(tmp_path); - Ok((opt_jobs, copy_jobs)) + (opt_jobs, copy_jobs) } pub fn optimize_thin_module( thin_module: ThinModule, _cgcx: &CodegenContext, -) -> Result, FatalError> { +) -> ModuleCodegen { //let dcx = cgcx.create_dcx(); //let module_name = &thin_module.shared.module_names[thin_module.idx]; @@ -634,7 +630,8 @@ pub fn optimize_thin_module( save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } }*/ - Ok(module) + #[allow(clippy::let_and_return)] + module } pub struct ThinBuffer { @@ -651,10 +648,6 @@ impl ThinBufferMethods for ThinBuffer { fn data(&self) -> &[u8] { &[] } - - fn thin_link_data(&self) -> &[u8] { - unimplemented!(); - } } pub struct ThinData; //(Arc); diff --git a/compiler/rustc_codegen_gcc/src/back/write.rs b/compiler/rustc_codegen_gcc/src/back/write.rs index c1231142c658..8ba730e32502 100644 --- a/compiler/rustc_codegen_gcc/src/back/write.rs +++ b/compiler/rustc_codegen_gcc/src/back/write.rs @@ -5,8 +5,8 @@ use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, ModuleConfig}; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; use rustc_fs_util::link_or_copy; +use rustc_log::tracing::debug; use rustc_session::config::OutputType; -use rustc_span::fatal_error::FatalError; use rustc_target::spec::SplitDebuginfo; use crate::base::add_pic_option; @@ -17,7 +17,7 @@ pub(crate) fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, -) -> Result { +) -> CompiledModule { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -246,7 +246,7 @@ pub(crate) fn codegen( } } - Ok(module.into_compiled_module( + module.into_compiled_module( config.emit_obj != EmitObj::None, cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo == SplitDebuginfo::Unpacked, config.emit_bc, @@ -254,7 +254,7 @@ pub(crate) fn codegen( config.emit_ir, &cgcx.output_filenames, cgcx.invocation_temp.as_deref(), - )) + ) } pub(crate) fn save_temp_bitcode( diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs index c105916bbb2b..e8672f49580b 100644 --- a/compiler/rustc_codegen_gcc/src/base.rs +++ b/compiler/rustc_codegen_gcc/src/base.rs @@ -8,16 +8,16 @@ use rustc_codegen_ssa::ModuleCodegen; use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::DebugInfoCodegenMethods; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; -use rustc_middle::mir::mono::Linkage; #[cfg(feature = "master")] use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; use rustc_session::config::DebugInfo; use rustc_span::Symbol; +use rustc_target::spec::RelocModel; #[cfg(feature = "master")] use rustc_target::spec::SymbolVisibility; -use rustc_target::spec::{PanicStrategy, RelocModel}; use crate::builder::Builder; use crate::context::CodegenCx; @@ -101,7 +101,7 @@ pub fn compile_codegen_unit( // Instantiate monomorphizations without filling out definitions yet... let context = new_context(tcx); - if tcx.sess.panic_strategy() == PanicStrategy::Unwind { + if tcx.sess.panic_strategy().unwinds() { context.add_command_line_option("-fexceptions"); context.add_driver_option("-fexceptions"); } diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index a4ec4bf8deac..5657620879ca 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -34,6 +34,7 @@ use rustc_target::spec::{HasTargetSpec, HasX86AbiOpt, Target, X86Abi}; use crate::common::{SignType, TypeReflection, type_is_pointer}; use crate::context::CodegenCx; +use crate::errors; use crate::intrinsic::llvm; use crate::type_of::LayoutGccExt; @@ -539,9 +540,15 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn ret(&mut self, mut value: RValue<'gcc>) { let expected_return_type = self.current_func().get_return_type(); - if !expected_return_type.is_compatible_with(value.get_type()) { - // NOTE: due to opaque pointers now being used, we need to cast here. - value = self.context.new_cast(self.location, value, expected_return_type); + let value_type = value.get_type(); + if !expected_return_type.is_compatible_with(value_type) { + // NOTE: due to opaque pointers now being used, we need to (bit)cast here. + if self.is_native_int_type(value_type) && self.is_native_int_type(expected_return_type) + { + value = self.context.new_cast(self.location, value, expected_return_type); + } else { + value = self.context.new_bitcast(self.location, value, expected_return_type); + } } self.llbb().end_with_return(self.location, value); } @@ -1278,11 +1285,19 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn intcast( &mut self, - value: RValue<'gcc>, + mut value: RValue<'gcc>, dest_typ: Type<'gcc>, - _is_signed: bool, + is_signed: bool, ) -> RValue<'gcc> { - // NOTE: is_signed is for value, not dest_typ. + let value_type = value.get_type(); + if is_signed && !value_type.is_signed(self.cx) { + let signed_type = value_type.to_signed(self.cx); + value = self.gcc_int_cast(value, signed_type); + } else if !is_signed && value_type.is_signed(self.cx) { + let unsigned_type = value_type.to_unsigned(self.cx); + value = self.gcc_int_cast(value, unsigned_type); + } + self.gcc_int_cast(value, dest_typ) } @@ -1368,6 +1383,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _src_align: Align, size: RValue<'gcc>, flags: MemFlags, + _tt: Option, // Autodiff TypeTrees are LLVM-only, ignored in GCC backend ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_size_t(), false); @@ -1656,6 +1672,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { dst: RValue<'gcc>, src: RValue<'gcc>, order: AtomicOrdering, + ret_ptr: bool, ) -> RValue<'gcc> { let size = get_maybe_pointer_size(src); let name = match op { @@ -1683,6 +1700,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let atomic_function = self.context.get_builtin_function(name); let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc()); + // FIXME: If `ret_ptr` is true and `src` is an integer, we should really tell GCC + // that this is a pointer operation that needs to preserve provenance -- but like LLVM, + // GCC does not currently seems to support that. let void_ptr_type = self.context.new_type::<*mut ()>(); let volatile_void_ptr_type = void_ptr_type.make_volatile(); let dst = self.context.new_cast(self.location, dst, volatile_void_ptr_type); @@ -1690,7 +1710,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let new_src_type = atomic_function.get_param(1).to_rvalue().get_type(); let src = self.context.new_bitcast(self.location, src, new_src_type); let res = self.context.new_call(self.location, atomic_function, &[dst, src, order]); - self.context.new_cast(self.location, res, src.get_type()) + let res_type = if ret_ptr { void_ptr_type } else { src.get_type() }; + self.context.new_cast(self.location, res, res_type) } fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope) { @@ -1742,6 +1763,20 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { call } + fn tail_call( + &mut self, + _llty: Self::Type, + _fn_attrs: Option<&CodegenFnAttrs>, + _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + _llfn: Self::Value, + _args: &[Self::Value], + _funclet: Option<&Self::Funclet>, + _instance: Option>, + ) { + // FIXME: implement support for explicit tail calls like rustc_codegen_llvm. + self.tcx.dcx().emit_fatal(errors::ExplicitTailCallsUnsupported); + } + fn zext(&mut self, value: RValue<'gcc>, dest_typ: Type<'gcc>) -> RValue<'gcc> { // FIXME(antoyo): this does not zero-extend. self.gcc_int_cast(value, dest_typ) diff --git a/compiler/rustc_codegen_gcc/src/callee.rs b/compiler/rustc_codegen_gcc/src/callee.rs index e7ca95af594c..8487a85bd035 100644 --- a/compiler/rustc_codegen_gcc/src/callee.rs +++ b/compiler/rustc_codegen_gcc/src/callee.rs @@ -106,7 +106,7 @@ pub fn get_fn<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, instance: Instance<'tcx>) // This is a monomorphization of a generic function. if !(cx.tcx.sess.opts.share_generics() || tcx.codegen_instance_attrs(instance.def).inline - == rustc_attr_data_structures::InlineAttr::Never) + == rustc_hir::attrs::InlineAttr::Never) { // When not sharing generics, all instances are in the same // crate and have hidden visibility. diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs index 28848ca61845..7c2969e58718 100644 --- a/compiler/rustc_codegen_gcc/src/common.rs +++ b/compiler/rustc_codegen_gcc/src/common.rs @@ -5,7 +5,7 @@ use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, MiscCodegenMethods, StaticCodegenMethods, }; use rustc_middle::mir::Mutability; -use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, PointerArithmetic, Scalar}; use rustc_middle::ty::layout::LayoutOf; use crate::context::CodegenCx; @@ -247,8 +247,8 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> { // This avoids generating a zero-sized constant value and actually needing a // real address at runtime. if alloc.inner().len() == 0 { - assert_eq!(offset.bytes(), 0); - let val = self.const_usize(alloc.inner().align.bytes()); + let val = alloc.inner().align.bytes().wrapping_add(offset.bytes()); + let val = self.const_usize(self.tcx.truncate_to_target_usize(val)); return if matches!(layout.primitive(), Pointer(_)) { self.context.new_cast(None, val, ty) } else { diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs index c04c75e1b11f..ec7d4b285a3f 100644 --- a/compiler/rustc_codegen_gcc/src/consts.rs +++ b/compiler/rustc_codegen_gcc/src/consts.rs @@ -5,12 +5,14 @@ use rustc_abi::{self as abi, Align, HasDataLayout, Primitive, Size, WrappingRang use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, }; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_log::tracing::trace; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::Linkage; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; @@ -80,6 +82,8 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> { if global.to_rvalue().get_type() != val_llty { global.to_rvalue().set_type(val_llty); } + + // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, global, alloc.align); global.global_set_initializer_rvalue(value); @@ -384,8 +388,8 @@ fn check_and_apply_linkage<'gcc, 'tcx>( // linkage and there are no definitions), then // `extern_with_linkage_foo` will instead be initialized to // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); + let real_name = + format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE)); let global2 = cx.define_global(&real_name, gcc_type, is_tls, attrs.link_section); // TODO(antoyo): set linkage. let value = cx.const_ptrcast(global1.get_address(None), gcc_type); diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 665cf22ddbae..c9ae96777de4 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -147,7 +147,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let layout = tcx .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(rust_type)) .unwrap(); - let align = layout.align.abi.bytes(); + let align = layout.align.bytes(); // For types with size 1, the alignment can be 1 and only 1 // So, we can skip the call to ``get_aligned`. // In the future, we can add a GCC API to query the type align, @@ -186,9 +186,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { (i128_type, u128_type) } else { /*let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.i128)).unwrap(); - let i128_align = layout.align.abi.bytes(); + let i128_align = layout.align.bytes(); let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.u128)).unwrap(); - let u128_align = layout.align.abi.bytes();*/ + let u128_align = layout.align.bytes();*/ // TODO(antoyo): re-enable the alignment when libgccjit fixed the issue in // gcc_jit_context_new_array_constructor (it should not use reinterpret_cast). @@ -529,7 +529,10 @@ impl<'gcc, 'tcx> HasX86AbiOpt for CodegenCx<'gcc, 'tcx> { impl<'gcc, 'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::InvalidSimd { .. } + | LayoutError::ReferencesError(_) = err + { self.tcx.dcx().emit_fatal(respan(span, err.into_diagnostic())) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -545,7 +548,9 @@ impl<'gcc, 'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'gcc, 'tcx> { span: Span, fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { - if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::InvalidSimd { .. }) = + err + { self.tcx.dcx().emit_fatal(respan(span, err)) } else { match fn_abi_request { diff --git a/compiler/rustc_codegen_gcc/src/debuginfo.rs b/compiler/rustc_codegen_gcc/src/debuginfo.rs index 4c8585192a1b..0f015cc23f52 100644 --- a/compiler/rustc_codegen_gcc/src/debuginfo.rs +++ b/compiler/rustc_codegen_gcc/src/debuginfo.rs @@ -29,13 +29,24 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> { _variable_alloca: Self::Value, _direct_offset: Size, _indirect_offsets: &[Size], - _fragment: Option>, + _fragment: &Option>, ) { // FIXME(tempdragon): Not sure if this is correct, probably wrong but still keep it here. #[cfg(feature = "master")] _variable_alloca.set_location(_dbg_loc); } + fn dbg_var_value( + &mut self, + _dbg_var: Self::DIVariable, + _dbg_loc: Self::DILocation, + _value: Self::Value, + _direct_offset: Size, + _indirect_offsets: &[Size], + _fragment: &Option>, + ) { + } + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { // TODO(antoyo): insert reference to gdb debug scripts section global. } diff --git a/compiler/rustc_codegen_gcc/src/errors.rs b/compiler/rustc_codegen_gcc/src/errors.rs index 0aa16bd88b43..b252c39c0c05 100644 --- a/compiler/rustc_codegen_gcc/src/errors.rs +++ b/compiler/rustc_codegen_gcc/src/errors.rs @@ -19,3 +19,7 @@ pub(crate) struct CopyBitcode { pub(crate) struct LtoBitcodeFromRlib { pub gcc_err: String, } + +#[derive(Diagnostic)] +#[diag(codegen_gcc_explicit_tail_calls_unsupported)] +pub(crate) struct ExplicitTailCallsUnsupported; diff --git a/compiler/rustc_codegen_gcc/src/gcc_util.rs b/compiler/rustc_codegen_gcc/src/gcc_util.rs index 42ba40692b75..702704ddf136 100644 --- a/compiler/rustc_codegen_gcc/src/gcc_util.rs +++ b/compiler/rustc_codegen_gcc/src/gcc_util.rs @@ -1,8 +1,8 @@ #[cfg(feature = "master")] use gccjit::Context; use rustc_codegen_ssa::target_features; +use rustc_data_structures::smallvec::{SmallVec, smallvec}; use rustc_session::Session; -use smallvec::{SmallVec, smallvec}; fn gcc_features_by_flags(sess: &Session, features: &mut Vec) { target_features::retpoline_features_by_flags(sess, features); diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs index 6f21ce9352b5..9fb7f6bad684 100644 --- a/compiler/rustc_codegen_gcc/src/int.rs +++ b/compiler/rustc_codegen_gcc/src/int.rs @@ -4,12 +4,15 @@ // cSpell:words cmpti divti modti mulodi muloti udivti umodti -use gccjit::{BinaryOp, ComparisonOp, FunctionType, Location, RValue, ToRValue, Type, UnaryOp}; +use gccjit::{ + BinaryOp, CType, ComparisonOp, FunctionType, Location, RValue, ToRValue, Type, UnaryOp, +}; use rustc_abi::{CanonAbi, Endian, ExternAbi}; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeCodegenMethods, BuilderMethods, OverflowOp}; use rustc_middle::ty::{self, Ty}; use rustc_target::callconv::{ArgAbi, ArgAttributes, FnAbi, PassMode}; +use rustc_type_ir::{Interner, TyKind}; use crate::builder::{Builder, ToGccComp}; use crate::common::{SignType, TypeReflection}; @@ -167,9 +170,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { if a_type.is_vector() { // Vector types need to be bitcast. // TODO(antoyo): perhaps use __builtin_convertvector for vector casting. - b = self.context.new_bitcast(self.location, b, a.get_type()); + b = self.context.new_bitcast(self.location, b, a_type); } else { - b = self.context.new_cast(self.location, b, a.get_type()); + b = self.context.new_cast(self.location, b, a_type); } } self.context.new_binary_op(self.location, operation, a_type, a, b) @@ -216,13 +219,22 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { operation_name: &str, signed: bool, a: RValue<'gcc>, - b: RValue<'gcc>, + mut b: RValue<'gcc>, ) -> RValue<'gcc> { let a_type = a.get_type(); let b_type = b.get_type(); if (self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type)) || (a_type.is_vector() && b_type.is_vector()) { + if !a_type.is_compatible_with(b_type) { + if a_type.is_vector() { + // Vector types need to be bitcast. + // TODO(antoyo): perhaps use __builtin_convertvector for vector casting. + b = self.context.new_bitcast(self.location, b, a_type); + } else { + b = self.context.new_cast(self.location, b, a_type); + } + } self.context.new_binary_op(self.location, operation, a_type, a, b) } else { debug_assert!(a_type.dyncast_array().is_some()); @@ -351,6 +363,11 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { // TODO(antoyo): is it correct to use rhs type instead of the parameter typ? .new_local(self.location, rhs.get_type(), "binopResult") .get_address(self.location); + let new_type = type_kind_to_gcc_type(new_kind); + let new_type = self.context.new_c_type(new_type); + let lhs = self.context.new_cast(self.location, lhs, new_type); + let rhs = self.context.new_cast(self.location, rhs, new_type); + let res = self.context.new_cast(self.location, res, new_type.make_pointer()); let overflow = self.overflow_call(intrinsic, &[lhs, rhs, res], None); (res.dereference(self.location).to_rvalue(), overflow) } @@ -477,11 +494,27 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let lhs_low = self.context.new_cast(self.location, self.low(lhs), unsigned_type); let rhs_low = self.context.new_cast(self.location, self.low(rhs), unsigned_type); + let mut lhs_high = self.high(lhs); + let mut rhs_high = self.high(rhs); + + match op { + IntPredicate::IntUGT + | IntPredicate::IntUGE + | IntPredicate::IntULT + | IntPredicate::IntULE => { + lhs_high = self.context.new_cast(self.location, lhs_high, unsigned_type); + rhs_high = self.context.new_cast(self.location, rhs_high, unsigned_type); + } + // TODO(antoyo): we probably need to handle signed comparison for unsigned + // integers. + _ => (), + } + let condition = self.context.new_comparison( self.location, ComparisonOp::LessThan, - self.high(lhs), - self.high(rhs), + lhs_high, + rhs_high, ); self.llbb().end_with_conditional(self.location, condition, block1, block2); @@ -495,8 +528,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { let condition = self.context.new_comparison( self.location, ComparisonOp::GreaterThan, - self.high(lhs), - self.high(rhs), + lhs_high, + rhs_high, ); block2.end_with_conditional(self.location, condition, block3, block4); @@ -620,7 +653,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { } } - pub fn gcc_xor(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> { + pub fn gcc_xor(&self, a: RValue<'gcc>, mut b: RValue<'gcc>) -> RValue<'gcc> { let a_type = a.get_type(); let b_type = b.get_type(); if a_type.is_vector() && b_type.is_vector() { @@ -628,6 +661,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { a ^ b } else if self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type) { + if !a_type.is_compatible_with(b_type) { + b = self.context.new_cast(self.location, b, a_type); + } a ^ b } else { self.concat_low_high_rvalues( @@ -1042,3 +1078,25 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { self.context.new_array_constructor(None, typ, &values) } } + +fn type_kind_to_gcc_type(kind: TyKind) -> CType { + use rustc_middle::ty::IntTy::*; + use rustc_middle::ty::UintTy::*; + use rustc_middle::ty::{Int, Uint}; + + match kind { + Int(I8) => CType::Int8t, + Int(I16) => CType::Int16t, + Int(I32) => CType::Int32t, + Int(I64) => CType::Int64t, + Int(I128) => CType::Int128t, + + Uint(U8) => CType::UInt8t, + Uint(U16) => CType::UInt16t, + Uint(U32) => CType::UInt32t, + Uint(U64) => CType::UInt64t, + Uint(U128) => CType::UInt128t, + + _ => unimplemented!("Kind: {:?}", kind), + } +} diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs index 915ed875e32f..d1b2a93243d2 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs @@ -95,8 +95,11 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "cubema" => "__builtin_amdgcn_cubema", "cubesc" => "__builtin_amdgcn_cubesc", "cubetc" => "__builtin_amdgcn_cubetc", + "cvt.f16.bf8" => "__builtin_amdgcn_cvt_f16_bf8", + "cvt.f16.fp8" => "__builtin_amdgcn_cvt_f16_fp8", "cvt.f32.bf8" => "__builtin_amdgcn_cvt_f32_bf8", "cvt.f32.fp8" => "__builtin_amdgcn_cvt_f32_fp8", + "cvt.f32.fp8.e5m3" => "__builtin_amdgcn_cvt_f32_fp8_e5m3", "cvt.off.f32.i4" => "__builtin_amdgcn_cvt_off_f32_i4", "cvt.pk.bf8.f32" => "__builtin_amdgcn_cvt_pk_bf8_f32", "cvt.pk.f16.bf8" => "__builtin_amdgcn_cvt_pk_f16_bf8", @@ -181,6 +184,12 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "dot4.f32.fp8.bf8" => "__builtin_amdgcn_dot4_f32_fp8_bf8", "dot4.f32.fp8.fp8" => "__builtin_amdgcn_dot4_f32_fp8_fp8", "ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn", + "ds.atomic.async.barrier.arrive.b64" => { + "__builtin_amdgcn_ds_atomic_async_barrier_arrive_b64" + } + "ds.atomic.barrier.arrive.rtn.b64" => { + "__builtin_amdgcn_ds_atomic_barrier_arrive_rtn_b64" + } "ds.bpermute" => "__builtin_amdgcn_ds_bpermute", "ds.bpermute.fi.b32" => "__builtin_amdgcn_ds_bpermute_fi_b32", "ds.gws.barrier" => "__builtin_amdgcn_ds_gws_barrier", @@ -198,8 +207,32 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "fdot2.f16.f16" => "__builtin_amdgcn_fdot2_f16_f16", "fdot2.f32.bf16" => "__builtin_amdgcn_fdot2_f32_bf16", "fdot2c.f32.bf16" => "__builtin_amdgcn_fdot2c_f32_bf16", + "flat.prefetch" => "__builtin_amdgcn_flat_prefetch", "fmul.legacy" => "__builtin_amdgcn_fmul_legacy", + "global.load.async.to.lds.b128" => { + "__builtin_amdgcn_global_load_async_to_lds_b128" + } + "global.load.async.to.lds.b32" => { + "__builtin_amdgcn_global_load_async_to_lds_b32" + } + "global.load.async.to.lds.b64" => { + "__builtin_amdgcn_global_load_async_to_lds_b64" + } + "global.load.async.to.lds.b8" => "__builtin_amdgcn_global_load_async_to_lds_b8", "global.load.lds" => "__builtin_amdgcn_global_load_lds", + "global.prefetch" => "__builtin_amdgcn_global_prefetch", + "global.store.async.from.lds.b128" => { + "__builtin_amdgcn_global_store_async_from_lds_b128" + } + "global.store.async.from.lds.b32" => { + "__builtin_amdgcn_global_store_async_from_lds_b32" + } + "global.store.async.from.lds.b64" => { + "__builtin_amdgcn_global_store_async_from_lds_b64" + } + "global.store.async.from.lds.b8" => { + "__builtin_amdgcn_global_store_async_from_lds_b8" + } "groupstaticsize" => "__builtin_amdgcn_groupstaticsize", "iglp.opt" => "__builtin_amdgcn_iglp_opt", "implicit.buffer.ptr" => "__builtin_amdgcn_implicit_buffer_ptr", @@ -291,6 +324,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "s.incperflevel" => "__builtin_amdgcn_s_incperflevel", "s.memrealtime" => "__builtin_amdgcn_s_memrealtime", "s.memtime" => "__builtin_amdgcn_s_memtime", + "s.monitor.sleep" => "__builtin_amdgcn_s_monitor_sleep", "s.sendmsg" => "__builtin_amdgcn_s_sendmsg", "s.sendmsghalt" => "__builtin_amdgcn_s_sendmsghalt", "s.setprio" => "__builtin_amdgcn_s_setprio", @@ -300,11 +334,15 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "s.sleep.var" => "__builtin_amdgcn_s_sleep_var", "s.ttracedata" => "__builtin_amdgcn_s_ttracedata", "s.ttracedata.imm" => "__builtin_amdgcn_s_ttracedata_imm", + "s.wait.asynccnt" => "__builtin_amdgcn_s_wait_asynccnt", "s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready", + "s.wait.tensorcnt" => "__builtin_amdgcn_s_wait_tensorcnt", "s.waitcnt" => "__builtin_amdgcn_s_waitcnt", "sad.hi.u8" => "__builtin_amdgcn_sad_hi_u8", "sad.u16" => "__builtin_amdgcn_sad_u16", "sad.u8" => "__builtin_amdgcn_sad_u8", + "sat.pk4.i4.i8" => "__builtin_amdgcn_sat_pk4_i4_i8", + "sat.pk4.u4.u8" => "__builtin_amdgcn_sat_pk4_u4_u8", "sched.barrier" => "__builtin_amdgcn_sched_barrier", "sched.group.barrier" => "__builtin_amdgcn_sched_group_barrier", "sdot2" => "__builtin_amdgcn_sdot2", @@ -346,8 +384,13 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "smfmac.i32.16x16x64.i8" => "__builtin_amdgcn_smfmac_i32_16x16x64_i8", "smfmac.i32.32x32x32.i8" => "__builtin_amdgcn_smfmac_i32_32x32x32_i8", "smfmac.i32.32x32x64.i8" => "__builtin_amdgcn_smfmac_i32_32x32x64_i8", + "struct.ptr.buffer.load.lds" => "__builtin_amdgcn_struct_ptr_buffer_load_lds", "sudot4" => "__builtin_amdgcn_sudot4", "sudot8" => "__builtin_amdgcn_sudot8", + "tensor.load.to.lds" => "__builtin_amdgcn_tensor_load_to_lds", + "tensor.load.to.lds.d2" => "__builtin_amdgcn_tensor_load_to_lds_d2", + "tensor.store.from.lds" => "__builtin_amdgcn_tensor_store_from_lds", + "tensor.store.from.lds.d2" => "__builtin_amdgcn_tensor_store_from_lds_d2", "udot2" => "__builtin_amdgcn_udot2", "udot4" => "__builtin_amdgcn_udot4", "udot8" => "__builtin_amdgcn_udot8", @@ -6326,6 +6369,23 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { } s390(name, full_name) } + "spv" => { + #[allow(non_snake_case)] + fn spv(name: &str, full_name: &str) -> &'static str { + match name { + // spv + "num.subgroups" => "__builtin_spirv_num_subgroups", + "subgroup.id" => "__builtin_spirv_subgroup_id", + "subgroup.local.invocation.id" => { + "__builtin_spirv_subgroup_local_invocation_id" + } + "subgroup.max.size" => "__builtin_spirv_subgroup_max_size", + "subgroup.size" => "__builtin_spirv_subgroup_size", + _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"), + } + } + spv(name, full_name) + } "ve" => { #[allow(non_snake_case)] fn ve(name: &str, full_name: &str) -> &'static str { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 0753ac1aeb84..99a4f9b9f7e7 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -29,7 +29,6 @@ use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance, Ty}; use rustc_span::{Span, Symbol, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; -use rustc_target::spec::PanicStrategy; #[cfg(feature = "master")] use crate::abi::FnAbiGccExt; @@ -730,7 +729,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { if self.is_sized_indirect() { OperandValue::Ref(PlaceValue::new_sized(val, self.layout.align.abi)).store(bx, dst) } else if self.is_unsized_indirect() { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + bug!("unsized `ArgAbi` cannot be stored"); } else if let PassMode::Cast { ref cast, .. } = self.mode { // FIXME(eddyb): Figure out when the simpler Store is safe, clang // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. @@ -771,6 +770,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(self.layout.size.bytes()), MemFlags::empty(), + None, ); bx.lifetime_end(scratch, scratch_size); @@ -797,12 +797,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Pair(next(), next()).store(bx, dst); } PassMode::Indirect { meta_attrs: Some(_), .. } => { - let place_val = PlaceValue { - llval: next(), - llextra: Some(next()), - align: self.layout.align.abi, - }; - OperandValue::Ref(place_val).store(bx, dst); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) | PassMode::Indirect { meta_attrs: None, .. } @@ -925,10 +920,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { // TODO(antoyo): use width? let arg_type = arg.get_type(); let result_type = self.u32_type; + let arg = if arg_type.is_signed(self.cx) { + let new_type = arg_type.to_unsigned(self.cx); + self.gcc_int_cast(arg, new_type) + } else { + arg + }; + let arg_type = arg.get_type(); let count_leading_zeroes = // TODO(antoyo): write a new function Type::is_compatible_with(&Type) and use it here // instead of using is_uint(). - if arg_type.is_uint(self.cx) { + if arg_type.is_uchar(self.cx) || arg_type.is_ushort(self.cx) || arg_type.is_uint(self.cx) { "__builtin_clz" } else if arg_type.is_ulong(self.cx) { @@ -1332,7 +1334,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( _catch_func: RValue<'gcc>, dest: PlaceRef<'tcx, RValue<'gcc>>, ) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { + if !bx.sess().panic_strategy().unwinds() { bx.call(bx.type_void(), None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs index 350915a277e3..41363d6313d6 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs @@ -206,6 +206,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( ); } + #[cfg(feature = "master")] + if name == sym::simd_funnel_shl { + return Ok(simd_funnel_shift( + bx, + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + true, + )); + } + + #[cfg(feature = "master")] + if name == sym::simd_funnel_shr { + return Ok(simd_funnel_shift( + bx, + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + false, + )); + } + if name == sym::simd_bswap { return Ok(simd_bswap(bx, args[0].immediate())); } @@ -1434,3 +1456,61 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>( unimplemented!("simd {}", name); } + +#[cfg(feature = "master")] +fn simd_funnel_shift<'a, 'gcc, 'tcx>( + bx: &mut Builder<'a, 'gcc, 'tcx>, + a: RValue<'gcc>, + b: RValue<'gcc>, + shift: RValue<'gcc>, + shift_left: bool, +) -> RValue<'gcc> { + use crate::common::SignType; + + let a_type = a.get_type(); + let vector_type = a_type.unqualified().dyncast_vector().expect("vector type"); + let num_units = vector_type.get_num_units(); + let elem_type = vector_type.get_element_type(); + + let (new_int_type, int_shift_val, int_mask) = if elem_type.is_compatible_with(bx.u8_type) + || elem_type.is_compatible_with(bx.i8_type) + { + (bx.u16_type, 8, u8::MAX as u64) + } else if elem_type.is_compatible_with(bx.u16_type) || elem_type.is_compatible_with(bx.i16_type) + { + (bx.u32_type, 16, u16::MAX as u64) + } else if elem_type.is_compatible_with(bx.u32_type) || elem_type.is_compatible_with(bx.i32_type) + { + (bx.u64_type, 32, u32::MAX as u64) + } else if elem_type.is_compatible_with(bx.u64_type) || elem_type.is_compatible_with(bx.i64_type) + { + (bx.u128_type, 64, u64::MAX) + } else { + unimplemented!("funnel shift on {:?}", elem_type); + }; + + let int_mask = bx.context.new_rvalue_from_long(new_int_type, int_mask as i64); + let int_shift_val = bx.context.new_rvalue_from_int(new_int_type, int_shift_val); + let mut elements = vec![]; + let unsigned_type = elem_type.to_unsigned(bx); + for i in 0..num_units { + let index = bx.context.new_rvalue_from_int(bx.int_type, i as i32); + let a_val = bx.context.new_vector_access(None, a, index).to_rvalue(); + let a_val = bx.context.new_bitcast(None, a_val, unsigned_type); + let a_val = bx.gcc_int_cast(a_val, new_int_type); + let b_val = bx.context.new_vector_access(None, b, index).to_rvalue(); + let b_val = bx.context.new_bitcast(None, b_val, unsigned_type); + let b_val = bx.gcc_int_cast(b_val, new_int_type); + let shift_val = bx.context.new_vector_access(None, shift, index).to_rvalue(); + let shift_val = bx.gcc_int_cast(shift_val, new_int_type); + let mut val = a_val << int_shift_val | b_val; + if shift_left { + val = (val << shift_val) >> int_shift_val; + } else { + val = (val >> shift_val) & int_mask; + } + let val = bx.gcc_int_cast(val, elem_type); + elements.push(val); + } + bx.context.new_rvalue_from_vector(None, a_type, &elements) +} diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs index a31206825007..71500ded0203 100644 --- a/compiler/rustc_codegen_gcc/src/lib.rs +++ b/compiler/rustc_codegen_gcc/src/lib.rs @@ -1,7 +1,5 @@ /* * TODO(antoyo): implement equality in libgccjit based on https://zpz.github.io/blog/overloading-equality-operator-in-cpp-class-hierarchy/ (for type equality?) - * TODO(antoyo): support #[inline] attributes. - * TODO(antoyo): support LTO (gcc's equivalent to Full LTO is -flto -flto-partition=one — https://documentation.suse.com/sbp/all/html/SBP-GCC-10/index.html). * For Thin LTO, this might be helpful: // cspell:disable-next-line * In gcc 4.6 -fwhopr was removed and became default with -flto. The non-whopr path can still be executed via -flto-partition=none. @@ -25,17 +23,10 @@ #![deny(clippy::pattern_type_mismatch)] #![allow(clippy::needless_lifetimes, clippy::uninlined_format_args)] -// These crates are pulled from the sysroot because they are part of -// rustc's public API, so we need to ensure version compatibility. -extern crate smallvec; -#[macro_use] -extern crate tracing; - // The rustc crates we need extern crate rustc_abi; extern crate rustc_apfloat; extern crate rustc_ast; -extern crate rustc_attr_data_structures; extern crate rustc_codegen_ssa; extern crate rustc_data_structures; extern crate rustc_errors; @@ -45,12 +36,14 @@ extern crate rustc_hir; extern crate rustc_index; #[cfg(feature = "master")] extern crate rustc_interface; +extern crate rustc_log; extern crate rustc_macros; extern crate rustc_middle; extern crate rustc_session; extern crate rustc_span; extern crate rustc_symbol_mangling; extern crate rustc_target; +extern crate rustc_type_ir; // This prevents duplicating functions and statics that are already part of the host rustc process. #[allow(unused_extern_crates)] @@ -92,8 +85,7 @@ use back::lto::{ThinBuffer, ThinData}; use gccjit::{CType, Context, OptimizationLevel}; #[cfg(feature = "master")] use gccjit::{TargetInfo, Version}; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; +use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn, @@ -111,7 +103,6 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames}; use rustc_span::Symbol; -use rustc_span::fatal_error::FatalError; use rustc_target::spec::RelocModel; use tempfile::TempDir; @@ -186,6 +177,10 @@ impl CodegenBackend for GccCodegenBackend { crate::DEFAULT_LOCALE_RESOURCE } + fn name(&self) -> &'static str { + "gcc" + } + fn init(&self, _sess: &Session) { #[cfg(feature = "master")] { @@ -282,8 +277,7 @@ impl ExtraBackendMethods for GccCodegenBackend { &self, tcx: TyCtxt<'_>, module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) -> Self::Module { let mut mods = GccContext { context: Arc::new(SyncContext::new(new_context(tcx))), @@ -293,7 +287,7 @@ impl ExtraBackendMethods for GccCodegenBackend { }; unsafe { - allocator::codegen(tcx, &mut mods, module_name, kind, alloc_error_handler_kind); + allocator::codegen(tcx, &mut mods, module_name, methods); } mods } @@ -363,12 +357,7 @@ impl WriteBackendMethods for GccCodegenBackend { _exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_fncs: Vec, - ) -> Result, FatalError> { - if !diff_fncs.is_empty() { - unimplemented!(); - } - + ) -> ModuleCodegen { back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules) } @@ -379,7 +368,7 @@ impl WriteBackendMethods for GccCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { + ) -> (Vec>, Vec) { back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules) } @@ -396,15 +385,14 @@ impl WriteBackendMethods for GccCodegenBackend { _dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError> { + ) { module.module_llvm.context.set_optimization_level(to_gcc_opt_level(config.opt_level)); - Ok(()) } fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError> { + ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) } @@ -412,15 +400,12 @@ impl WriteBackendMethods for GccCodegenBackend { cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result { + ) -> CompiledModule { back::write::codegen(cgcx, module, config) } - fn prepare_thin( - module: ModuleCodegen, - emit_summary: bool, - ) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module, emit_summary) + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) } fn serialize_module(_module: ModuleCodegen) -> (String, Self::ModuleBuffer) { diff --git a/compiler/rustc_codegen_gcc/src/mono_item.rs b/compiler/rustc_codegen_gcc/src/mono_item.rs index ff188c437dae..35d44d21bcbf 100644 --- a/compiler/rustc_codegen_gcc/src/mono_item.rs +++ b/compiler/rustc_codegen_gcc/src/mono_item.rs @@ -1,11 +1,12 @@ #[cfg(feature = "master")] use gccjit::{FnAttribute, VarAttribute}; use rustc_codegen_ssa::traits::PreDefineCodegenMethods; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index 093f902bc3d8..93202483eed8 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -240,7 +240,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { // Make sure lifetimes are erased, to avoid generating distinct LLVM // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); + let normal_ty = cx.tcx.erase_and_anonymize_regions(self.ty); let mut defer = None; let ty = if self.ty != normal_ty { diff --git a/compiler/rustc_codegen_gcc/target_specs/m68k-unknown-linux-gnu.json b/compiler/rustc_codegen_gcc/target_specs/m68k-unknown-linux-gnu.json index 95ea06106fb1..b13b640a7c7e 100644 --- a/compiler/rustc_codegen_gcc/target_specs/m68k-unknown-linux-gnu.json +++ b/compiler/rustc_codegen_gcc/target_specs/m68k-unknown-linux-gnu.json @@ -22,5 +22,5 @@ "unix" ], "target-mcount": "_mcount", - "target-pointer-width": "32" + "target-pointer-width": 32 } diff --git a/compiler/rustc_codegen_gcc/tests/failing-lto-tests.txt b/compiler/rustc_codegen_gcc/tests/failing-lto-tests.txt index b9126fb73a77..bf0633f73200 100644 --- a/compiler/rustc_codegen_gcc/tests/failing-lto-tests.txt +++ b/compiler/rustc_codegen_gcc/tests/failing-lto-tests.txt @@ -28,6 +28,6 @@ tests/ui/macros/macro-comma-behavior-rpass.rs tests/ui/macros/rfc-2011-nicer-assert-messages/assert-with-custom-errors-does-not-create-unnecessary-code.rs tests/ui/macros/rfc-2011-nicer-assert-messages/feature-gate-generic_assert.rs tests/ui/macros/stringify.rs -tests/ui/reexport-test-harness-main.rs tests/ui/rfcs/rfc-1937-termination-trait/termination-trait-in-test.rs tests/ui/binding/fn-arg-incomplete-pattern-drop-order.rs +tests/ui/lto/debuginfo-lto-alloc.rs diff --git a/compiler/rustc_codegen_gcc/tests/failing-run-make-tests.txt b/compiler/rustc_codegen_gcc/tests/failing-run-make-tests.txt index 842533cd3c62..c5e22970c660 100644 --- a/compiler/rustc_codegen_gcc/tests/failing-run-make-tests.txt +++ b/compiler/rustc_codegen_gcc/tests/failing-run-make-tests.txt @@ -6,8 +6,6 @@ tests/run-make/doctests-keep-binaries/ tests/run-make/doctests-runtool/ tests/run-make/emit-shared-files/ tests/run-make/exit-code/ -tests/run-make/issue-22131/ -tests/run-make/issue-64153/ tests/run-make/llvm-ident/ tests/run-make/native-link-modifier-bundle/ tests/run-make/remap-path-prefix-dwarf/ @@ -35,8 +33,6 @@ tests/run-make/c-link-to-rust-staticlib/ tests/run-make/foreign-double-unwind/ tests/run-make/foreign-exceptions/ tests/run-make/glibc-staticlib-args/ -tests/run-make/issue-36710/ -tests/run-make/issue-68794-textrel-on-minimal-lib/ tests/run-make/lto-smoke-c/ tests/run-make/return-non-c-like-enum/ diff --git a/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt b/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt index 6979c04d5343..e2615bce190e 100644 --- a/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt +++ b/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt @@ -10,18 +10,17 @@ tests/ui/iterators/iter-sum-overflow-overflow-checks.rs tests/ui/mir/mir_drop_order.rs tests/ui/mir/mir_let_chains_drop_order.rs tests/ui/mir/mir_match_guard_let_chains_drop_order.rs -tests/ui/oom_unwind.rs +tests/ui/panics/oom-panic-unwind.rs tests/ui/panic-runtime/abort-link-to-unwinding-crates.rs tests/ui/panic-runtime/abort.rs tests/ui/panic-runtime/link-to-abort.rs -tests/ui/unwind-no-uwtable.rs tests/ui/parser/unclosed-delimiter-in-dep.rs tests/ui/consts/missing_span_in_backtrace.rs tests/ui/drop/dynamic-drop.rs tests/ui/rfcs/rfc-2091-track-caller/std-panic-locations.rs tests/ui/simd/issue-17170.rs tests/ui/simd/issue-39720.rs -tests/ui/issues/issue-14875.rs +tests/ui/drop/panic-during-drop-14875.rs tests/ui/issues/issue-29948.rs tests/ui/process/println-with-broken-pipe.rs tests/ui/lto/thin-lto-inlines2.rs @@ -82,3 +81,10 @@ tests/ui/coroutine/panic-drops.rs tests/ui/coroutine/panic-safe.rs tests/ui/process/nofile-limit.rs tests/ui/simd/intrinsic/generic-arithmetic-pass.rs +tests/ui/linking/no-gc-encapsulation-symbols.rs +tests/ui/panics/unwind-force-no-unwind-tables.rs +tests/ui/attributes/fn-align-dyn.rs +tests/ui/linkage-attr/raw-dylib/elf/glibc-x86_64.rs +tests/ui/explicit-tail-calls/recursion-etc.rs +tests/ui/explicit-tail-calls/indexer.rs +tests/ui/explicit-tail-calls/drop-order.rs diff --git a/compiler/rustc_codegen_gcc/tests/run/int.rs b/compiler/rustc_codegen_gcc/tests/run/int.rs index e20ecc23679d..78675acb5447 100644 --- a/compiler/rustc_codegen_gcc/tests/run/int.rs +++ b/compiler/rustc_codegen_gcc/tests/run/int.rs @@ -7,12 +7,10 @@ fn main() { use std::hint::black_box; macro_rules! check { - ($ty:ty, $expr:expr) => { - { - const EXPECTED: $ty = $expr; - assert_eq!($expr, EXPECTED); - } - }; + ($ty:ty, $expr:expr) => {{ + const EXPECTED: $ty = $expr; + assert_eq!($expr, EXPECTED); + }}; } check!(u32, (2220326408_u32 + black_box(1)) >> (32 - 6)); diff --git a/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs b/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs index 78872159f62d..78e1cac57e03 100644 --- a/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs +++ b/compiler/rustc_codegen_gcc/tests/run/int_overflow.rs @@ -12,7 +12,7 @@ fn main() { let arg_count = std::env::args().count(); let int = isize::MAX; - let _int = int + arg_count as isize; // overflow + let _int = int + arg_count as isize; // overflow // If overflow checking is disabled, we should reach here. #[cfg(not(debug_assertions))] diff --git a/compiler/rustc_codegen_gcc/tests/run/structs.rs b/compiler/rustc_codegen_gcc/tests/run/structs.rs index da73cbed9ae9..e08e67837bec 100644 --- a/compiler/rustc_codegen_gcc/tests/run/structs.rs +++ b/compiler/rustc_codegen_gcc/tests/run/structs.rs @@ -27,12 +27,8 @@ fn one() -> isize { #[no_mangle] extern "C" fn main(argc: i32, _argv: *const *const u8) -> i32 { - let test = Test { - field: one(), - }; - let two = Two { - two: 2, - }; + let test = Test { field: one() }; + let two = Two { two: 2 }; unsafe { libc::printf(b"%ld\n\0" as *const u8 as *const i8, test.field); libc::printf(b"%ld\n\0" as *const u8 as *const i8, two.two); diff --git a/compiler/rustc_codegen_gcc/tests/run/volatile.rs b/compiler/rustc_codegen_gcc/tests/run/volatile.rs index 94a7bdc5c066..dc11fbfa600d 100644 --- a/compiler/rustc_codegen_gcc/tests/run/volatile.rs +++ b/compiler/rustc_codegen_gcc/tests/run/volatile.rs @@ -12,15 +12,11 @@ struct Struct { func: unsafe fn(*const ()), } -fn func(_ptr: *const ()) { -} +fn func(_ptr: *const ()) {} fn main() { let mut x = MaybeUninit::<&Struct>::uninit(); - x.write(&Struct { - pointer: std::ptr::null(), - func, - }); + x.write(&Struct { pointer: std::ptr::null(), func }); let x = unsafe { x.assume_init() }; let value = unsafe { (x as *const Struct).read_volatile() }; println!("{:?}", value); diff --git a/compiler/rustc_codegen_gcc/tests/run/volatile2.rs b/compiler/rustc_codegen_gcc/tests/run/volatile2.rs index bdcb82598789..ada112687d3b 100644 --- a/compiler/rustc_codegen_gcc/tests/run/volatile2.rs +++ b/compiler/rustc_codegen_gcc/tests/run/volatile2.rs @@ -7,7 +7,14 @@ mod libc { #[link(name = "c")] extern "C" { pub fn sigaction(signum: i32, act: *const sigaction, oldact: *mut sigaction) -> i32; - pub fn mmap(addr: *mut (), len: usize, prot: i32, flags: i32, fd: i32, offset: i64) -> *mut (); + pub fn mmap( + addr: *mut (), + len: usize, + prot: i32, + flags: i32, + fd: i32, + offset: i64, + ) -> *mut (); pub fn mprotect(addr: *mut (), len: usize, prot: i32) -> i32; } @@ -54,7 +61,8 @@ fn main() { libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, -1, 0, - ).cast(); + ) + .cast(); if STORAGE == libc::MAP_FAILED { panic!("error: mmap failed"); } diff --git a/compiler/rustc_codegen_gcc/tools/cspell_dicts/rustc_codegen_gcc.txt b/compiler/rustc_codegen_gcc/tools/cspell_dicts/rustc_codegen_gcc.txt index 31023e50ffa1..4fb018b3ecd8 100644 --- a/compiler/rustc_codegen_gcc/tools/cspell_dicts/rustc_codegen_gcc.txt +++ b/compiler/rustc_codegen_gcc/tools/cspell_dicts/rustc_codegen_gcc.txt @@ -8,6 +8,7 @@ clzll cmse codegened csky +ctfe ctlz ctpop cttz @@ -25,6 +26,7 @@ fwrapv gimple hrtb immediates +interner liblto llbb llcx @@ -47,6 +49,7 @@ mavx mcmodel minimumf minnumf +miri monomorphization monomorphizations monomorphized diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 5ab22f8fc4d9..67bd1e59bb0c 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -19,7 +19,6 @@ object = { version = "0.37.0", default-features = false, features = ["std", "rea rustc-demangle = "0.1.21" rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } @@ -38,11 +37,15 @@ rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_target = { path = "../rustc_target" } -serde = { version = "1", features = [ "derive" ]} +serde = { version = "1", features = ["derive"] } serde_json = "1" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } tracing = "0.1" # tidy-alphabetical-end [features] +# tidy-alphabetical-start check_only = ["rustc_llvm/check_only"] +llvm_enzyme = [] +# tidy-alphabetical-end + diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index 009e7e2487b6..e03c2868b0f1 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -1,4 +1,3 @@ -use std::borrow::Borrow; use std::cmp; use libc::c_uint; @@ -13,7 +12,7 @@ use rustc_codegen_ssa::traits::*; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::{bug, ty}; -use rustc_session::config; +use rustc_session::{Session, config}; use rustc_target::callconv::{ ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, }; @@ -23,10 +22,9 @@ use smallvec::SmallVec; use crate::attributes::{self, llfn_attrs_from_instance}; use crate::builder::Builder; use crate::context::CodegenCx; -use crate::llvm::{self, Attribute, AttributePlace}; -use crate::type_::Type; +use crate::llvm::{self, Attribute, AttributePlace, Type, Value}; +use crate::llvm_util; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; trait ArgAttributesExt { fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); @@ -41,9 +39,8 @@ trait ArgAttributesExt { const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; -const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 4] = [ (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), - (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), @@ -85,6 +82,20 @@ fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&' attrs.push(llattr.create_attr(cx.llcx)); } } + // captures(...) is only available since LLVM 21. + if (21, 0, 0) <= llvm_util::get_version() { + const CAPTURES_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 3] = [ + (ArgAttribute::CapturesNone, llvm::AttributeKind::CapturesNone), + (ArgAttribute::CapturesAddress, llvm::AttributeKind::CapturesAddress), + (ArgAttribute::CapturesReadOnly, llvm::AttributeKind::CapturesReadOnly), + ]; + for (attr, llattr) in CAPTURES_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + break; + } + } + } } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects // memory sanitizer's behavior. @@ -207,9 +218,9 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); } - // Unsized indirect qrguments + // Unsized indirect arguments cannot be stored PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Cast { cast, pad_i32: _ } => { // The ABI mandates that the value is passed as a different struct representation. @@ -238,6 +249,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { scratch_align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); bx.lifetime_end(llscratch, scratch_size); } @@ -264,12 +276,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { OperandValue::Pair(next(), next()).store(bx, dst); } PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { - let place_val = PlaceValue { - llval: next(), - llextra: Some(next()), - align: self.layout.align.abi, - }; - OperandValue::Ref(place_val).store(bx, dst); + bug!("unsized `ArgAbi` cannot be stored"); } PassMode::Direct(_) | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } @@ -396,7 +403,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv { - llvm::CallConv::from_conv(self.conv, cx.tcx.sess.target.arch.borrow()) + to_llvm_calling_convention(cx.tcx.sess, self.conv) } fn apply_attrs_llfn( @@ -500,7 +507,16 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { } } PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => { - apply(attrs); + let i = apply(attrs); + if cx.sess().opts.optimize != config::OptLevel::No + && llvm_util::get_version() >= (21, 0, 0) + { + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Argument(i), + &[llvm::AttributeKind::DeadOnReturn.create_attr(cx.llcx)], + ); + } } PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); @@ -526,7 +542,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { // If the declaration has an associated instance, compute extra attributes based on that. if let Some(instance) = instance { - llfn_attrs_from_instance(cx, llfn, instance); + llfn_attrs_from_instance( + cx, + cx.tcx, + llfn, + &cx.tcx.codegen_instance_attrs(instance.def), + Some(instance), + ); } } @@ -644,43 +666,44 @@ impl AbiBuilderMethods for Builder<'_, '_, '_> { } } -impl llvm::CallConv { - pub(crate) fn from_conv(conv: CanonAbi, arch: &str) -> Self { - match conv { - CanonAbi::C | CanonAbi::Rust => llvm::CCallConv, - CanonAbi::RustCold => llvm::PreserveMost, - // Functions with this calling convention can only be called from assembly, but it is - // possible to declare an `extern "custom"` block, so the backend still needs a calling - // convention for declaring foreign functions. - CanonAbi::Custom => llvm::CCallConv, - CanonAbi::GpuKernel => { - if arch == "amdgpu" { - llvm::AmdgpuKernel - } else if arch == "nvptx64" { - llvm::PtxKernel - } else { - panic!("Architecture {arch} does not support GpuKernel calling convention"); - } +/// Determines the appropriate [`llvm::CallConv`] to use for a given function +/// ABI, for the current target. +pub(crate) fn to_llvm_calling_convention(sess: &Session, abi: CanonAbi) -> llvm::CallConv { + match abi { + CanonAbi::C | CanonAbi::Rust => llvm::CCallConv, + CanonAbi::RustCold => llvm::PreserveMost, + // Functions with this calling convention can only be called from assembly, but it is + // possible to declare an `extern "custom"` block, so the backend still needs a calling + // convention for declaring foreign functions. + CanonAbi::Custom => llvm::CCallConv, + CanonAbi::GpuKernel => { + let arch = sess.target.arch.as_ref(); + if arch == "amdgpu" { + llvm::AmdgpuKernel + } else if arch == "nvptx64" { + llvm::PtxKernel + } else { + panic!("Architecture {arch} does not support GpuKernel calling convention"); } - CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind { - InterruptKind::Avr => llvm::AvrInterrupt, - InterruptKind::AvrNonBlocking => llvm::AvrNonBlockingInterrupt, - InterruptKind::Msp430 => llvm::Msp430Intr, - InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => llvm::CCallConv, - InterruptKind::X86 => llvm::X86_Intr, - }, - CanonAbi::Arm(arm_call) => match arm_call { - ArmCall::Aapcs => llvm::ArmAapcsCallConv, - ArmCall::CCmseNonSecureCall | ArmCall::CCmseNonSecureEntry => llvm::CCallConv, - }, - CanonAbi::X86(x86_call) => match x86_call { - X86Call::Fastcall => llvm::X86FastcallCallConv, - X86Call::Stdcall => llvm::X86StdcallCallConv, - X86Call::SysV64 => llvm::X86_64_SysV, - X86Call::Thiscall => llvm::X86_ThisCall, - X86Call::Vectorcall => llvm::X86_VectorCall, - X86Call::Win64 => llvm::X86_64_Win64, - }, } + CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind { + InterruptKind::Avr => llvm::AvrInterrupt, + InterruptKind::AvrNonBlocking => llvm::AvrNonBlockingInterrupt, + InterruptKind::Msp430 => llvm::Msp430Intr, + InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => llvm::CCallConv, + InterruptKind::X86 => llvm::X86_Intr, + }, + CanonAbi::Arm(arm_call) => match arm_call { + ArmCall::Aapcs => llvm::ArmAapcsCallConv, + ArmCall::CCmseNonSecureCall | ArmCall::CCmseNonSecureEntry => llvm::CCallConv, + }, + CanonAbi::X86(x86_call) => match x86_call { + X86Call::Fastcall => llvm::X86FastcallCallConv, + X86Call::Stdcall => llvm::X86StdcallCallConv, + X86Call::SysV64 => llvm::X86_64_SysV, + X86Call::Thiscall => llvm::X86_ThisCall, + X86Call::Vectorcall => llvm::X86_VectorCall, + X86Call::Win64 => llvm::X86_64_Win64, + }, } } diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs index 2b5090ed6dba..de0b85ebb63b 100644 --- a/compiler/rustc_codegen_llvm/src/allocator.rs +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -1,25 +1,26 @@ use libc::c_uint; use rustc_ast::expand::allocator::{ - ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, - alloc_error_handler_name, default_fn_name, global_fn_name, + AllocatorMethod, AllocatorTy, NO_ALLOC_SHIM_IS_UNSTABLE, SpecialAllocatorMethod, + default_fn_name, global_fn_name, }; use rustc_codegen_ssa::traits::BaseTypeCodegenMethods as _; use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{DebugInfo, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; +use crate::attributes::llfn_attrs_from_instance; use crate::builder::SBuilder; use crate::declare::declare_simple_fn; -use crate::llvm::{self, False, True, Type, Value}; +use crate::llvm::{self, FALSE, FromGeneric, TRUE, Type, Value}; use crate::{SimpleCx, attributes, debuginfo}; pub(crate) unsafe fn codegen( tcx: TyCtxt<'_>, cx: SimpleCx<'_>, module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) { let usize = match tcx.sess.target.pointer_width { 16 => cx.type_i16(), @@ -30,70 +31,85 @@ pub(crate) unsafe fn codegen( let i8 = cx.type_i8(); let i8p = cx.type_ptr(); - if kind == AllocatorKind::Default { - for method in ALLOCATOR_METHODS { - let mut args = Vec::with_capacity(method.inputs.len()); - for input in method.inputs.iter() { - match input.ty { - AllocatorTy::Layout => { - args.push(usize); // size - args.push(usize); // align - } - AllocatorTy::Ptr => args.push(i8p), - AllocatorTy::Usize => args.push(usize), + for method in methods { + let mut args = Vec::with_capacity(method.inputs.len()); + for input in method.inputs.iter() { + match input.ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + AllocatorTy::Never | AllocatorTy::ResultPtr | AllocatorTy::Unit => { + panic!("invalid allocator arg") } } - let output = match method.output { - AllocatorTy::ResultPtr => Some(i8p), - AllocatorTy::Unit => None, - - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; - - let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); - let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); - - create_wrapper_function(tcx, &cx, &from_name, Some(&to_name), &args, output, false); } - } - // rust alloc error handler - create_wrapper_function( - tcx, - &cx, - &mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - Some(&mangle_internal_symbol(tcx, alloc_error_handler_name(alloc_error_handler_kind))), - &[usize, usize], // size, align - None, - true, - ); + let mut no_return = false; + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, + AllocatorTy::Never => { + no_return = true; + None + } - unsafe { - // __rust_alloc_error_handler_should_panic_v2 - create_const_value_function( - tcx, - &cx, - &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - &i8, - &llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, False), - ); + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; - // __rust_no_alloc_shim_is_unstable_v2 + let from_name = mangle_internal_symbol(tcx, &global_fn_name(method.name)); + let to_name = mangle_internal_symbol(tcx, &default_fn_name(method.name)); + + let alloc_attr_flag = match method.special { + Some(SpecialAllocatorMethod::Alloc) => CodegenFnAttrFlags::ALLOCATOR, + Some(SpecialAllocatorMethod::Dealloc) => CodegenFnAttrFlags::DEALLOCATOR, + Some(SpecialAllocatorMethod::Realloc) => CodegenFnAttrFlags::REALLOCATOR, + Some(SpecialAllocatorMethod::AllocZeroed) => CodegenFnAttrFlags::ALLOCATOR_ZEROED, + None => CodegenFnAttrFlags::empty(), + }; + + let mut attrs = CodegenFnAttrs::new(); + attrs.flags |= alloc_attr_flag; create_wrapper_function( tcx, &cx, - &mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), - None, - &[], - None, - false, + &from_name, + Some(&to_name), + &args, + output, + no_return, + &attrs, ); } + // __rust_alloc_error_handler_should_panic_v2 + create_const_value_function( + tcx, + &cx, + &mangle_internal_symbol(tcx, OomStrategy::SYMBOL), + &i8, + unsafe { + llvm::LLVMConstInt(i8, tcx.sess.opts.unstable_opts.oom.should_panic() as u64, FALSE) + }, + ); + + // __rust_no_alloc_shim_is_unstable_v2 + create_wrapper_function( + tcx, + &cx, + &mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), + None, + &[], + None, + false, + &CodegenFnAttrs::new(), + ); + if tcx.sess.opts.debuginfo != DebugInfo::None { let dbg_cx = debuginfo::CodegenUnitDebugContext::new(cx.llmod); debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); @@ -137,6 +153,7 @@ fn create_wrapper_function( args: &[&Type], output: Option<&Type>, no_return: bool, + attrs: &CodegenFnAttrs, ) { let ty = cx.type_func(args, output.unwrap_or_else(|| cx.type_void())); let llfn = declare_simple_fn( @@ -147,6 +164,9 @@ fn create_wrapper_function( llvm::Visibility::from_generic(tcx.sess.default_visibility()), ty, ); + + llfn_attrs_from_instance(cx, tcx, llfn, attrs, None); + let no_return = if no_return { // -> ! DIFlagNoReturn let no_return = llvm::AttributeKind::NoReturn.create_attr(cx.llcx); @@ -156,12 +176,6 @@ fn create_wrapper_function( None }; - if tcx.sess.must_emit_unwind_tables() { - let uwtable = - attributes::uwtable_attr(cx.llcx, tcx.sess.opts.unstable_opts.use_sync_unwind); - attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); - } - let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, c"entry".as_ptr()) }; let mut bx = SBuilder::build(&cx, llbb); @@ -186,7 +200,7 @@ fn create_wrapper_function( .map(|(i, _)| llvm::get_param(llfn, i as c_uint)) .collect::>(); let ret = bx.call(ty, callee, &args, None); - llvm::LLVMSetTailCall(ret, True); + llvm::LLVMSetTailCall(ret, TRUE); if output.is_some() { bx.ret(ret); } else { diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a643a91141e1..8cd4bdc37278 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -13,13 +13,12 @@ use rustc_target::asm::*; use smallvec::SmallVec; use tracing::debug; +use crate::attributes; use crate::builder::Builder; use crate::common::Funclet; use crate::context::CodegenCx; -use crate::type_::Type; +use crate::llvm::{self, ToLlvmBool, Type, Value}; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use crate::{attributes, llvm}; impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { fn codegen_inline_asm( @@ -239,6 +238,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { constraints.extend_from_slice(&[ + "~{fflags}".to_string(), "~{vtype}".to_string(), "~{vl}".to_string(), "~{vxsat}".to_string(), @@ -338,8 +338,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); } else if options.contains(InlineAsmOptions::NOMEM) { attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx)); - } else { - // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx)); } attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); @@ -470,10 +470,6 @@ pub(crate) fn inline_asm_call<'ll>( dest: Option<&'ll llvm::BasicBlock>, catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { - let volatile = if volatile { llvm::True } else { llvm::False }; - let alignstack = if alignstack { llvm::True } else { llvm::False }; - let can_throw = if unwind { llvm::True } else { llvm::False }; - let argtys = inputs .iter() .map(|v| { @@ -500,10 +496,10 @@ pub(crate) fn inline_asm_call<'ll>( asm.len(), cons.as_ptr(), cons.len(), - volatile, - alignstack, + volatile.to_llvm_bool(), + alignstack.to_llvm_bool(), dia, - can_throw, + unwind.to_llvm_bool(), ) }; @@ -540,9 +536,7 @@ pub(crate) fn inline_asm_call<'ll>( bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)), ) })); - let md = unsafe { llvm::LLVMMDNodeInContext2(bx.llcx, srcloc.as_ptr(), srcloc.len()) }; - let md = bx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, kind, md); + bx.cx.set_metadata_node(call, kind, &srcloc); Some(call) } @@ -664,7 +658,13 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", PowerPC(PowerPCInlineAsmRegClass::freg) => "f", PowerPC(PowerPCInlineAsmRegClass::vreg) => "v", - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa", + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => "r", @@ -749,6 +749,12 @@ fn modifier_to_llvm( LoongArch(_) => None, Mips(_) => None, Nvptx(_) => None, + PowerPC(PowerPCInlineAsmRegClass::vsreg) => { + // The documentation for the 'x' modifier is missing for llvm, and the gcc + // documentation is simply "use this for any vsx argument". It is needed + // to ensure the correct vsx register number is used. + if modifier.is_none() { Some('x') } else { modifier } + } PowerPC(_) => None, RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None, RiscV(RiscVInlineAsmRegClass::vreg) => unreachable!("clobber-only"), @@ -832,7 +838,13 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &' PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4), - PowerPC(PowerPCInlineAsmRegClass::cr) | PowerPC(PowerPCInlineAsmRegClass::xer) => { + PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4), + PowerPC( + PowerPCInlineAsmRegClass::cr + | PowerPCInlineAsmRegClass::ctr + | PowerPCInlineAsmRegClass::lr + | PowerPCInlineAsmRegClass::xer, + ) => { unreachable!("clobber-only") } RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), @@ -1057,9 +1069,10 @@ fn llvm_fixup_input<'ll, 'tcx>( let value = bx.or(value, bx.const_u32(0xFFFF_0000)); bx.bitcast(value, bx.type_f32()) } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F32) => - { + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F32) => { let value = bx.insert_element( bx.const_undef(bx.type_vector(bx.type_f32(), 4)), value, @@ -1067,9 +1080,10 @@ fn llvm_fixup_input<'ll, 'tcx>( ); bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)) } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F64) => - { + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F64) => { let value = bx.insert_element( bx.const_undef(bx.type_vector(bx.type_f64(), 2)), value, @@ -1220,15 +1234,17 @@ fn llvm_fixup_output<'ll, 'tcx>( let value = bx.trunc(value, bx.type_i16()); bx.bitcast(value, bx.type_f16()) } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F32) => - { + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F32) => { let value = bx.bitcast(value, bx.type_vector(bx.type_f32(), 4)); bx.extract_element(value, bx.const_usize(0)) } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F64) => - { + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F64) => { let value = bx.bitcast(value, bx.type_vector(bx.type_f64(), 2)); bx.extract_element(value, bx.const_usize(0)) } @@ -1362,16 +1378,14 @@ fn llvm_fixup_output_type<'ll, 'tcx>( { cx.type_f32() } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F32) => - { - cx.type_vector(cx.type_f32(), 4) - } - (PowerPC(PowerPCInlineAsmRegClass::vreg), BackendRepr::Scalar(s)) - if s.primitive() == Primitive::Float(Float::F64) => - { - cx.type_vector(cx.type_f64(), 2) - } + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F32) => cx.type_vector(cx.type_f32(), 4), + ( + PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg), + BackendRepr::Scalar(s), + ) if s.primitive() == Primitive::Float(Float::F64) => cx.type_vector(cx.type_f64(), 2), _ => layout.llvm_type(cx), } } diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs index c32f11b27f3d..209b8efa2c3b 100644 --- a/compiler/rustc_codegen_llvm/src/attributes.rs +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -1,20 +1,22 @@ //! Set and unset common attributes on LLVM values. -use rustc_attr_data_structures::{InlineAttr, InstructionSetAttr, OptimizeAttr}; -use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::{InlineAttr, InstructionSetAttr, OptimizeAttr}; use rustc_hir::def_id::DefId; -use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, PatchableFunctionEntry}; +use rustc_middle::middle::codegen_fn_attrs::{ + CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, +}; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::config::{BranchProtection, FunctionReturn, OptLevel, PAuthKey, PacRet}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; use smallvec::SmallVec; -use crate::context::CodegenCx; +use crate::context::SimpleCx; use crate::errors::SanitizerMemtagRequiresMte; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; -use crate::value::Value; -use crate::{attributes, llvm_util}; +use crate::llvm::{ + self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects, Value, +}; +use crate::{Session, attributes, llvm_util}; pub(crate) fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { if !attrs.is_empty() { @@ -28,26 +30,21 @@ pub(crate) fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[ } } -pub(crate) fn has_attr(llfn: &Value, idx: AttributePlace, attr: AttributeKind) -> bool { - llvm::HasAttributeAtIndex(llfn, idx, attr) -} - -pub(crate) fn has_string_attr(llfn: &Value, name: &str) -> bool { - llvm::HasStringAttribute(llfn, name) -} - -pub(crate) fn remove_from_llfn(llfn: &Value, place: AttributePlace, kind: AttributeKind) { - llvm::RemoveRustEnumAttributeAtIndex(llfn, place, kind); -} - -pub(crate) fn remove_string_attr_from_llfn(llfn: &Value, name: &str) { - llvm::RemoveStringAttrFromFn(llfn, name); -} - /// Get LLVM attribute for the provided inline heuristic. -#[inline] -fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { - if !cx.tcx.sess.opts.unstable_opts.inline_llvm { +pub(crate) fn inline_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, +) -> Option<&'ll Attribute> { + // `optnone` requires `noinline` + let codegen_fn_attrs = tcx.codegen_fn_attrs(instance.def_id()); + let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { + (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, + (InlineAttr::None, _) if instance.def.requires_inline(tcx) => InlineAttr::Hint, + (inline, _) => inline, + }; + + if !tcx.sess.opts.unstable_opts.inline_llvm { // disable LLVM inlining return Some(AttributeKind::NoInline.create_attr(cx.llcx)); } @@ -57,7 +54,7 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)) } InlineAttr::Never => { - if cx.sess().target.arch != "amdgpu" { + if tcx.sess.target.arch != "amdgpu" { Some(AttributeKind::NoInline.create_attr(cx.llcx)) } else { None @@ -69,12 +66,13 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll #[inline] fn patchable_function_entry_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, attr: Option, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); let patchable_spec = attr.unwrap_or_else(|| { - PatchableFunctionEntry::from_config(cx.tcx.sess.opts.unstable_opts.patchable_function_entry) + PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry) }); let entry = patchable_spec.entry(); let prefix = patchable_spec.prefix(); @@ -97,12 +95,13 @@ fn patchable_function_entry_attrs<'ll>( /// Get LLVM sanitize attributes. #[inline] -pub(crate) fn sanitize_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, +pub(crate) fn sanitize_attrs<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, no_sanitize: SanitizerSet, ) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + let enabled = tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); } @@ -120,11 +119,11 @@ pub(crate) fn sanitize_attrs<'ll>( } if enabled.contains(SanitizerSet::MEMTAG) { // Check to make sure the mte target feature is actually enabled. - let features = cx.tcx.global_backend_features(()); + let features = tcx.global_backend_features(()); let mte_feature = features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); if let None | Some("-mte") = mte_feature { - cx.tcx.dcx().emit_err(SanitizerMemtagRequiresMte); + tcx.dcx().emit_err(SanitizerMemtagRequiresMte); } attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); @@ -145,9 +144,12 @@ pub(crate) fn uwtable_attr(llcx: &llvm::Context, use_sync_unwind: Option) llvm::CreateUWTableAttr(llcx, async_unwind) } -pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let mut fp = cx.sess().target.frame_pointer; - let opts = &cx.sess().opts; +pub(crate) fn frame_pointer_type_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { + let mut fp = sess.target.frame_pointer; + let opts = &sess.opts; // "mcount" function relies on stack pointer. // See . if opts.unstable_opts.instrument_mcount { @@ -162,8 +164,8 @@ pub(crate) fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&' Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) } -fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let function_return_attr = match cx.sess().opts.unstable_opts.function_return { +fn function_return_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let function_return_attr = match sess.opts.unstable_opts.function_return { FunctionReturn::Keep => return None, FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern, }; @@ -173,17 +175,20 @@ fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> /// Tell LLVM what instrument function to insert. #[inline] -fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { +fn instrument_function_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> SmallVec<[&'ll Attribute; 4]> { let mut attrs = SmallVec::new(); - if cx.sess().opts.unstable_opts.instrument_mcount { + if sess.opts.unstable_opts.instrument_mcount { // Similar to `clang -pg` behavior. Handled by the // `post-inline-ee-instrument` LLVM pass. // The function name varies on platforms. // See test/CodeGen/mcount.c in clang. - let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { + let mcount_name = match &sess.target.llvm_mcount_intrinsic { Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), - None => cx.sess().target.mcount.as_ref(), + None => sess.target.mcount.as_ref(), }; attrs.push(llvm::CreateAttrStringValue( @@ -192,7 +197,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr mcount_name, )); } - if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { + if let Some(options) = &sess.opts.unstable_opts.instrument_xray { // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. // Function prologue and epilogue are instrumented with NOP sleds, // a runtime library later replaces them with detours into tracing code. @@ -223,20 +228,20 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr attrs } -fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if !cx.sess().opts.unstable_opts.no_jump_tables { +fn nojumptables_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if !sess.opts.unstable_opts.no_jump_tables { return None; } Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) } -fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +fn probestack_attr<'ll, 'tcx>(cx: &SimpleCx<'ll>, tcx: TyCtxt<'tcx>) -> Option<&'ll Attribute> { // Currently stack probes seem somewhat incompatible with the address // sanitizer and thread sanitizer. With asan we're already protected from // stack overflow anyway so we don't really need stack probes regardless. - if cx - .sess() + if tcx + .sess .opts .unstable_opts .sanitizer @@ -246,22 +251,22 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { } // probestack doesn't play nice either with `-C profile-generate`. - if cx.sess().opts.cg.profile_generate.enabled() { + if tcx.sess.opts.cg.profile_generate.enabled() { return None; } - let attr_value = match cx.sess().target.stack_probes { + let attr_value = match tcx.sess.target.stack_probes { StackProbeType::None => return None, // Request LLVM to generate the probes inline. If the given LLVM version does not support // this, no probe is generated at all (even if the attribute is specified). StackProbeType::Inline => "inline-asm", // Flag our internal `__rust_probestack` function as the stack probe symbol. // This is defined in the `compiler-builtins` crate for each architecture. - StackProbeType::Call => &mangle_internal_symbol(cx.tcx, "__rust_probestack"), + StackProbeType::Call => &mangle_internal_symbol(tcx, "__rust_probestack"), // Pick from the two above based on the LLVM version. StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { if llvm_util::get_version() < min_llvm_version_for_inline { - &mangle_internal_symbol(cx.tcx, "__rust_probestack") + &mangle_internal_symbol(tcx, "__rust_probestack") } else { "inline-asm" } @@ -270,8 +275,8 @@ fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) } -fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - let sspattr = match cx.sess().stack_protector() { +fn stackprotector_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + let sspattr = match sess.stack_protector() { StackProtector::None => return None, StackProtector::All => AttributeKind::StackProtectReq, StackProtector::Strong => AttributeKind::StackProtectStrong, @@ -281,45 +286,59 @@ fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { Some(sspattr.create_attr(cx.llcx)) } -fn backchain_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - if cx.sess().target.arch != "s390x" { +fn backchain_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + if sess.target.arch != "s390x" { return None; } - let requested_features = cx.sess().opts.cg.target_feature.split(','); + let requested_features = sess.opts.cg.target_feature.split(','); let found_positive = requested_features.clone().any(|r| r == "+backchain"); if found_positive { Some(llvm::CreateAttrString(cx.llcx, "backchain")) } else { None } } -pub(crate) fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { - let target_cpu = llvm_util::target_cpu(cx.tcx.sess); +pub(crate) fn target_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(sess); llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) } -pub(crate) fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { - llvm_util::tune_cpu(cx.tcx.sess) +pub(crate) fn tune_cpu_attr<'ll>(cx: &SimpleCx<'ll>, sess: &Session) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(sess) .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) } +/// Get the `target-features` LLVM attribute. +pub(crate) fn target_features_attr<'ll, 'tcx>( + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, + function_features: Vec, +) -> Option<&'ll Attribute> { + let global_features = tcx.global_backend_features(()).iter().map(String::as_str); + let function_features = function_features.iter().map(String::as_str); + let target_features = + global_features.chain(function_features).intersperse(",").collect::(); + (!target_features.is_empty()) + .then(|| llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)) +} + /// Get the `NonLazyBind` LLVM attribute, /// if the codegen options allow skipping the PLT. -pub(crate) fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { +pub(crate) fn non_lazy_bind_attr<'ll>( + cx: &SimpleCx<'ll>, + sess: &Session, +) -> Option<&'ll Attribute> { // Don't generate calls through PLT if it's not necessary - if !cx.sess().needs_plt() { - Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) - } else { - None - } + if !sess.needs_plt() { Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) } else { None } } /// Get the default optimizations attrs for a function. #[inline] pub(crate) fn default_optimisation_attrs<'ll>( - cx: &CodegenCx<'ll, '_>, + cx: &SimpleCx<'ll>, + sess: &Session, ) -> SmallVec<[&'ll Attribute; 2]> { let mut attrs = SmallVec::new(); - match cx.sess().opts.optimize { + match sess.opts.optimize { OptLevel::Size => { attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); } @@ -340,17 +359,18 @@ fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) /// attributes. pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( - cx: &CodegenCx<'ll, 'tcx>, + cx: &SimpleCx<'ll>, + tcx: TyCtxt<'tcx>, llfn: &'ll Value, - instance: ty::Instance<'tcx>, + codegen_fn_attrs: &CodegenFnAttrs, + instance: Option>, ) { - let codegen_fn_attrs = cx.tcx.codegen_instance_attrs(instance.def); - + let sess = tcx.sess; let mut to_add = SmallVec::<[_; 16]>::new(); match codegen_fn_attrs.optimize { OptimizeAttr::Default => { - to_add.extend(default_optimisation_attrs(cx)); + to_add.extend(default_optimisation_attrs(cx, sess)); } OptimizeAttr::DoNotOptimize => { to_add.push(llvm::AttributeKind::OptimizeNone.create_attr(cx.llcx)); @@ -362,29 +382,21 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( OptimizeAttr::Speed => {} } - // `optnone` requires `noinline` - let inline = match (codegen_fn_attrs.inline, &codegen_fn_attrs.optimize) { - (_, OptimizeAttr::DoNotOptimize) => InlineAttr::Never, - (InlineAttr::None, _) if instance.def.requires_inline(cx.tcx) => InlineAttr::Hint, - (inline, _) => inline, - }; - to_add.extend(inline_attr(cx, inline)); - - if cx.sess().must_emit_unwind_tables() { - to_add.push(uwtable_attr(cx.llcx, cx.sess().opts.unstable_opts.use_sync_unwind)); + if sess.must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx, sess.opts.unstable_opts.use_sync_unwind)); } - if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + if sess.opts.unstable_opts.profile_sample_use.is_some() { to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); } // FIXME: none of these functions interact with source level attributes. - to_add.extend(frame_pointer_type_attr(cx)); - to_add.extend(function_return_attr(cx)); - to_add.extend(instrument_function_attr(cx)); - to_add.extend(nojumptables_attr(cx)); - to_add.extend(probestack_attr(cx)); - to_add.extend(stackprotector_attr(cx)); + to_add.extend(frame_pointer_type_attr(cx, sess)); + to_add.extend(function_return_attr(cx, sess)); + to_add.extend(instrument_function_attr(cx, sess)); + to_add.extend(nojumptables_attr(cx, sess)); + to_add.extend(probestack_attr(cx, tcx)); + to_add.extend(stackprotector_attr(cx, sess)); if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); @@ -405,16 +417,19 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( // not used. } else { // Do not set sanitizer attributes for naked functions. - to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + to_add.extend(sanitize_attrs(cx, tcx, codegen_fn_attrs.no_sanitize)); // For non-naked functions, set branch protection attributes on aarch64. - if let Some(BranchProtection { bti, pac_ret }) = - cx.sess().opts.unstable_opts.branch_protection + if let Some(BranchProtection { bti, pac_ret, gcs }) = + sess.opts.unstable_opts.branch_protection { - assert!(cx.sess().target.arch == "aarch64"); + assert!(sess.target.arch == "aarch64"); if bti { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement")); } + if gcs { + to_add.push(llvm::CreateAttrString(cx.llcx, "guarded-control-stack")); + } if let Some(PacRet { leaf, pc, key }) = pac_ret { if pc { to_add.push(llvm::CreateAttrString(cx.llcx, "branch-protection-pauth-lr")); @@ -436,6 +451,17 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) { to_add.push(create_alloc_family_attr(cx.llcx)); + if let Some(instance) = instance + && let Some(zv) = + tcx.get_attr(instance.def_id(), rustc_span::sym::rustc_allocator_zeroed_variant) + && let Some(name) = zv.value_str() + { + to_add.push(llvm::CreateAttrStringValue( + cx.llcx, + "alloc-variant-zeroed", + &mangle_internal_symbol(tcx, name.as_str()), + )); + } // apply to argument place instead of function let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); @@ -478,26 +504,40 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( if let Some(align) = codegen_fn_attrs.alignment { llvm::set_alignment(llfn, align); } - if let Some(backchain) = backchain_attr(cx) { + if let Some(backchain) = backchain_attr(cx, sess) { to_add.push(backchain); } - to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry)); + to_add.extend(patchable_function_entry_attrs( + cx, + sess, + codegen_fn_attrs.patchable_function_entry, + )); // Always annotate functions with the target-cpu they are compiled for. // Without this, ThinLTO won't inline Rust functions into Clang generated // functions (because Clang annotates functions this way too). - to_add.push(target_cpu_attr(cx)); + to_add.push(target_cpu_attr(cx, sess)); // tune-cpu is only conveyed through the attribute for our purpose. // The target doesn't care; the subtarget reads our attribute. - to_add.extend(tune_cpu_attr(cx)); + to_add.extend(tune_cpu_attr(cx, sess)); let function_features = codegen_fn_attrs.target_features.iter().map(|f| f.name.as_str()).collect::>(); + // Apply function attributes as per usual if there are no user defined + // target features otherwise this will get applied at the callsite. + if function_features.is_empty() { + if let Some(instance) = instance + && let Some(inline_attr) = inline_attr(cx, tcx, instance) + { + to_add.push(inline_attr); + } + } + let function_features = function_features .iter() // Convert to LLVMFeatures and filter out unavailable ones - .flat_map(|feat| llvm_util::to_llvm_features(cx.tcx.sess, feat)) + .flat_map(|feat| llvm_util::to_llvm_features(sess, feat)) // Convert LLVMFeatures & dependencies to +s .flat_map(|feat| feat.into_iter().map(|f| format!("+{f}"))) .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { @@ -506,26 +546,22 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>( })) .collect::>(); - if cx.tcx.sess.target.is_like_wasm { + if sess.target.is_like_wasm { // If this function is an import from the environment but the wasm // import has a specific module/name, apply them here. - if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + if let Some(instance) = instance + && let Some(module) = wasm_import_module(tcx, instance.def_id()) + { to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module)); let name = - codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + codegen_fn_attrs.symbol_name.unwrap_or_else(|| tcx.item_name(instance.def_id())); let name = name.as_str(); to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); } } - let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); - let function_features = function_features.iter().map(|s| s.as_str()); - let target_features: String = - global_features.chain(function_features).intersperse(",").collect(); - if !target_features.is_empty() { - to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); - } + to_add.extend(target_features_attr(cx, tcx, function_features)); attributes::apply_to_llfn(llfn, Function, &to_add); } diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs index 0a161442933a..f9dc48e3aba3 100644 --- a/compiler/rustc_codegen_llvm/src/back/archive.rs +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -1,104 +1,21 @@ //! A helper class for dealing with static archives -use std::ffi::{CStr, CString, c_char, c_void}; -use std::path::{Path, PathBuf}; -use std::{io, mem, ptr, str}; +use std::ffi::{CStr, c_char, c_void}; +use std::io; use rustc_codegen_ssa::back::archive::{ - ArArchiveBuilder, ArchiveBuildFailure, ArchiveBuilder, ArchiveBuilderBuilder, - DEFAULT_OBJECT_READER, ObjectReader, UnknownArchiveKind, try_extract_macho_fat_archive, + ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER, ObjectReader, }; use rustc_session::Session; -use crate::llvm::archive_ro::{ArchiveRO, Child}; -use crate::llvm::{self, ArchiveKind, last_error}; - -/// Helper for adding many files to an archive. -#[must_use = "must call build() to finish building the archive"] -pub(crate) struct LlvmArchiveBuilder<'a> { - sess: &'a Session, - additions: Vec, -} - -enum Addition { - File { path: PathBuf, name_in_archive: String }, - Archive { path: PathBuf, archive: ArchiveRO, skip: Box bool> }, -} - -impl Addition { - fn path(&self) -> &Path { - match self { - Addition::File { path, .. } | Addition::Archive { path, .. } => path, - } - } -} - -fn is_relevant_child(c: &Child<'_>) -> bool { - match c.name() { - Some(name) => !name.contains("SYMDEF"), - None => false, - } -} - -impl<'a> ArchiveBuilder for LlvmArchiveBuilder<'a> { - fn add_archive( - &mut self, - archive: &Path, - skip: Box bool + 'static>, - ) -> io::Result<()> { - let mut archive = archive.to_path_buf(); - if self.sess.target.llvm_target.contains("-apple-macosx") { - if let Some(new_archive) = try_extract_macho_fat_archive(self.sess, &archive)? { - archive = new_archive - } - } - let archive_ro = match ArchiveRO::open(&archive) { - Ok(ar) => ar, - Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), - }; - if self.additions.iter().any(|ar| ar.path() == archive) { - return Ok(()); - } - self.additions.push(Addition::Archive { - path: archive, - archive: archive_ro, - skip: Box::new(skip), - }); - Ok(()) - } - - /// Adds an arbitrary file to this archive - fn add_file(&mut self, file: &Path) { - let name = file.file_name().unwrap().to_str().unwrap(); - self.additions - .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); - } - - /// Combine the provided files, rlibs, and native libraries into a single - /// `Archive`. - fn build(mut self: Box, output: &Path) -> bool { - match self.build_with_llvm(output) { - Ok(any_members) => any_members, - Err(error) => { - self.sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error }) - } - } - } -} +use crate::llvm; pub(crate) struct LlvmArchiveBuilderBuilder; impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box { - // Keeping LlvmArchiveBuilder around in case of a regression caused by using - // ArArchiveBuilder. - // FIXME(#128955) remove a couple of months after #128936 gets merged in case - // no regression is found. - if false { - Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) - } else { - Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) - } + // Use the `object` crate to build archives, with a little bit of help from LLVM. + Box::new(ArArchiveBuilder::new(sess, &LLVM_OBJECT_READER)) } } @@ -109,6 +26,7 @@ static LLVM_OBJECT_READER: ObjectReader = ObjectReader { get_symbols: get_llvm_object_symbols, is_64_bit_object_file: llvm_is_64_bit_object_file, is_ec_object_file: llvm_is_ec_object_file, + is_any_arm64_coff: llvm_is_any_arm64_coff, get_xcoff_member_alignment: DEFAULT_OBJECT_READER.get_xcoff_member_alignment, }; @@ -179,90 +97,6 @@ fn llvm_is_ec_object_file(buf: &[u8]) -> bool { unsafe { llvm::LLVMRustIsECObject(buf.as_ptr(), buf.len()) } } -impl<'a> LlvmArchiveBuilder<'a> { - fn build_with_llvm(&mut self, output: &Path) -> io::Result { - let kind = &*self.sess.target.archive_format; - let kind = kind - .parse::() - .map_err(|_| kind) - .unwrap_or_else(|kind| self.sess.dcx().emit_fatal(UnknownArchiveKind { kind })); - - let mut additions = mem::take(&mut self.additions); - // Values in the `members` list below will contain pointers to the strings allocated here. - // So they need to get dropped after all elements of `members` get freed. - let mut strings = Vec::new(); - let mut members = Vec::new(); - - let dst = CString::new(output.to_str().unwrap())?; - - unsafe { - for addition in &mut additions { - match addition { - Addition::File { path, name_in_archive } => { - let path = CString::new(path.to_str().unwrap())?; - let name = CString::new(name_in_archive.as_bytes())?; - members.push(llvm::LLVMRustArchiveMemberNew( - path.as_ptr(), - name.as_ptr(), - None, - )); - strings.push(path); - strings.push(name); - } - Addition::Archive { archive, skip, .. } => { - for child in archive.iter() { - let child = child.map_err(string_to_io_error)?; - if !is_relevant_child(&child) { - continue; - } - let child_name = child.name().unwrap(); - if skip(child_name) { - continue; - } - - // It appears that LLVM's archive writer is a little - // buggy if the name we pass down isn't just the - // filename component, so chop that off here and - // pass it in. - // - // See LLVM bug 25877 for more info. - let child_name = - Path::new(child_name).file_name().unwrap().to_str().unwrap(); - let name = CString::new(child_name)?; - let m = llvm::LLVMRustArchiveMemberNew( - ptr::null(), - name.as_ptr(), - Some(child.raw), - ); - members.push(m); - strings.push(name); - } - } - } - } - - let r = llvm::LLVMRustWriteArchive( - dst.as_ptr(), - members.len() as libc::size_t, - members.as_ptr() as *const &_, - true, - kind, - self.sess.target.arch == "arm64ec", - ); - let ret = if r.into_result().is_err() { - let msg = last_error().unwrap_or_else(|| "failed to write archive".into()); - Err(io::Error::new(io::ErrorKind::Other, msg)) - } else { - Ok(!members.is_empty()) - }; - for member in members { - llvm::LLVMRustArchiveMemberFree(member); - } - ret - } - } -} - -fn string_to_io_error(s: String) -> io::Error { - io::Error::new(io::ErrorKind::Other, format!("bad archive: {s}")) +fn llvm_is_any_arm64_coff(buf: &[u8]) -> bool { + unsafe { llvm::LLVMRustIsAnyArm64Coff(buf.as_ptr(), buf.len()) } } diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index c269f11e931b..5ac3a87c158e 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -14,7 +14,8 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::memmap::Mmap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; +use rustc_hir::attrs::SanitizerSet; use rustc_middle::bug; use rustc_middle::dep_graph::WorkProduct; use rustc_session::config::{self, Lto}; @@ -24,9 +25,8 @@ use crate::back::write::{ self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode, }; use crate::errors::{LlvmError, LtoBitcodeFromRlib}; -use crate::llvm::AttributePlace::Function; use crate::llvm::{self, build_string}; -use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes}; +use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx}; /// We keep track of the computed LTO cache keys from the previous /// session to determine which CGUs we can reuse. @@ -37,12 +37,41 @@ fn prepare_lto( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], dcx: DiagCtxtHandle<'_>, -) -> Result<(Vec, Vec<(SerializedModule, CString)>), FatalError> { +) -> (Vec, Vec<(SerializedModule, CString)>) { let mut symbols_below_threshold = exported_symbols_for_lto .iter() .map(|symbol| CString::new(symbol.to_owned()).unwrap()) .collect::>(); + if cgcx.module_config.instrument_coverage || cgcx.module_config.pgo_gen.enabled() { + // These are weak symbols that point to the profile version and the + // profile name, which need to be treated as exported so LTO doesn't nix + // them. + const PROFILER_WEAK_SYMBOLS: [&CStr; 2] = + [c"__llvm_profile_raw_version", c"__llvm_profile_filename"]; + + symbols_below_threshold.extend(PROFILER_WEAK_SYMBOLS.iter().map(|&sym| sym.to_owned())); + } + + if cgcx.module_config.sanitizer.contains(SanitizerSet::MEMORY) { + let mut msan_weak_symbols = Vec::new(); + + // Similar to profiling, preserve weak msan symbol during LTO. + if cgcx.module_config.sanitizer_recover.contains(SanitizerSet::MEMORY) { + msan_weak_symbols.push(c"__msan_keep_going"); + } + + if cgcx.module_config.sanitizer_memory_track_origins != 0 { + msan_weak_symbols.push(c"__msan_track_origins"); + } + + symbols_below_threshold.extend(msan_weak_symbols.into_iter().map(|sym| sym.to_owned())); + } + + // Preserve LLVM-injected, ASAN-related symbols. + // See also https://github.com/rust-lang/rust/issues/113404. + symbols_below_threshold.push(c"___asan_globals_registered".to_owned()); + // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to // __llvm_profile_runtime, therefore we won't know until link time if this symbol // should have default visibility. @@ -80,16 +109,13 @@ fn prepare_lto( let module = SerializedModule::FromRlib(data.to_vec()); upstream_modules.push((module, CString::new(name).unwrap())); } - Err(e) => { - dcx.emit_err(e); - return Err(FatalError); - } + Err(e) => dcx.emit_fatal(e), } } } } - Ok((symbols_below_threshold, upstream_modules)) + (symbols_below_threshold, upstream_modules) } fn get_bitcode_slice_from_object_data<'a>( @@ -124,11 +150,11 @@ pub(crate) fn run_fat( exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold) @@ -143,11 +169,11 @@ pub(crate) fn run_thin( each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); let (symbols_below_threshold, upstream_modules) = - prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?; + prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx); let symbols_below_threshold = symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); if cgcx.opts.cg.linker_plugin_lto.enabled() { @@ -159,12 +185,9 @@ pub(crate) fn run_thin( thin_lto(cgcx, dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold) } -pub(crate) fn prepare_thin( - module: ModuleCodegen, - emit_summary: bool, -) -> (String, ThinBuffer) { +pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { let name = module.name; - let buffer = ThinBuffer::new(module.module_llvm.llmod(), true, emit_summary); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); (name, buffer) } @@ -174,7 +197,7 @@ fn fat_lto( modules: Vec>, mut serialized_modules: Vec<(SerializedModule, CString)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result, FatalError> { +) -> ModuleCodegen { let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); info!("going for a fat lto"); @@ -225,7 +248,7 @@ fn fat_lto( assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); let (buffer, name) = serialized_modules.remove(0); info!("no in-memory regular modules to choose from, parsing {:?}", name); - let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?; + let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx); ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module) } }; @@ -266,7 +289,9 @@ fn fat_lto( }); info!("linking {:?}", name); let data = bc_decoded.data(); - linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?; + linker + .add(data) + .unwrap_or_else(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name })); } drop(linker); save_temp_bitcode(cgcx, &module, "lto.input"); @@ -283,7 +308,7 @@ fn fat_lto( save_temp_bitcode(cgcx, &module, "lto.after-restriction"); } - Ok(module) + module } pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); @@ -353,7 +378,7 @@ fn thin_lto( serialized_modules: Vec<(SerializedModule, CString)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, symbols_below_threshold: &[*const libc::c_char], -) -> Result<(Vec>, Vec), FatalError> { +) -> (Vec>, Vec) { let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); unsafe { info!("going for that thin, thin LTO"); @@ -423,7 +448,7 @@ fn thin_lto( symbols_below_threshold.as_ptr(), symbols_below_threshold.len(), ) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?; + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext)); let data = ThinData(data); @@ -493,10 +518,10 @@ fn thin_lto( if let Some(path) = key_map_path && let Err(err) = curr_key_map.save_to_file(&path) { - return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err })); + write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }); } - Ok((opt_jobs, copy_jobs)) + (opt_jobs, copy_jobs) } } @@ -538,6 +563,8 @@ fn enable_autodiff_settings(ad: &[config::AutoDiff]) { config::AutoDiff::Enable => {} // We handle this below config::AutoDiff::NoPostopt => {} + // Disables TypeTree generation + config::AutoDiff::NoTT => {} } } // This helps with handling enums for now. @@ -551,9 +578,9 @@ pub(crate) fn run_pass_manager( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, thin: bool, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name); - let config = cgcx.config(module.kind); + let config = &cgcx.module_config; // Now we have one massive module inside of llmod. Time to run the // LTO-specific optimization passes that LLVM provides. @@ -583,7 +610,7 @@ pub(crate) fn run_pass_manager( } unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } if enable_gpu && !thin { @@ -592,37 +619,12 @@ pub(crate) fn run_pass_manager( crate::builder::gpu_offload::handle_gpu_code(cgcx, &cx); } - if cfg!(llvm_enzyme) && enable_ad && !thin { - let cx = - SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size); - - for function in cx.get_functions() { - let enzyme_marker = "enzyme_marker"; - if attributes::has_string_attr(function, enzyme_marker) { - // Sanity check: Ensure 'noinline' is present before replacing it. - assert!( - attributes::has_attr(function, Function, llvm::AttributeKind::NoInline), - "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'" - ); - - attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline); - attributes::remove_string_attr_from_llfn(function, enzyme_marker); - - assert!( - !attributes::has_string_attr(function, enzyme_marker), - "Expected function to not have 'enzyme_marker'" - ); - - let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx); - attributes::apply_to_llfn(function, Function, &[always_inline]); - } - } - + if cfg!(feature = "llvm_enzyme") && enable_ad && !thin { let opt_stage = llvm::OptStage::FatLTO; let stage = write::AutodiffStage::PostAD; if !config.autodiff.contains(&config::AutoDiff::NoPostopt) { unsafe { - write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?; + write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage); } } @@ -634,7 +636,6 @@ pub(crate) fn run_pass_manager( } debug!("lto done"); - Ok(()) } pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); @@ -685,9 +686,9 @@ unsafe impl Send for ThinBuffer {} unsafe impl Sync for ThinBuffer {} impl ThinBuffer { - pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer { + pub(crate) fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { unsafe { - let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); ThinBuffer(buffer) } } @@ -696,6 +697,14 @@ impl ThinBuffer { let mut ptr = NonNull::new(ptr).unwrap(); ThinBuffer(unsafe { ptr.as_mut() }) } + + pub(crate) fn thin_link_data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); + slice::from_raw_parts(ptr, len) + } + } } impl ThinBufferMethods for ThinBuffer { @@ -706,14 +715,6 @@ impl ThinBufferMethods for ThinBuffer { slice::from_raw_parts(ptr, len) } } - - fn thin_link_data(&self) -> &[u8] { - unsafe { - let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _; - let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0); - slice::from_raw_parts(ptr, len) - } - } } impl Drop for ThinBuffer { @@ -727,7 +728,7 @@ impl Drop for ThinBuffer { pub(crate) fn optimize_thin_module( thin_module: ThinModule, cgcx: &CodegenContext, -) -> Result, FatalError> { +) -> ModuleCodegen { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -738,10 +739,10 @@ pub(crate) fn optimize_thin_module( // into that context. One day, however, we may do this for upstream // crates but for locally codegened modules we may be able to reuse // that LLVM Context and Module. - let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?; + let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx); let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm); // Given that the newly created module lacks a thinlto buffer for embedding, we need to re-add it here. - if cgcx.config(ModuleKind::Regular).embed_bitcode() { + if cgcx.module_config.embed_bitcode() { module.thin_lto_buffer = Some(thin_module.data().to_vec()); } { @@ -772,7 +773,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); } @@ -783,7 +784,7 @@ pub(crate) fn optimize_thin_module( .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); if unsafe { !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); } @@ -794,7 +795,7 @@ pub(crate) fn optimize_thin_module( if unsafe { !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target.raw()) } { - return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule)); + write::llvm_err(dcx, LlvmError::PrepareThinLtoModule); } save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); } @@ -806,11 +807,11 @@ pub(crate) fn optimize_thin_module( // little differently. { info!("running thin lto passes over {}", module.name); - run_pass_manager(cgcx, dcx, &mut module, true)?; + run_pass_manager(cgcx, dcx, &mut module, true); save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); } } - Ok(module) + module } /// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys @@ -876,9 +877,9 @@ pub(crate) fn parse_module<'a>( name: &CStr, data: &[u8], dcx: DiagCtxtHandle<'_>, -) -> Result<&'a llvm::Module, FatalError> { +) -> &'a llvm::Module { unsafe { llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()) - .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) + .unwrap_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode)) } } diff --git a/compiler/rustc_codegen_llvm/src/back/mod.rs b/compiler/rustc_codegen_llvm/src/back/mod.rs new file mode 100644 index 000000000000..6cb89f80ab89 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod archive; +pub(crate) mod lto; +pub(crate) mod owned_target_machine; +mod profiling; +pub(crate) mod write; diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs index 8e82013e94ad..f88932d43d2a 100644 --- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs +++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs @@ -1,4 +1,4 @@ -use std::ffi::{CStr, c_char}; +use std::ffi::CStr; use std::marker::PhantomData; use std::ptr::NonNull; @@ -36,17 +36,10 @@ impl OwnedTargetMachine { use_init_array: bool, split_dwarf_file: &CStr, output_obj_file: &CStr, - debug_info_compression: &CStr, + debug_info_compression: llvm::CompressionKind, use_emulated_tls: bool, - args_cstr_buff: &[u8], use_wasm_eh: bool, ) -> Result> { - assert!(args_cstr_buff.len() > 0); - assert!( - *args_cstr_buff.last().unwrap() == 0, - "The last character must be a null terminator." - ); - // SAFETY: llvm::LLVMRustCreateTargetMachine copies pointed to data let tm_ptr = unsafe { llvm::LLVMRustCreateTargetMachine( @@ -69,10 +62,8 @@ impl OwnedTargetMachine { use_init_array, split_dwarf_file.as_ptr(), output_obj_file.as_ptr(), - debug_info_compression.as_ptr(), + debug_info_compression, use_emulated_tls, - args_cstr_buff.as_ptr() as *const c_char, - args_cstr_buff.len(), use_wasm_eh, ) }; @@ -98,8 +89,6 @@ impl Drop for OwnedTargetMachine { // SAFETY: constructing ensures we have a valid pointer created by // llvm::LLVMRustCreateTargetMachine OwnedTargetMachine is not copyable so there is no // double free or use after free. - unsafe { - llvm::LLVMRustDisposeTargetMachine(self.tm_unique.as_mut()); - } + unsafe { llvm::LLVMDisposeTargetMachine(self.tm_unique) }; } } diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 85a06f457ebe..cfbb9541ecd2 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -6,9 +6,6 @@ use std::sync::Arc; use std::{fs, slice, str}; use libc::{c_char, c_int, c_void, size_t}; -use llvm::{ - LLVMRustLLVMHasZlibCompressionForDebugSymbols, LLVMRustLLVMHasZstdCompressionForDebugSymbols, -}; use rustc_codegen_ssa::back::link::ensure_removed; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::back::write::{ @@ -20,7 +17,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CompiledModule, ModuleCodegen, ModuleKind}; use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::small_c_str::SmallCStr; -use rustc_errors::{DiagCtxtHandle, FatalError, Level}; +use rustc_errors::{DiagCtxtHandle, Level}; use rustc_fs_util::{link_or_copy, path_to_c_string}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -43,13 +40,13 @@ use crate::errors::{ }; use crate::llvm::diagnostic::OptimizationDiagnosticKind::*; use crate::llvm::{self, DiagnosticInfo}; -use crate::type_::Type; +use crate::type_::llvm_type_ptr; use crate::{LlvmCodegenBackend, ModuleLlvm, base, common, llvm_util}; -pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> FatalError { +pub(crate) fn llvm_err<'a>(dcx: DiagCtxtHandle<'_>, err: LlvmError<'a>) -> ! { match llvm::last_error() { - Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)), - None => dcx.emit_almost_fatal(err), + Some(llvm_err) => dcx.emit_fatal(WithLlvmError(err, llvm_err)), + None => dcx.emit_fatal(err), } } @@ -63,7 +60,7 @@ fn write_output_file<'ll>( file_type: llvm::FileType, self_profiler_ref: &SelfProfilerRef, verify_llvm_ir: bool, -) -> Result<(), FatalError> { +) { debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); let output_c = path_to_c_string(output); let dwo_output_c; @@ -100,7 +97,7 @@ fn write_output_file<'ll>( } } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output })) } pub(crate) fn create_informational_target_machine( @@ -112,7 +109,7 @@ pub(crate) fn create_informational_target_machine( // system/tcx is set up. let features = llvm_util::global_llvm_features(sess, false, only_base_features); target_machine_factory(sess, config::OptLevel::No, &features)(config) - .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(sess.dcx(), err)) } pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine { @@ -139,7 +136,7 @@ pub(crate) fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTar tcx.backend_optimization_level(()), tcx.global_backend_features(()), )(config) - .unwrap_or_else(|err| llvm_err(tcx.dcx(), err).raise()) + .unwrap_or_else(|err| llvm_err(tcx.dcx(), err)) } fn to_llvm_opt_settings(cfg: config::OptLevel) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { @@ -203,6 +200,9 @@ pub(crate) fn target_machine_factory( optlvl: config::OptLevel, target_features: &[String], ) -> TargetMachineFactoryFn { + // Self-profile timer for creating a _factory_. + let _prof_timer = sess.prof.generic_activity("target_machine_factory"); + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); let (opt_level, _) = to_llvm_opt_settings(optlvl); @@ -249,46 +249,36 @@ pub(crate) fn target_machine_factory( let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated); - // copy the exe path, followed by path all into one buffer - // null terminating them so we can use them as null terminated strings - let args_cstr_buff = { - let mut args_cstr_buff: Vec = Vec::new(); - let exe_path = std::env::current_exe().unwrap_or_default(); - let exe_path_str = exe_path.into_os_string().into_string().unwrap_or_default(); - - args_cstr_buff.extend_from_slice(exe_path_str.as_bytes()); - args_cstr_buff.push(0); - - for arg in sess.expanded_args.iter() { - args_cstr_buff.extend_from_slice(arg.as_bytes()); - args_cstr_buff.push(0); - } - - args_cstr_buff - }; - - let debuginfo_compression = sess.opts.debuginfo_compression.to_string(); - match sess.opts.debuginfo_compression { - rustc_session::config::DebugInfoCompression::Zlib => { - if !unsafe { LLVMRustLLVMHasZlibCompressionForDebugSymbols() } { + let debuginfo_compression = match sess.opts.debuginfo_compression { + config::DebugInfoCompression::None => llvm::CompressionKind::None, + config::DebugInfoCompression::Zlib => { + if llvm::LLVMRustLLVMHasZlibCompression() { + llvm::CompressionKind::Zlib + } else { sess.dcx().emit_warn(UnknownCompression { algorithm: "zlib" }); + llvm::CompressionKind::None } } - rustc_session::config::DebugInfoCompression::Zstd => { - if !unsafe { LLVMRustLLVMHasZstdCompressionForDebugSymbols() } { + config::DebugInfoCompression::Zstd => { + if llvm::LLVMRustLLVMHasZstdCompression() { + llvm::CompressionKind::Zstd + } else { sess.dcx().emit_warn(UnknownCompression { algorithm: "zstd" }); + llvm::CompressionKind::None } } - rustc_session::config::DebugInfoCompression::None => {} }; - let debuginfo_compression = SmallCStr::new(&debuginfo_compression); let file_name_display_preference = sess.filename_display_preference(RemapPathScopeComponents::DEBUGINFO); let use_wasm_eh = wants_wasm_eh(sess); + let prof = SelfProfilerRef::clone(&sess.prof); Arc::new(move |config: TargetMachineFactoryConfig| { + // Self-profile timer for invoking a factory to create a target machine. + let _prof_timer = prof.generic_activity("target_machine_factory_inner"); + let path_to_cstring_helper = |path: Option| -> CString { let path = path.unwrap_or_default(); let path = path_mapping @@ -321,9 +311,8 @@ pub(crate) fn target_machine_factory( use_init_array, &split_dwarf_file, &output_obj_file, - &debuginfo_compression, + debuginfo_compression, use_emulated_tls, - &args_cstr_buff, use_wasm_eh, ) }) @@ -354,7 +343,7 @@ fn write_bitcode_to_file(module: &ModuleCodegen, path: &Path) { } } -/// In what context is a dignostic handler being attached to a codegen unit? +/// In what context is a diagnostic handler being attached to a codegen unit? pub(crate) enum CodegenDiagnosticsStage { /// Prelink optimization stage. Opt, @@ -565,7 +554,7 @@ pub(crate) unsafe fn llvm_optimize( opt_level: config::OptLevel, opt_stage: llvm::OptStage, autodiff_stage: AutodiffStage, -) -> Result<(), FatalError> { +) { // Enzyme: // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized // source code. However, benchmarks show that optimizations increasing the code size @@ -574,7 +563,8 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore); let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter); @@ -704,7 +694,7 @@ pub(crate) unsafe fn llvm_optimize( llvm_plugins.len(), ) }; - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) + result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) } // Unsafe due to LLVM calls. @@ -713,7 +703,7 @@ pub(crate) fn optimize( dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, -) -> Result<(), FatalError> { +) { let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); let llcx = &*module.module_llvm.llcx; @@ -740,7 +730,8 @@ pub(crate) fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = + cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; // The embedded bitcode is used to run LTO/ThinLTO. // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. @@ -765,7 +756,7 @@ pub(crate) fn optimize( opt_stage, autodiff_stage, ) - }?; + }; if let Some(thin_lto_buffer) = thin_lto_buffer { let thin_lto_buffer = unsafe { ThinBuffer::from_raw_ptr(thin_lto_buffer) }; module.thin_lto_buffer = Some(thin_lto_buffer.data().to_vec()); @@ -793,14 +784,13 @@ pub(crate) fn optimize( } } } - Ok(()) } pub(crate) fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, -) -> Result { +) -> CompiledModule { let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); @@ -838,7 +828,7 @@ pub(crate) fn codegen( "LLVM_module_codegen_make_bitcode", &*module.name, ); - ThinBuffer::new(llmod, config.emit_thin_lto, false) + ThinBuffer::new(llmod, config.emit_thin_lto) }; let data = thin.data(); let _timer = cgcx @@ -862,7 +852,7 @@ pub(crate) fn codegen( .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); let thin_bc = module.thin_lto_buffer.as_deref().expect("cannot find embedded bitcode"); - embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, &thin_bc); + embed_bitcode(cgcx, llcx, llmod, &thin_bc); } } @@ -909,7 +899,9 @@ pub(crate) fn codegen( record_artifact_size(&cgcx.prof, "llvm_ir", &out); } - result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?; + result + .into_result() + .unwrap_or_else(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out })); } if config.emit_asm { @@ -940,7 +932,7 @@ pub(crate) fn codegen( llvm::FileType::AssemblyFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } match config.emit_obj { @@ -976,7 +968,7 @@ pub(crate) fn codegen( llvm::FileType::ObjectFile, &cgcx.prof, config.verify_llvm_ir, - )?; + ); } EmitObj::Bitcode => { @@ -1009,7 +1001,7 @@ pub(crate) fn codegen( && cgcx.target_can_use_split_dwarf && cgcx.split_debuginfo != SplitDebuginfo::Off && cgcx.split_dwarf_kind == SplitDwarfKind::Split; - Ok(module.into_compiled_module( + module.into_compiled_module( config.emit_obj != EmitObj::None, dwarf_object_emitted, config.emit_bc, @@ -1017,7 +1009,7 @@ pub(crate) fn codegen( config.emit_ir, &cgcx.output_filenames, cgcx.invocation_temp.as_deref(), - )) + ) } fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { @@ -1058,7 +1050,6 @@ fn embed_bitcode( cgcx: &CodegenContext, llcx: &llvm::Context, llmod: &llvm::Module, - cmdline: &str, bitcode: &[u8], ) { // We're adding custom sections to the output object file, but we definitely @@ -1074,7 +1065,9 @@ fn embed_bitcode( // * Mach-O - this is for macOS. Inspecting the source code for the native // linker here shows that the `.llvmbc` and `.llvmcmd` sections are // automatically skipped by the linker. In that case there's nothing extra - // that we need to do here. + // that we need to do here. We do need to make sure that the + // `__LLVM,__cmdline` section exists even though it is empty as otherwise + // ld64 rejects the object file. // // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and // `.llvmcmd` sections, so there's nothing extra we need to do. @@ -1109,9 +1102,9 @@ fn embed_bitcode( llvm::set_section(llglobal, bitcode_section_name(cgcx)); llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); - llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + llvm::LLVMSetGlobalConstant(llglobal, llvm::TRUE); - let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llconst = common::bytes_in_context(llcx, &[]); let llglobal = llvm::add_global(llmod, common::val_ty(llconst), c"rustc.embedded.cmdline"); llvm::set_initializer(llglobal, llconst); let section = if cgcx.target_is_like_darwin { @@ -1128,7 +1121,7 @@ fn embed_bitcode( let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); llvm::append_module_inline_asm(llmod, &asm); - let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, &[]); llvm::append_module_inline_asm(llmod, &asm); } } @@ -1152,7 +1145,7 @@ fn create_msvc_imps( // underscores added in front). let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; - let ptr_ty = Type::ptr_llcx(llcx); + let ptr_ty = llvm_type_ptr(llcx); let globals = base::iter_globals(llmod) .filter(|&val| { llvm::get_linkage(val) == llvm::Linkage::ExternalLinkage && !llvm::is_declaration(val) diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 5dda836988c8..4523d629b1ef 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -18,19 +18,20 @@ use rustc_codegen_ssa::base::maybe_create_entry_wrapper; use rustc_codegen_ssa::mono_item::MonoItemExt; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::attrs::Linkage; use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; use rustc_session::config::DebugInfo; use rustc_span::Symbol; use rustc_target::spec::SanitizerSet; use super::ModuleLlvm; +use crate::attributes; use crate::builder::Builder; use crate::context::CodegenCx; -use crate::value::Value; -use crate::{attributes, llvm}; +use crate::llvm::{self, Value}; pub(crate) struct ValueIter<'ll> { cur: Option<&'ll Value>, @@ -104,22 +105,40 @@ pub(crate) fn compile_codegen_unit( if let Some(entry) = maybe_create_entry_wrapper::>(&cx, cx.codegen_unit) { - let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + let attrs = attributes::sanitize_attrs(&cx, tcx, SanitizerSet::empty()); attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); } + // Define Objective-C module info and module flags. Note, the module info will + // also be added to the `llvm.compiler.used` variable, created later. + // + // These are only necessary when we need the linker to do its Objective-C-specific + // magic. We could theoretically do it unconditionally, but at a slight cost to linker + // performance in the common case where it's unnecessary. + if !cx.objc_classrefs.borrow().is_empty() || !cx.objc_selrefs.borrow().is_empty() { + if cx.objc_abi_version() == 1 { + cx.define_objc_module_info(); + } + cx.add_objc_module_flags(); + } + // Finalize code coverage by injecting the coverage map. Note, the coverage map will // also be added to the `llvm.compiler.used` variable, created next. if cx.sess().instrument_coverage() { cx.coverageinfo_finalize(); } - // Create the llvm.used and llvm.compiler.used variables. + // Create the llvm.used variable. if !cx.used_statics.is_empty() { cx.create_used_variable_impl(c"llvm.used", &cx.used_statics); } - if !cx.compiler_used_statics.is_empty() { - cx.create_used_variable_impl(c"llvm.compiler.used", &cx.compiler_used_statics); + + // Create the llvm.compiler.used variable. + { + let compiler_used_statics = cx.compiler_used_statics.borrow(); + if !compiler_used_statics.is_empty() { + cx.create_used_variable_impl(c"llvm.compiler.used", &compiler_used_statics); + } } // Run replace-all-uses-with for statics that need it. This must diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index f712b3b83faa..c082a8230684 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1,11 +1,12 @@ use std::borrow::{Borrow, Cow}; +use std::iter; use std::ops::Deref; -use std::{iter, ptr}; +use rustc_ast::expand::typetree::FncTree; pub(crate) mod autodiff; pub(crate) mod gpu_offload; -use libc::{c_char, c_uint, size_t}; +use libc::{c_char, c_uint}; use rustc_abi as abi; use rustc_abi::{Align, Size, WrappingRange}; use rustc_codegen_ssa::MemFlags; @@ -15,6 +16,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::*; use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; +use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, @@ -24,7 +26,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; use rustc_session::config::OptLevel; use rustc_span::Span; -use rustc_target::callconv::FnAbi; +use rustc_target::callconv::{FnAbi, PassMode}; use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target}; use smallvec::SmallVec; use tracing::{debug, instrument}; @@ -34,11 +36,10 @@ use crate::attributes; use crate::common::Funclet; use crate::context::{CodegenCx, FullCx, GenericCx, SCx}; use crate::llvm::{ - self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True, + self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE, + ToLlvmBool, Type, Value, }; -use crate::type_::Type; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; #[must_use] pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow>> { @@ -394,10 +395,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { md.push(weight(is_cold)); } - unsafe { - let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t); - self.cx.set_metadata(switch, llvm::MD_prof, md_node); - } + self.cx.set_metadata_node(switch, llvm::MD_prof, &md); } fn invoke( @@ -492,8 +490,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(add).is_some() { - llvm::LLVMSetNUW(add, True); - llvm::LLVMSetNSW(add, True); + llvm::LLVMSetNUW(add, TRUE); + llvm::LLVMSetNSW(add, TRUE); } add } @@ -502,8 +500,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(sub).is_some() { - llvm::LLVMSetNUW(sub, True); - llvm::LLVMSetNSW(sub, True); + llvm::LLVMSetNUW(sub, TRUE); + llvm::LLVMSetNSW(sub, TRUE); } sub } @@ -512,8 +510,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED); if llvm::LLVMIsAInstruction(mul).is_some() { - llvm::LLVMSetNUW(mul, True); - llvm::LLVMSetNSW(mul, True); + llvm::LLVMSetNUW(mul, TRUE); + llvm::LLVMSetNSW(mul, TRUE); } mul } @@ -527,7 +525,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // an instruction, so we need to check before setting the flag. // (See also `LLVMBuildNUWNeg` which also needs a check.) if llvm::LLVMIsAInstruction(or).is_some() { - llvm::LLVMSetIsDisjoint(or, True); + llvm::LLVMSetIsDisjoint(or, TRUE); } or } @@ -556,13 +554,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let (size, signed) = ty.int_size_and_signed(self.tcx); let width = size.bits(); - if oop == OverflowOp::Sub && !signed { - // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these - // to be the canonical form. It will attempt to reform llvm.usub.with.overflow - // in the backend if profitable. - let sub = self.sub(lhs, rhs); - let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); - return (sub, cmp); + if !signed { + match oop { + OverflowOp::Sub => { + // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these + // to be the canonical form. It will attempt to reform llvm.usub.with.overflow + // in the backend if profitable. + let sub = self.sub(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); + return (sub, cmp); + } + OverflowOp::Add => { + // Like with sub above, using icmp is the preferred form. See + // + let add = self.add(lhs, rhs); + let cmp = self.icmp(IntPredicate::IntULT, add, lhs); + return (add, cmp); + } + OverflowOp::Mul => {} + } } let oop_str = match oop { @@ -616,7 +626,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); - llvm::LLVMSetVolatile(load, llvm::True); + llvm::LLVMSetVolatile(load, llvm::TRUE); load } } @@ -629,13 +639,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { size: Size, ) -> &'ll Value { unsafe { - let load = llvm::LLVMRustBuildAtomicLoad( - self.llbuilder, - ty, - ptr, - UNNAMED, - AtomicOrdering::from_generic(order), - ); + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + // Set atomic ordering + llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order)); // LLVM requires the alignment of atomic loads to be at least the size of the type. llvm::LLVMSetAlignment(load, size.bytes() as c_uint); load @@ -704,7 +710,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let mut const_llval = None; let llty = place.layout.llvm_type(self); if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { - if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if llvm::LLVMIsGlobalConstant(global).is_true() { if let Some(init) = llvm::LLVMGetInitializer(global) { if self.val_ty(init) == llty { const_llval = Some(init); @@ -787,22 +793,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { return; } - unsafe { - let llty = self.cx.val_ty(load); - let md = [ - llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)), - llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))), - ]; - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len()); - self.set_metadata(load, llvm::MD_range, md); - } + let llty = self.cx.val_ty(load); + let md = [ + llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)), + llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))), + ]; + self.set_metadata_node(load, llvm::MD_range, &md); } fn nonnull_metadata(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_nonnull, md); - } + self.set_metadata_node(load, llvm::MD_nonnull, &[]); } fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { @@ -825,7 +825,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; llvm::LLVMSetAlignment(store, align); if flags.contains(MemFlags::VOLATILE) { - llvm::LLVMSetVolatile(store, llvm::True); + llvm::LLVMSetVolatile(store, llvm::TRUE); } if flags.contains(MemFlags::NONTEMPORAL) { // Make sure that the current target architectures supports "sane" non-temporal @@ -851,8 +851,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // // [1]: https://llvm.org/docs/LangRef.html#store-instruction let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1)); - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1); - self.set_metadata(store, llvm::MD_nontemporal, md); + self.set_metadata_node(store, llvm::MD_nontemporal, &[one]); } } store @@ -869,12 +868,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { debug!("Store {:?} -> {:?}", val, ptr); assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer); unsafe { - let store = llvm::LLVMRustBuildAtomicStore( - self.llbuilder, - val, - ptr, - AtomicOrdering::from_generic(order), - ); + let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + // Set atomic ordering + llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order)); // LLVM requires the alignment of atomic stores to be at least the size of the type. llvm::LLVMSetAlignment(store, size.bytes() as c_uint); } @@ -943,7 +939,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNUW(trunc, True); + llvm::LLVMSetNUW(trunc, TRUE); } } trunc @@ -955,7 +951,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let trunc = self.trunc(val, dest_ty); unsafe { if llvm::LLVMIsAInstruction(trunc).is_some() { - llvm::LLVMSetNSW(trunc, True); + llvm::LLVMSetNSW(trunc, TRUE); } } trunc @@ -1054,13 +1050,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { unsafe { - llvm::LLVMBuildIntCast2( - self.llbuilder, - val, - dest_ty, - if is_signed { True } else { False }, - UNNAMED, - ) + llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED) } } @@ -1084,16 +1074,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { ty: Ty<'tcx>, lhs: Self::Value, rhs: Self::Value, - ) -> Option { - // FIXME: See comment on the definition of `three_way_compare`. - if crate::llvm_util::get_version() < (20, 0, 0) { - return None; - } - + ) -> Self::Value { let size = ty.primitive_size(self.tcx); let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" }; - Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) + self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]) } /* Miscellaneous instructions */ @@ -1105,11 +1090,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align: Align, size: &'ll Value, flags: MemFlags, + tt: Option, ) { assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); let size = self.intcast(size, self.type_isize(), false); let is_volatile = flags.contains(MemFlags::VOLATILE); - unsafe { + let memcpy = unsafe { llvm::LLVMRustBuildMemCpy( self.llbuilder, dst, @@ -1118,7 +1104,16 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src_align.bytes() as c_uint, size, is_volatile, - ); + ) + }; + + // TypeTree metadata for memcpy is especially important: when Enzyme encounters + // a memcpy during autodiff, it needs to know the structure of the data being + // copied to properly track derivatives. For example, copying an array of floats + // vs. copying a struct with mixed types requires different derivative handling. + // The TypeTree tells Enzyme exactly what memory layout to expect. + if let Some(tt) = tt { + crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt); } } @@ -1216,7 +1211,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false); let landing_pad = self.landing_pad(ty, pers_fn, 0); unsafe { - llvm::LLVMSetCleanup(landing_pad, llvm::True); + llvm::LLVMSetCleanup(landing_pad, llvm::TRUE); } (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) } @@ -1304,7 +1299,6 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { failure_order: rustc_middle::ty::AtomicOrdering, weak: bool, ) -> (&'ll Value, &'ll Value) { - let weak = if weak { llvm::True } else { llvm::False }; unsafe { let value = llvm::LLVMBuildAtomicCmpXchg( self.llbuilder, @@ -1313,9 +1307,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { src, AtomicOrdering::from_generic(order), AtomicOrdering::from_generic(failure_order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ); - llvm::LLVMSetWeak(value, weak); + llvm::LLVMSetWeak(value, weak.to_llvm_bool()); let val = self.extract_value(value, 0); let success = self.extract_value(value, 1); (val, success) @@ -1326,15 +1320,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { &mut self, op: rustc_codegen_ssa::common::AtomicRmwBinOp, dst: &'ll Value, - mut src: &'ll Value, + src: &'ll Value, order: rustc_middle::ty::AtomicOrdering, + ret_ptr: bool, ) -> &'ll Value { - // The only RMW operation that LLVM supports on pointers is compare-exchange. - let requires_cast_to_int = self.val_ty(src) == self.type_ptr() - && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg; - if requires_cast_to_int { - src = self.ptrtoint(src, self.type_isize()); - } + // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the + // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not + // currently support that (https://github.com/llvm/llvm-project/issues/120837). let mut res = unsafe { llvm::LLVMBuildAtomicRMW( self.llbuilder, @@ -1342,10 +1334,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { dst, src, AtomicOrdering::from_generic(order), - llvm::False, // SingleThreaded + llvm::FALSE, // SingleThreaded ) }; - if requires_cast_to_int { + if ret_ptr && self.val_ty(res) != self.type_ptr() { res = self.inttoptr(res, self.type_ptr()); } res @@ -1357,24 +1349,21 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { scope: SynchronizationScope, ) { let single_threaded = match scope { - SynchronizationScope::SingleThread => llvm::True, - SynchronizationScope::CrossThread => llvm::False, + SynchronizationScope::SingleThread => true, + SynchronizationScope::CrossThread => false, }; unsafe { llvm::LLVMBuildFence( self.llbuilder, AtomicOrdering::from_generic(order), - single_threaded, + single_threaded.to_llvm_bool(), UNNAMED, ); } } fn set_invariant_load(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_invariant_load, md); - } + self.set_metadata_node(load, llvm::MD_invariant_load, &[]); } fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { @@ -1388,7 +1377,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn call( &mut self, llty: &'ll Type, - fn_attrs: Option<&CodegenFnAttrs>, + fn_call_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, args: &[&'ll Value], @@ -1405,10 +1394,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } // Emit CFI pointer type membership test - self.cfi_type_test(fn_attrs, fn_abi, instance, llfn); + self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn); // Emit KCFI operand bundle - let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn); + let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn); if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) { bundles.push(kcfi_bundle); } @@ -1425,12 +1414,57 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { c"".as_ptr(), ) }; + + if let Some(instance) = instance { + // Attributes on the function definition being called + let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id()); + if let Some(fn_call_attrs) = fn_call_attrs + && !fn_call_attrs.target_features.is_empty() + // If there is an inline attribute and a target feature that matches + // we will add the attribute to the callsite otherwise we'll omit + // this and not add the attribute to prevent soundness issues. + && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance) + && self.cx.tcx.is_target_feature_call_safe( + &fn_call_attrs.target_features, + &fn_defn_attrs.target_features, + ) + { + attributes::apply_to_callsite( + call, + llvm::AttributePlace::Function, + &[inlining_rule], + ); + } + } + if let Some(fn_abi) = fn_abi { fn_abi.apply_attrs_callsite(self, call); } call } + fn tail_call( + &mut self, + llty: Self::Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Value, + args: &[Self::Value], + funclet: Option<&Self::Funclet>, + instance: Option>, + ) { + let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance); + llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); + + match &fn_abi.ret.mode { + PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(), + PassMode::Direct(_) | PassMode::Pair { .. } => self.ret(call), + mode @ PassMode::Cast { .. } => { + bug!("Encountered `PassMode::{mode:?}` during codegen") + } + } + } + fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } @@ -1473,34 +1507,24 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { } impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { fn align_metadata(&mut self, load: &'ll Value, align: Align) { - unsafe { - let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))]; - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len()); - self.set_metadata(load, llvm::MD_align, md); - } + let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))]; + self.set_metadata_node(load, llvm::MD_align, &md); } fn noundef_metadata(&mut self, load: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(load, llvm::MD_noundef, md); - } + self.set_metadata_node(load, llvm::MD_noundef, &[]); } pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) { - unsafe { - let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0); - self.set_metadata(inst, llvm::MD_unpredictable, md); - } + self.set_metadata_node(inst, llvm::MD_unpredictable, &[]); } -} -impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { + pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } + self.call_intrinsic("llvm.minnum", &[self.val_ty(lhs)], &[lhs, rhs]) } pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } + self.call_intrinsic("llvm.maxnum", &[self.val_ty(lhs)], &[lhs, rhs]) } pub(crate) fn insert_element( @@ -1522,10 +1546,10 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { } pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } + self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]) } pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } + self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]) } pub(crate) fn vector_reduce_fadd_reassoc( &mut self, @@ -1533,7 +1557,8 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { src: &'ll Value, ) -> &'ll Value { unsafe { - let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); + let instr = + self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]); llvm::LLVMRustSetAllowReassoc(instr); instr } @@ -1544,43 +1569,49 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { src: &'ll Value, ) -> &'ll Value { unsafe { - let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); + let instr = + self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]); llvm::LLVMRustSetAllowReassoc(instr); instr } } pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } + self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } + self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } + self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } + self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } + self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) - } + self.call_intrinsic("llvm.vector.reduce.fmin", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { - unsafe { - llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) - } + self.call_intrinsic("llvm.vector.reduce.fmax", &[self.val_ty(src)], &[src]) } pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } + self.call_intrinsic( + if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" }, + &[self.val_ty(src)], + &[src], + ) } pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { - unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } + self.call_intrinsic( + if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" }, + &[self.val_ty(src)], + &[src], + ) } - +} +impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { unsafe { llvm::LLVMAddClause(landing_pad, clause); @@ -1663,7 +1694,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + if crate::llvm_util::get_version() >= (22, 0, 0) { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]); + } else { + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); + } } } impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { @@ -1863,48 +1898,4 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) { self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]); } - - /// Emits a call to `llvm.instrprof.mcdc.parameters`. - /// - /// This doesn't produce any code directly, but is used as input by - /// the LLVM pass that handles coverage instrumentation. - /// - /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.) - /// - /// [`CodeGenPGO::emitMCDCParameters`]: - /// https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124 - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_parameters( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - bitmap_bits: &'ll Value, - ) { - self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_tvbitmap_update( - &mut self, - fn_name: &'ll Value, - hash: &'ll Value, - bitmap_index: &'ll Value, - mcdc_temp: &'ll Value, - ) { - let args = &[fn_name, hash, bitmap_index, mcdc_temp]; - self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) { - self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi); - } - - #[instrument(level = "debug", skip(self))] - pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) { - let align = self.tcx.data_layout.i32_align.abi; - let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align); - let new_tv_index = self.add(current_tv_index, cond_index); - self.store(new_tv_index, mcdc_temp, align); - } } diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index 829b3c513c25..4b433e2b6361 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -1,40 +1,131 @@ use std::ptr; -use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivity, DiffMode}; -use rustc_codegen_ssa::ModuleCodegen; +use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; +use rustc_ast::expand::typetree::FncTree; use rustc_codegen_ssa::common::TypeKind; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; -use rustc_errors::FatalError; -use rustc_middle::bug; -use tracing::{debug, trace}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_middle::ty::{Instance, PseudoCanonicalInput, TyCtxt, TypingEnv}; +use rustc_middle::{bug, ty}; +use rustc_target::callconv::PassMode; +use tracing::debug; -use crate::back::write::llvm_err; -use crate::builder::{SBuilder, UNNAMED}; +use crate::builder::{Builder, PlaceRef, UNNAMED}; use crate::context::SimpleCx; use crate::declare::declare_simple_fn; -use crate::errors::{AutoDiffWithoutEnable, LlvmError}; -use crate::llvm::AttributePlace::Function; -use crate::llvm::{Metadata, True}; -use crate::value::Value; -use crate::{CodegenContext, LlvmCodegenBackend, ModuleLlvm, attributes, llvm}; +use crate::llvm::{self, TRUE, Type, Value}; -fn get_params(fnc: &Value) -> Vec<&Value> { - let param_num = llvm::LLVMCountParams(fnc) as usize; - let mut fnc_args: Vec<&Value> = vec![]; - fnc_args.reserve(param_num); - unsafe { - llvm::LLVMGetParams(fnc, fnc_args.as_mut_ptr()); - fnc_args.set_len(param_num); +pub(crate) fn adjust_activity_to_abi<'tcx>( + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, + typing_env: TypingEnv<'tcx>, + da: &mut Vec, +) { + let fn_ty = instance.ty(tcx, typing_env); + + if !matches!(fn_ty.kind(), ty::FnDef(..)) { + bug!("expected fn def for autodiff, got {:?}", fn_ty); } - fnc_args -} -fn has_sret(fnc: &Value) -> bool { - let num_args = llvm::LLVMCountParams(fnc) as usize; - if num_args == 0 { - false - } else { - unsafe { llvm::LLVMRustHasAttributeAtIndex(fnc, 0, llvm::AttributeKind::StructRet) } + // We don't actually pass the types back into the type system. + // All we do is decide how to handle the arguments. + let sig = fn_ty.fn_sig(tcx).skip_binder(); + + // FIXME(Sa4dUs): pass proper varargs once we have support for differentiating variadic functions + let Ok(fn_abi) = + tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) + else { + bug!("failed to get fn_abi of instance with empty varargs"); + }; + + let mut new_activities = vec![]; + let mut new_positions = vec![]; + let mut del_activities = 0; + for (i, ty) in sig.inputs().iter().enumerate() { + if let Some(inner_ty) = ty.builtin_deref(true) { + if inner_ty.is_slice() { + // Now we need to figure out the size of each slice element in memory to allow + // safety checks and usability improvements in the backend. + let sty = match inner_ty.builtin_index() { + Some(sty) => sty, + None => { + panic!("slice element type unknown"); + } + }; + let pci = PseudoCanonicalInput { + typing_env: TypingEnv::fully_monomorphized(), + value: sty, + }; + + let layout = tcx.layout_of(pci); + let elem_size = match layout { + Ok(layout) => layout.size, + Err(_) => { + bug!("autodiff failed to compute slice element size"); + } + }; + let elem_size: u32 = elem_size.bytes() as u32; + + // We know that the length will be passed as extra arg. + if !da.is_empty() { + // We are looking at a slice. The length of that slice will become an + // extra integer on llvm level. Integers are always const. + // However, if the slice get's duplicated, we want to know to later check the + // size. So we mark the new size argument as FakeActivitySize. + // There is one FakeActivitySize per slice, so for convenience we store the + // slice element size in bytes in it. We will use the size in the backend. + let activity = match da[i] { + DiffActivity::DualOnly + | DiffActivity::Dual + | DiffActivity::Dualv + | DiffActivity::DuplicatedOnly + | DiffActivity::Duplicated => { + DiffActivity::FakeActivitySize(Some(elem_size)) + } + DiffActivity::Const => DiffActivity::Const, + _ => bug!("unexpected activity for ptr/ref"), + }; + new_activities.push(activity); + new_positions.push(i + 1); + } + + continue; + } + } + + let pci = PseudoCanonicalInput { typing_env: TypingEnv::fully_monomorphized(), value: *ty }; + + let layout = match tcx.layout_of(pci) { + Ok(layout) => layout.layout, + Err(_) => { + bug!("failed to compute layout for type {:?}", ty); + } + }; + + let pass_mode = &fn_abi.args[i].mode; + + // For ZST, just ignore and don't add its activity, as this arg won't be present + // in the LLVM passed to Enzyme. + // Some targets pass ZST indirectly in the C ABI, in that case, handle it as a normal arg + // FIXME(Sa4dUs): Enforce ZST corresponding diff activity be `Const` + if *pass_mode == PassMode::Ignore { + del_activities += 1; + da.remove(i); + } + + // If the argument is lowered as a `ScalarPair`, we need to duplicate its activity. + // Otherwise, the number of activities won't match the number of LLVM arguments and + // this will lead to errors when verifying the Enzyme call. + if let rustc_abi::BackendRepr::ScalarPair(_, _) = layout.backend_repr() { + new_activities.push(da[i].clone()); + new_positions.push(i + 1 - del_activities); + } + } + // now add the extra activities coming from slices + // Reverse order to not invalidate the indices + for _ in 0..new_activities.len() { + let pos = new_positions.pop().unwrap(); + let activity = new_activities.pop().unwrap(); + da.insert(pos, activity); } } @@ -48,14 +139,13 @@ fn has_sret(fnc: &Value) -> bool { // need to match those. // FIXME(ZuseZ4): This logic is a bit more complicated than it should be, can we simplify it // using iterators and peek()? -fn match_args_from_caller_to_enzyme<'ll>( +fn match_args_from_caller_to_enzyme<'ll, 'tcx>( cx: &SimpleCx<'ll>, - builder: &SBuilder<'ll, 'll>, + builder: &mut Builder<'_, 'll, 'tcx>, width: u32, - args: &mut Vec<&'ll llvm::Value>, + args: &mut Vec<&'ll Value>, inputs: &[DiffActivity], - outer_args: &[&'ll llvm::Value], - has_sret: bool, + outer_args: &[&'ll Value], ) { debug!("matching autodiff arguments"); // We now handle the issue that Rust level arguments not always match the llvm-ir level @@ -67,40 +157,36 @@ fn match_args_from_caller_to_enzyme<'ll>( let mut outer_pos: usize = 0; let mut activity_pos = 0; - if has_sret { - // Then the first outer arg is the sret pointer. Enzyme doesn't know about sret, so the - // inner function will still return something. We increase our outer_pos by one, - // and once we're done with all other args we will take the return of the inner call and - // update the sret pointer with it - outer_pos = 1; - } - - let enzyme_const = cx.create_metadata(b"enzyme_const"); - let enzyme_out = cx.create_metadata(b"enzyme_out"); - let enzyme_dup = cx.create_metadata(b"enzyme_dup"); - let enzyme_dupv = cx.create_metadata(b"enzyme_dupv"); - let enzyme_dupnoneed = cx.create_metadata(b"enzyme_dupnoneed"); - let enzyme_dupnoneedv = cx.create_metadata(b"enzyme_dupnoneedv"); + // We used to use llvm's metadata to instruct enzyme how to differentiate a function. + // In debug mode we would use incremental compilation which caused the metadata to be + // dropped. This is prevented by now using named globals, which are also understood + // by Enzyme. + let global_const = cx.declare_global("enzyme_const", cx.type_ptr()); + let global_out = cx.declare_global("enzyme_out", cx.type_ptr()); + let global_dup = cx.declare_global("enzyme_dup", cx.type_ptr()); + let global_dupv = cx.declare_global("enzyme_dupv", cx.type_ptr()); + let global_dupnoneed = cx.declare_global("enzyme_dupnoneed", cx.type_ptr()); + let global_dupnoneedv = cx.declare_global("enzyme_dupnoneedv", cx.type_ptr()); while activity_pos < inputs.len() { let diff_activity = inputs[activity_pos as usize]; // Duplicated arguments received a shadow argument, into which enzyme will write the // gradient. - let (activity, duplicated): (&Metadata, bool) = match diff_activity { + let (activity, duplicated): (&Value, bool) = match diff_activity { DiffActivity::None => panic!("not a valid input activity"), - DiffActivity::Const => (enzyme_const, false), - DiffActivity::Active => (enzyme_out, false), - DiffActivity::ActiveOnly => (enzyme_out, false), - DiffActivity::Dual => (enzyme_dup, true), - DiffActivity::Dualv => (enzyme_dupv, true), - DiffActivity::DualOnly => (enzyme_dupnoneed, true), - DiffActivity::DualvOnly => (enzyme_dupnoneedv, true), - DiffActivity::Duplicated => (enzyme_dup, true), - DiffActivity::DuplicatedOnly => (enzyme_dupnoneed, true), - DiffActivity::FakeActivitySize(_) => (enzyme_const, false), + DiffActivity::Const => (global_const, false), + DiffActivity::Active => (global_out, false), + DiffActivity::ActiveOnly => (global_out, false), + DiffActivity::Dual => (global_dup, true), + DiffActivity::Dualv => (global_dupv, true), + DiffActivity::DualOnly => (global_dupnoneed, true), + DiffActivity::DualvOnly => (global_dupnoneedv, true), + DiffActivity::Duplicated => (global_dup, true), + DiffActivity::DuplicatedOnly => (global_dupnoneed, true), + DiffActivity::FakeActivitySize(_) => (global_const, false), }; let outer_arg = outer_args[outer_pos]; - args.push(cx.get_metadata_value(activity)); + args.push(activity); if matches!(diff_activity, DiffActivity::Dualv) { let next_outer_arg = outer_args[outer_pos + 1]; let elem_bytes_size: u64 = match inputs[activity_pos + 1] { @@ -160,7 +246,7 @@ fn match_args_from_caller_to_enzyme<'ll>( assert_eq!(cx.type_kind(next_outer_ty3), TypeKind::Integer); args.push(next_outer_arg2); } - args.push(cx.get_metadata_value(enzyme_const)); + args.push(global_const); args.push(next_outer_arg); outer_pos += 2 + 2 * iterations; activity_pos += 2; @@ -193,92 +279,6 @@ fn match_args_from_caller_to_enzyme<'ll>( } } -// On LLVM-IR, we can luckily declare __enzyme_ functions without specifying the input -// arguments. We do however need to declare them with their correct return type. -// We already figured the correct return type out in our frontend, when generating the outer_fn, -// so we can now just go ahead and use that. This is not always trivial, e.g. because sret. -// Beyond sret, this article describes our challenges nicely: -// -// I.e. (i32, f32) will get merged into i64, but we don't handle that yet. -fn compute_enzyme_fn_ty<'ll>( - cx: &SimpleCx<'ll>, - attrs: &AutoDiffAttrs, - fn_to_diff: &'ll Value, - outer_fn: &'ll Value, -) -> &'ll llvm::Type { - let fn_ty = cx.get_type_of_global(outer_fn); - let mut ret_ty = cx.get_return_type(fn_ty); - - let has_sret = has_sret(outer_fn); - - if has_sret { - // Now we don't just forward the return type, so we have to figure it out based on the - // primal return type, in combination with the autodiff settings. - let fn_ty = cx.get_type_of_global(fn_to_diff); - let inner_ret_ty = cx.get_return_type(fn_ty); - - let void_ty = unsafe { llvm::LLVMVoidTypeInContext(cx.llcx) }; - if inner_ret_ty == void_ty { - // This indicates that even the inner function has an sret. - // Right now I only look for an sret in the outer function. - // This *probably* needs some extra handling, but I never ran - // into such a case. So I'll wait for user reports to have a test case. - bug!("sret in inner function"); - } - - if attrs.width == 1 { - // Enzyme returns a struct of style: - // `{ original_ret(if requested), float, float, ... }` - let mut struct_elements = vec![]; - if attrs.has_primal_ret() { - struct_elements.push(inner_ret_ty); - } - // Next, we push the list of active floats, since they will be lowered to `enzyme_out`, - // and therefore part of the return struct. - let param_tys = cx.func_params_types(fn_ty); - for (act, param_ty) in attrs.input_activity.iter().zip(param_tys) { - if matches!(act, DiffActivity::Active) { - // Now find the float type at position i based on the fn_ty, - // to know what (f16/f32/f64/...) to add to the struct. - struct_elements.push(param_ty); - } - } - ret_ty = cx.type_struct(&struct_elements, false); - } else { - // First we check if we also have to deal with the primal return. - match attrs.mode { - DiffMode::Forward => match attrs.ret_activity { - DiffActivity::Dual => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64 + 1) }; - ret_ty = arr_ty; - } - DiffActivity::DualOnly => { - let arr_ty = - unsafe { llvm::LLVMArrayType2(inner_ret_ty, attrs.width as u64) }; - ret_ty = arr_ty; - } - DiffActivity::Const => { - todo!("Not sure, do we need to do something here?"); - } - _ => { - bug!("unreachable"); - } - }, - DiffMode::Reverse => { - todo!("Handle sret for reverse mode"); - } - _ => { - bug!("unreachable"); - } - } - } - } - - // LLVM can figure out the input types on it's own, so we take a shortcut here. - unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, True) } -} - /// When differentiating `fn_to_diff`, take a `outer_fn` and generate another /// function with expected naming and calling conventions[^1] which will be /// discovered by the enzyme LLVM pass and its body populated with the differentiated @@ -288,11 +288,16 @@ fn compute_enzyme_fn_ty<'ll>( /// [^1]: // FIXME(ZuseZ4): `outer_fn` should include upstream safety checks to // cover some assumptions of enzyme/autodiff, which could lead to UB otherwise. -fn generate_enzyme_call<'ll>( +pub(crate) fn generate_enzyme_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, cx: &SimpleCx<'ll>, fn_to_diff: &'ll Value, - outer_fn: &'ll Value, + outer_name: &str, + ret_ty: &'ll Type, + fn_args: &[&'ll Value], attrs: AutoDiffAttrs, + dest: PlaceRef<'tcx, &'ll Value>, + fnc_tree: FncTree, ) { // We have to pick the name depending on whether we want forward or reverse mode autodiff. let mut ad_name: String = match attrs.mode { @@ -302,11 +307,9 @@ fn generate_enzyme_call<'ll>( } .to_string(); - // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple + // add outer_name to ad_name to make it unique, in case users apply autodiff to multiple // functions. Unwrap will only panic, if LLVM gave us an invalid string. - let name = llvm::get_value_name(outer_fn); - let outer_fn_name = std::str::from_utf8(&name).unwrap(); - ad_name.push_str(outer_fn_name); + ad_name.push_str(outer_name); // Let us assume the user wrote the following function square: // @@ -316,14 +319,8 @@ fn generate_enzyme_call<'ll>( // %0 = fmul double %x, %x // ret double %0 // } - // ``` // - // The user now applies autodiff to the function square, in which case fn_to_diff will be `square`. - // Our macro generates the following placeholder code (slightly simplified): - // - // ```llvm // define double @dsquare(double %x) { - // ; placeholder code // return 0.0; // } // ``` @@ -340,175 +337,55 @@ fn generate_enzyme_call<'ll>( // ret double %0 // } // ``` - unsafe { - let enzyme_ty = compute_enzyme_fn_ty(cx, &attrs, fn_to_diff, outer_fn); + let enzyme_ty = unsafe { llvm::LLVMFunctionType(ret_ty, ptr::null(), 0, TRUE) }; - // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and - // think a bit more about what should go here. - let cc = llvm::LLVMGetFunctionCallConv(outer_fn); - let ad_fn = declare_simple_fn( - cx, - &ad_name, - llvm::CallConv::try_from(cc).expect("invalid callconv"), - llvm::UnnamedAddr::No, - llvm::Visibility::Default, - enzyme_ty, - ); + // FIXME(ZuseZ4): the CC/Addr/Vis values are best effort guesses, we should look at tests and + // think a bit more about what should go here. + let cc = unsafe { llvm::LLVMGetFunctionCallConv(fn_to_diff) }; + let ad_fn = declare_simple_fn( + cx, + &ad_name, + llvm::CallConv::try_from(cc).expect("invalid callconv"), + llvm::UnnamedAddr::No, + llvm::Visibility::Default, + enzyme_ty, + ); - // Otherwise LLVM might inline our temporary code before the enzyme pass has a chance to - // do it's work. - let attr = llvm::AttributeKind::NoInline.create_attr(cx.llcx); - attributes::apply_to_llfn(ad_fn, Function, &[attr]); + let num_args = llvm::LLVMCountParams(&fn_to_diff); + let mut args = Vec::with_capacity(num_args as usize + 1); + args.push(fn_to_diff); - // We add a made-up attribute just such that we can recognize it after AD to update - // (no)-inline attributes. We'll then also remove this attribute. - let enzyme_marker_attr = llvm::CreateAttrString(cx.llcx, "enzyme_marker"); - attributes::apply_to_llfn(outer_fn, Function, &[enzyme_marker_attr]); + let global_primal_ret = cx.declare_global("enzyme_primal_return", cx.type_ptr()); + if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { + args.push(global_primal_ret); + } + if attrs.width > 1 { + let global_width = cx.declare_global("enzyme_width", cx.type_ptr()); + args.push(global_width); + args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); + } - // first, remove all calls from fnc - let entry = llvm::LLVMGetFirstBasicBlock(outer_fn); - let br = llvm::LLVMRustGetTerminator(entry); - llvm::LLVMRustEraseInstFromParent(br); + match_args_from_caller_to_enzyme( + &cx, + builder, + attrs.width, + &mut args, + &attrs.input_activity, + fn_args, + ); - let last_inst = llvm::LLVMRustGetLastInstruction(entry).unwrap(); - let mut builder = SBuilder::build(cx, entry); + if !fnc_tree.args.is_empty() || !fnc_tree.ret.0.is_empty() { + crate::typetree::add_tt(cx.llmod, cx.llcx, fn_to_diff, fnc_tree); + } - let num_args = llvm::LLVMCountParams(&fn_to_diff); - let mut args = Vec::with_capacity(num_args as usize + 1); - args.push(fn_to_diff); + let call = builder.call(enzyme_ty, None, None, ad_fn, &args, None, None); - let enzyme_primal_ret = cx.create_metadata(b"enzyme_primal_return"); - if matches!(attrs.ret_activity, DiffActivity::Dual | DiffActivity::Active) { - args.push(cx.get_metadata_value(enzyme_primal_ret)); - } - if attrs.width > 1 { - let enzyme_width = cx.create_metadata(b"enzyme_width"); - args.push(cx.get_metadata_value(enzyme_width)); - args.push(cx.get_const_int(cx.type_i64(), attrs.width as u64)); - } - - let has_sret = has_sret(outer_fn); - let outer_args: Vec<&llvm::Value> = get_params(outer_fn); - match_args_from_caller_to_enzyme( - &cx, - &builder, - attrs.width, - &mut args, - &attrs.input_activity, - &outer_args, - has_sret, - ); - - let call = builder.call(enzyme_ty, ad_fn, &args, None); - - // This part is a bit iffy. LLVM requires that a call to an inlineable function has some - // metadata attached to it, but we just created this code oota. Given that the - // differentiated function already has partly confusing metadata, and given that this - // affects nothing but the auttodiff IR, we take a shortcut and just steal metadata from the - // dummy code which we inserted at a higher level. - // FIXME(ZuseZ4): Work with Enzyme core devs to clarify what debug metadata issues we have, - // and how to best improve it for enzyme core and rust-enzyme. - let md_ty = cx.get_md_kind_id("dbg"); - if llvm::LLVMRustHasMetadata(last_inst, md_ty) { - let md = llvm::LLVMRustDIGetInstMetadata(last_inst) - .expect("failed to get instruction metadata"); - let md_todiff = cx.get_metadata_value(md); - llvm::LLVMSetMetadata(call, md_ty, md_todiff); - } else { - // We don't panic, since depending on whether we are in debug or release mode, we might - // have no debug info to copy, which would then be ok. - trace!("no dbg info"); - } - - // Now that we copied the metadata, get rid of dummy code. - llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst); - - if cx.val_ty(call) == cx.type_void() || has_sret { - if has_sret { - // This is what we already have in our outer_fn (shortened): - // define void @_foo(ptr <..> sret([32 x i8]) initializes((0, 32)) %0, <...>) { - // %7 = call [4 x double] (...) @__enzyme_fwddiff_foo(ptr @square, metadata !"enzyme_width", i64 4, <...>) - // - // store [4 x double] %7, ptr %0, align 8 - // ret void - // } - - // now store the result of the enzyme call into the sret pointer. - let sret_ptr = outer_args[0]; - let call_ty = cx.val_ty(call); - if attrs.width == 1 { - assert_eq!(cx.type_kind(call_ty), TypeKind::Struct); - } else { - assert_eq!(cx.type_kind(call_ty), TypeKind::Array); - } - llvm::LLVMBuildStore(&builder.llbuilder, call, sret_ptr); - } - builder.ret_void(); - } else { - builder.ret(call); - } - - // Let's crash in case that we messed something up above and generated invalid IR. - llvm::LLVMRustVerifyFunction( - outer_fn, - llvm::LLVMRustVerifierFailureAction::LLVMAbortProcessAction, - ); + let fn_ret_ty = builder.cx.val_ty(call); + if fn_ret_ty != builder.cx.type_void() && fn_ret_ty != builder.cx.type_struct(&[], false) { + // If we return void or an empty struct, then our caller (due to how we generated it) + // does not expect a return value. As such, we have no pointer (or place) into which + // we could store our value, and would store into an undef, which would cause UB. + // As such, we just ignore the return value in those cases. + builder.store_to_place(call, dest.val); } } - -pub(crate) fn differentiate<'ll>( - module: &'ll ModuleCodegen, - cgcx: &CodegenContext, - diff_items: Vec, -) -> Result<(), FatalError> { - for item in &diff_items { - trace!("{}", item); - } - - let diag_handler = cgcx.create_dcx(); - - let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - - // First of all, did the user try to use autodiff without using the -Zautodiff=Enable flag? - if !diff_items.is_empty() - && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) - { - return Err(diag_handler.handle().emit_almost_fatal(AutoDiffWithoutEnable)); - } - - // Here we replace the placeholder code with the actual autodiff code, which calls Enzyme. - for item in diff_items.iter() { - let name = item.source.clone(); - let fn_def: Option<&llvm::Value> = cx.get_function(&name); - let Some(fn_def) = fn_def else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find source function".to_owned(), - }, - )); - }; - debug!(?item.target); - let fn_target: Option<&llvm::Value> = cx.get_function(&item.target); - let Some(fn_target) = fn_target else { - return Err(llvm_err( - diag_handler.handle(), - LlvmError::PrepareAutoDiff { - src: item.source.clone(), - target: item.target.clone(), - error: "could not find target function".to_owned(), - }, - )); - }; - - generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone()); - } - - // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts - - trace!("done with differentiate()"); - - Ok(()) -} diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 1280ab1442a0..3d55064ea130 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -16,25 +16,42 @@ pub(crate) fn handle_gpu_code<'ll>( cx: &'ll SimpleCx<'_>, ) { // The offload memory transfer type for each kernel - let mut o_types = vec![]; - let mut kernels = vec![]; - let offload_entry_ty = add_tgt_offload_entry(&cx); + let mut memtransfer_types = vec![]; + let mut region_ids = vec![]; + let offload_entry_ty = TgtOffloadEntry::new_decl(&cx); for num in 0..9 { let kernel = cx.get_function(&format!("kernel_{num}")); if let Some(kernel) = kernel { - o_types.push(gen_define_handling(&cx, kernel, offload_entry_ty, num)); - kernels.push(kernel); + let (o, k) = gen_define_handling(&cx, kernel, offload_entry_ty, num); + memtransfer_types.push(o); + region_ids.push(k); } } - gen_call_handling(&cx, &kernels, &o_types); + gen_call_handling(&cx, &memtransfer_types, ®ion_ids); +} + +// ; Function Attrs: nounwind +// declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2 +fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm::Type) { + let tptr = cx.type_ptr(); + let ti64 = cx.type_i64(); + let ti32 = cx.type_i32(); + let args = vec![tptr, ti64, ti32, ti32, tptr, tptr]; + let tgt_fn_ty = cx.type_func(&args, ti32); + let name = "__tgt_target_kernel"; + let tgt_decl = declare_offload_fn(&cx, name, tgt_fn_ty); + let nounwind = llvm::AttributeKind::NoUnwind.create_attr(cx.llcx); + attributes::apply_to_llfn(tgt_decl, Function, &[nounwind]); + (tgt_decl, tgt_fn_ty) } // What is our @1 here? A magic global, used in our data_{begin/update/end}_mapper: // @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 +// FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be +// offloaded was defined. fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value { - // @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1 let unknown_txt = ";unknown;unknown;0;0;;"; let c_entry_name = CString::new(unknown_txt).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); @@ -59,15 +76,7 @@ fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value { at_one } -pub(crate) fn add_tgt_offload_entry<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type { - let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); - let tptr = cx.type_ptr(); - let ti64 = cx.type_i64(); - let ti32 = cx.type_i32(); - let ti16 = cx.type_i16(); - // For each kernel to run on the gpu, we will later generate one entry of this type. - // copied from LLVM - // typedef struct { +struct TgtOffloadEntry { // uint64_t Reserved; // uint16_t Version; // uint16_t Kind; @@ -77,21 +86,40 @@ pub(crate) fn add_tgt_offload_entry<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Ty // uint64_t Size; Size of the entry info (0 if it is a function) // uint64_t Data; // void *AuxAddr; - // } __tgt_offload_entry; - let entry_elements = vec![ti64, ti16, ti16, ti32, tptr, tptr, ti64, ti64, tptr]; - cx.set_struct_body(offload_entry_ty, &entry_elements, false); - offload_entry_ty } -fn gen_tgt_kernel_global<'ll>(cx: &'ll SimpleCx<'_>) { - let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); - let tptr = cx.type_ptr(); - let ti64 = cx.type_i64(); - let ti32 = cx.type_i32(); - let tarr = cx.type_array(ti32, 3); +impl TgtOffloadEntry { + pub(crate) fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type { + let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); + let tptr = cx.type_ptr(); + let ti64 = cx.type_i64(); + let ti32 = cx.type_i32(); + let ti16 = cx.type_i16(); + // For each kernel to run on the gpu, we will later generate one entry of this type. + // copied from LLVM + let entry_elements = vec![ti64, ti16, ti16, ti32, tptr, tptr, ti64, ti64, tptr]; + cx.set_struct_body(offload_entry_ty, &entry_elements, false); + offload_entry_ty + } - // Taken from the LLVM APITypes.h declaration: - //struct KernelArgsTy { + fn new<'ll>( + cx: &'ll SimpleCx<'_>, + region_id: &'ll Value, + llglobal: &'ll Value, + ) -> [&'ll Value; 9] { + let reserved = cx.get_const_i64(0); + let version = cx.get_const_i16(1); + let kind = cx.get_const_i16(1); + let flags = cx.get_const_i32(0); + let size = cx.get_const_i64(0); + let data = cx.get_const_i64(0); + let aux_addr = cx.const_null(cx.type_ptr()); + [reserved, version, kind, flags, region_id, llglobal, size, data, aux_addr] + } +} + +// Taken from the LLVM APITypes.h declaration: +struct KernelArgsTy { // uint32_t Version = 0; // Version of this struct for ABI compatibility. // uint32_t NumArgs = 0; // Number of arguments in each input pointer. // void **ArgBasePtrs = @@ -102,25 +130,65 @@ fn gen_tgt_kernel_global<'ll>(cx: &'ll SimpleCx<'_>) { // void **ArgNames = nullptr; // Name of the data for debugging, possibly null. // void **ArgMappers = nullptr; // User-defined mappers, possibly null. // uint64_t Tripcount = - // 0; // Tripcount for the teams / distribute loop, 0 otherwise. - // struct { + // 0; // Tripcount for the teams / distribute loop, 0 otherwise. + // struct { // uint64_t NoWait : 1; // Was this kernel spawned with a `nowait` clause. // uint64_t IsCUDA : 1; // Was this kernel spawned via CUDA. // uint64_t Unused : 62; - // } Flags = {0, 0, 0}; + // } Flags = {0, 0, 0}; // totals to 64 Bit, 8 Byte // // The number of teams (for x,y,z dimension). // uint32_t NumTeams[3] = {0, 0, 0}; // // The number of threads (for x,y,z dimension). // uint32_t ThreadLimit[3] = {0, 0, 0}; // uint32_t DynCGroupMem = 0; // Amount of dynamic cgroup memory requested. - //}; - let kernel_elements = - vec![ti32, ti32, tptr, tptr, tptr, tptr, tptr, tptr, ti64, ti64, tarr, tarr, ti32]; +} - cx.set_struct_body(kernel_arguments_ty, &kernel_elements, false); - // For now we don't handle kernels, so for now we just add a global dummy - // to make sure that the __tgt_offload_entry is defined and handled correctly. - cx.declare_global("my_struct_global2", kernel_arguments_ty); +impl KernelArgsTy { + const OFFLOAD_VERSION: u64 = 3; + const FLAGS: u64 = 0; + const TRIPCOUNT: u64 = 0; + fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll Type { + let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); + let tptr = cx.type_ptr(); + let ti64 = cx.type_i64(); + let ti32 = cx.type_i32(); + let tarr = cx.type_array(ti32, 3); + + let kernel_elements = + vec![ti32, ti32, tptr, tptr, tptr, tptr, tptr, tptr, ti64, ti64, tarr, tarr, ti32]; + + cx.set_struct_body(kernel_arguments_ty, &kernel_elements, false); + kernel_arguments_ty + } + + fn new<'ll>( + cx: &'ll SimpleCx<'_>, + num_args: u64, + memtransfer_types: &[&'ll Value], + geps: [&'ll Value; 3], + ) -> [(Align, &'ll Value); 13] { + let four = Align::from_bytes(4).expect("4 Byte alignment should work"); + let eight = Align::EIGHT; + + let ti32 = cx.type_i32(); + let ci32_0 = cx.get_const_i32(0); + [ + (four, cx.get_const_i32(KernelArgsTy::OFFLOAD_VERSION)), + (four, cx.get_const_i32(num_args)), + (eight, geps[0]), + (eight, geps[1]), + (eight, geps[2]), + (eight, memtransfer_types[0]), + // The next two are debug infos. FIXME(offload): set them + (eight, cx.const_null(cx.type_ptr())), // dbg + (eight, cx.const_null(cx.type_ptr())), // dbg + (eight, cx.get_const_i64(KernelArgsTy::TRIPCOUNT)), + (eight, cx.get_const_i64(KernelArgsTy::FLAGS)), + (four, cx.const_array(ti32, &[cx.get_const_i32(2097152), ci32_0, ci32_0])), + (four, cx.const_array(ti32, &[cx.get_const_i32(256), ci32_0, ci32_0])), + (four, cx.get_const_i32(0)), + ] + } } fn gen_tgt_data_mappers<'ll>( @@ -182,18 +250,21 @@ pub(crate) fn add_global<'ll>( llglobal } +// This function returns a memtransfer value which encodes how arguments to this kernel shall be +// mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be +// concatenated into the list of region_ids. fn gen_define_handling<'ll>( cx: &'ll SimpleCx<'_>, kernel: &'ll llvm::Value, offload_entry_ty: &'ll llvm::Type, num: i64, -) -> &'ll llvm::Value { +) -> (&'ll llvm::Value, &'ll llvm::Value) { let types = cx.func_params_types(cx.get_type_of_global(kernel)); // It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or // reference) types. let num_ptr_types = types .iter() - .map(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) + .filter(|&x| matches!(cx.type_kind(x), rustc_codegen_ssa::common::TypeKind::Pointer)) .count(); // We do not know their size anymore at this level, so hardcode a placeholder. @@ -205,10 +276,14 @@ fn gen_define_handling<'ll>( // or both to and from the gpu (=3). Other values shouldn't affect us for now. // A non-mutable reference or pointer will be 1, an array that's not read, but fully overwritten // will be 2. For now, everything is 3, until we have our frontend set up. - let o_types = - add_priv_unnamed_arr(&cx, &format!(".offload_maptypes.{num}"), &vec![3; num_ptr_types]); + // 1+2+32: 1 (MapTo), 2 (MapFrom), 32 (Add one extra input ptr per function, to be used later). + let memtransfer_types = add_priv_unnamed_arr( + &cx, + &format!(".offload_maptypes.{num}"), + &vec![1 + 2 + 32; num_ptr_types], + ); // Next: For each function, generate these three entries. A weak constant, - // the llvm.rodata entry name, and the omp_offloading_entries value + // the llvm.rodata entry name, and the llvm_offload_entries value let name = format!(".kernel_{num}.region_id"); let initializer = cx.get_const_i8(0); @@ -222,19 +297,10 @@ fn gen_define_handling<'ll>( let llglobal = add_unnamed_global(&cx, &offload_entry_name, initializer, InternalLinkage); llvm::set_alignment(llglobal, Align::ONE); llvm::set_section(llglobal, c".llvm.rodata.offloading"); - - // Not actively used yet, for calling real kernels let name = format!(".offloading.entry.kernel_{num}"); // See the __tgt_offload_entry documentation above. - let reserved = cx.get_const_i64(0); - let version = cx.get_const_i16(1); - let kind = cx.get_const_i16(1); - let flags = cx.get_const_i32(0); - let size = cx.get_const_i64(0); - let data = cx.get_const_i64(0); - let aux_addr = cx.const_null(cx.type_ptr()); - let elems = vec![reserved, version, kind, flags, region_id, llglobal, size, data, aux_addr]; + let elems = TgtOffloadEntry::new(&cx, region_id, llglobal); let initializer = crate::common::named_struct(offload_entry_ty, &elems); let c_name = CString::new(name).unwrap(); @@ -242,13 +308,13 @@ fn gen_define_handling<'ll>( llvm::set_global_constant(llglobal, true); llvm::set_linkage(llglobal, WeakAnyLinkage); llvm::set_initializer(llglobal, initializer); - llvm::set_alignment(llglobal, Align::ONE); - let c_section_name = CString::new(".omp_offloading_entries").unwrap(); + llvm::set_alignment(llglobal, Align::EIGHT); + let c_section_name = CString::new("llvm_offload_entries").unwrap(); llvm::set_section(llglobal, &c_section_name); - o_types + (memtransfer_types, region_id) } -fn declare_offload_fn<'ll>( +pub(crate) fn declare_offload_fn<'ll>( cx: &'ll SimpleCx<'_>, name: &str, ty: &'ll llvm::Type, @@ -285,9 +351,10 @@ fn declare_offload_fn<'ll>( // 6. generate __tgt_target_data_end calls to move data from the GPU fn gen_call_handling<'ll>( cx: &'ll SimpleCx<'_>, - _kernels: &[&'ll llvm::Value], - o_types: &[&'ll llvm::Value], + memtransfer_types: &[&'ll llvm::Value], + region_ids: &[&'ll llvm::Value], ) { + let (tgt_decl, tgt_target_kernel_ty) = generate_launcher(&cx); // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } let tptr = cx.type_ptr(); let ti32 = cx.type_i32(); @@ -295,7 +362,7 @@ fn gen_call_handling<'ll>( let tgt_bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); cx.set_struct_body(tgt_bin_desc, &tgt_bin_desc_ty, false); - gen_tgt_kernel_global(&cx); + let tgt_kernel_decl = KernelArgsTy::new_decl(&cx); let (begin_mapper_decl, _, end_mapper_decl, fn_ty) = gen_tgt_data_mappers(&cx); let main_fn = cx.get_function("main"); @@ -329,35 +396,32 @@ fn gen_call_handling<'ll>( // These represent the sizes in bytes, e.g. the entry for `&[f64; 16]` will be 8*16. let ty2 = cx.type_array(cx.type_i64(), num_args); let a4 = builder.direct_alloca(ty2, Align::EIGHT, ".offload_sizes"); - // Now we allocate once per function param, a copy to be passed to one of our maps. - let mut vals = vec![]; - let mut geps = vec![]; - let i32_0 = cx.get_const_i32(0); - for (index, in_ty) in types.iter().enumerate() { - // get function arg, store it into the alloca, and read it. - let p = llvm::get_param(called, index as u32); - let name = llvm::get_value_name(p); - let name = str::from_utf8(&name).unwrap(); - let arg_name = format!("{name}.addr"); - let alloca = builder.direct_alloca(in_ty, Align::EIGHT, &arg_name); - builder.store(p, alloca, Align::EIGHT); - let val = builder.load(in_ty, alloca, Align::EIGHT); - let gep = builder.inbounds_gep(cx.type_f32(), val, &[i32_0]); - vals.push(val); - geps.push(gep); - } + //%kernel_args = alloca %struct.__tgt_kernel_arguments, align 8 + let a5 = builder.direct_alloca(tgt_kernel_decl, Align::EIGHT, "kernel_args"); // Step 1) unsafe { llvm::LLVMRustPositionBefore(builder.llbuilder, kernel_call) }; builder.memset(tgt_bin_desc_alloca, cx.get_const_i8(0), cx.get_const_i64(32), Align::EIGHT); + // Now we allocate once per function param, a copy to be passed to one of our maps. + let mut vals = vec![]; + let mut geps = vec![]; + let i32_0 = cx.get_const_i32(0); + for index in 0..types.len() { + let v = unsafe { llvm::LLVMGetOperand(kernel_call, index as u32).unwrap() }; + let gep = builder.inbounds_gep(cx.type_f32(), v, &[i32_0]); + vals.push(v); + geps.push(gep); + } + let mapper_fn_ty = cx.type_func(&[cx.type_ptr()], cx.type_void()); let register_lib_decl = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); let unregister_lib_decl = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); let init_ty = cx.type_func(&[], cx.type_void()); let init_rtls_decl = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + // FIXME(offload): Later we want to add them to the wrapper code, rather than our main function. // call void @__tgt_register_lib(ptr noundef %6) builder.call(mapper_fn_ty, register_lib_decl, &[tgt_bin_desc_alloca], None); // call void @__tgt_init_all_rtls() @@ -386,19 +450,19 @@ fn gen_call_handling<'ll>( a1: &'ll Value, a2: &'ll Value, a4: &'ll Value, - ) -> (&'ll Value, &'ll Value, &'ll Value) { + ) -> [&'ll Value; 3] { let i32_0 = cx.get_const_i32(0); let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); let gep2 = builder.inbounds_gep(ty, a2, &[i32_0, i32_0]); let gep3 = builder.inbounds_gep(ty2, a4, &[i32_0, i32_0]); - (gep1, gep2, gep3) + [gep1, gep2, gep3] } fn generate_mapper_call<'a, 'll>( builder: &mut SBuilder<'a, 'll>, cx: &'ll SimpleCx<'ll>, - geps: (&'ll Value, &'ll Value, &'ll Value), + geps: [&'ll Value; 3], o_type: &'ll Value, fn_to_call: &'ll Value, fn_ty: &'ll Type, @@ -409,31 +473,51 @@ fn gen_call_handling<'ll>( let i64_max = cx.get_const_i64(u64::MAX); let num_args = cx.get_const_i32(num_args); let args = - vec![s_ident_t, i64_max, num_args, geps.0, geps.1, geps.2, o_type, nullptr, nullptr]; + vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; builder.call(fn_ty, fn_to_call, &args, None); } // Step 2) let s_ident_t = generate_at_one(&cx); - let o = o_types[0]; + let o = memtransfer_types[0]; let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); generate_mapper_call(&mut builder, &cx, geps, o, begin_mapper_decl, fn_ty, num_args, s_ident_t); + let values = KernelArgsTy::new(&cx, num_args, memtransfer_types, geps); // Step 3) - // Here we will add code for the actual kernel launches in a follow-up PR. - // FIXME(offload): launch kernels + // Here we fill the KernelArgsTy, see the documentation above + for (i, value) in values.iter().enumerate() { + let ptr = builder.inbounds_gep(tgt_kernel_decl, a5, &[i32_0, cx.get_const_i32(i as u64)]); + builder.store(value.1, ptr, value.0); + } + + let args = vec![ + s_ident_t, + // FIXME(offload) give users a way to select which GPU to use. + cx.get_const_i64(u64::MAX), // MAX == -1. + // FIXME(offload): Don't hardcode the numbers of threads in the future. + cx.get_const_i32(2097152), + cx.get_const_i32(256), + region_ids[0], + a5, + ]; + let offload_success = builder.call(tgt_target_kernel_ty, tgt_decl, &args, None); + // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) + unsafe { + let next = llvm::LLVMGetNextInstruction(offload_success).unwrap(); + llvm::LLVMRustPositionAfter(builder.llbuilder, next); + llvm::LLVMInstructionEraseFromParent(next); + } // Step 4) - unsafe { llvm::LLVMRustPositionAfter(builder.llbuilder, kernel_call) }; - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); generate_mapper_call(&mut builder, &cx, geps, o, end_mapper_decl, fn_ty, num_args, s_ident_t); builder.call(mapper_fn_ty, unregister_lib_decl, &[tgt_bin_desc_alloca], None); - // With this we generated the following begin and end mappers. We could easily generate the - // update mapper in an update. - // call void @__tgt_target_data_begin_mapper(ptr @1, i64 -1, i32 3, ptr %27, ptr %28, ptr %29, ptr @.offload_maptypes, ptr null, ptr null) - // call void @__tgt_target_data_update_mapper(ptr @1, i64 -1, i32 2, ptr %46, ptr %47, ptr %48, ptr @.offload_maptypes.1, ptr null, ptr null) - // call void @__tgt_target_data_end_mapper(ptr @1, i64 -1, i32 3, ptr %49, ptr %50, ptr %51, ptr @.offload_maptypes, ptr null, ptr null) + drop(builder); + // FIXME(offload) The issue is that we right now add a call to the gpu version of the function, + // and then delete the call to the CPU version. In the future, we should use an intrinsic which + // directly resolves to a call to the GPU version. + unsafe { llvm::LLVMDeleteFunction(called) }; } diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 5a3dd90ab240..b86b32b92df0 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -10,8 +10,7 @@ use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use tracing::debug; use crate::context::CodegenCx; -use crate::llvm; -use crate::value::Value; +use crate::llvm::{self, Value}; /// Codegens a reference to a fn/method item, monomorphizing and /// inlining as it goes. @@ -103,7 +102,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t // This is a monomorphization of a generic function. if !(cx.tcx.sess.opts.share_generics() || tcx.codegen_instance_attrs(instance.def).inline - == rustc_attr_data_structures::InlineAttr::Never) + == rustc_hir::attrs::InlineAttr::Never) { // When not sharing generics, all instances are in the same // crate and have hidden visibility. diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f29fefb66f0f..b0cf9925019d 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -12,7 +12,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_hashes::Hash128; use rustc_hir::def_id::DefId; use rustc_middle::bug; -use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, PointerArithmetic, Scalar}; use rustc_middle::ty::TyCtxt; use rustc_session::cstore::DllImport; use tracing::debug; @@ -20,9 +20,7 @@ use tracing::debug; use crate::consts::const_alloc_to_llvm; pub(crate) use crate::context::CodegenCx; use crate::context::{GenericCx, SCx}; -use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, Metadata, True}; -use crate::type_::Type; -use crate::value::Value; +use crate::llvm::{self, BasicBlock, ConstantInt, FALSE, Metadata, TRUE, ToLlvmBool, Type, Value}; /* * A note on nomenclature of linking: "extern", "foreign", and "upcall". @@ -108,6 +106,10 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { bytes_in_context(self.llcx(), bytes) } + pub(crate) fn null_terminate_const_bytes(&self, bytes: &[u8]) -> &'ll Value { + null_terminate_bytes_in_context(self.llcx(), bytes) + } + pub(crate) fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { unsafe { let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow"); @@ -158,7 +160,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_int" ); - unsafe { llvm::LLVMConstInt(t, i as u64, True) } + unsafe { llvm::LLVMConstInt(t, i as u64, TRUE) } } fn const_u8(&self, i: u8) -> &'ll Value { @@ -192,7 +194,7 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.type_kind(t) == TypeKind::Integer, "only allows integer types in const_uint" ); - unsafe { llvm::LLVMConstInt(t, i, False) } + unsafe { llvm::LLVMConstInt(t, i, FALSE) } } fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { @@ -279,8 +281,8 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { // This avoids generating a zero-sized constant value and actually needing a // real address at runtime. if alloc.inner().len() == 0 { - assert_eq!(offset.bytes(), 0); - let llval = self.const_usize(alloc.inner().align.bytes()); + let val = alloc.inner().align.bytes().wrapping_add(offset.bytes()); + let llval = self.const_usize(self.tcx.truncate_to_target_usize(val)); return if matches!(layout.primitive(), Pointer(_)) { unsafe { llvm::LLVMConstIntToPtr(llval, llty) } } else { @@ -377,7 +379,17 @@ pub(crate) fn val_ty(v: &Value) -> &Type { pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { unsafe { let ptr = bytes.as_ptr() as *const c_char; - llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), True) + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), TRUE) + } +} + +pub(crate) fn null_terminate_bytes_in_context<'ll>( + llcx: &'ll llvm::Context, + bytes: &[u8], +) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext2(llcx, ptr, bytes.len(), FALSE) } } @@ -392,7 +404,7 @@ fn struct_in_context<'ll>( packed: bool, ) -> &'ll Value { let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow"); - unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed as Bool) } + unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed.to_llvm_bool()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 0b96b63bc857..9844c9444b3d 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -4,25 +4,26 @@ use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar, read_target_uint, }; -use rustc_middle::mir::mono::{Linkage, MonoItem}; +use rustc_middle::mir::mono::MonoItem; use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; +use rustc_span::Symbol; use tracing::{debug, instrument, trace}; use crate::common::CodegenCx; use crate::errors::SymbolAlreadyDefined; -use crate::type_::Type; +use crate::llvm::{self, Type, Value}; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use crate::{base, debuginfo, llvm}; +use crate::{base, debuginfo}; pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, @@ -191,8 +192,8 @@ fn check_and_apply_linkage<'ll, 'tcx>( // linkage and there are no definitions), then // `extern_with_linkage_foo` will instead be initialized to // zero. - let mut real_name = "_rust_extern_with_linkage_".to_string(); - real_name.push_str(sym); + let real_name = + format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE)); let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { cx.sess().dcx().emit_fatal(SymbolAlreadyDefined { span: cx.tcx.def_span(def_id), @@ -239,11 +240,13 @@ impl<'ll> CodegenCx<'ll, '_> { let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { bug!("symbol `{}` is already defined", name); }); - llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); gv } - _ => self.define_private_global(self.val_ty(cv)), + _ => self.define_global("", self.val_ty(cv)).unwrap_or_else(|| { + bug!("anonymous global symbol is already defined"); + }), }; + llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); @@ -330,6 +333,10 @@ impl<'ll> CodegenCx<'ll, '_> { } g + } else if let Some(classname) = fn_attrs.objc_class { + self.get_objc_classref(classname) + } else if let Some(methname) = fn_attrs.objc_selector { + self.get_objc_selref(methname) } else { check_and_apply_linkage(self, fn_attrs, llty, sym, def_id) }; @@ -451,6 +458,8 @@ impl<'ll> CodegenCx<'ll, '_> { self.statics_to_rauw.borrow_mut().push((g, new_g)); new_g }; + + // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, g, alloc.align); llvm::set_initializer(g, v); @@ -486,16 +495,7 @@ impl<'ll> CodegenCx<'ll, '_> { let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); let alloc = self.create_metadata(bytes); let data = [section, alloc]; - let meta = - unsafe { llvm::LLVMMDNodeInContext2(self.llcx, data.as_ptr(), data.len()) }; - let val = self.get_metadata_value(meta); - unsafe { - llvm::LLVMAddNamedMetadataOperand( - self.llmod, - c"wasm.custom_sections".as_ptr(), - val, - ) - }; + self.module_add_named_metadata_node(self.llmod(), c"wasm.custom_sections", &data); } } else { base::set_link_section(g, attrs); @@ -540,8 +540,225 @@ impl<'ll> CodegenCx<'ll, '_> { /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, /// an array of ptr. - pub(crate) fn add_compiler_used_global(&mut self, global: &'ll Value) { - self.compiler_used_statics.push(global); + pub(crate) fn add_compiler_used_global(&self, global: &'ll Value) { + self.compiler_used_statics.borrow_mut().push(global); + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + fn define_objc_classname(&self, classname: &str) -> &'ll Value { + assert_eq!(self.objc_abi_version(), 1); + + let llval = self.null_terminate_const_bytes(classname.as_bytes()); + let llty = self.val_ty(llval); + let sym = self.generate_local_symbol_name("OBJC_CLASS_NAME_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.i8_align); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__TEXT,__cstring,cstring_literals"); + llvm::LLVMSetGlobalConstant(g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(g); + + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCNonFragileABITypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L6052 + fn get_objc_class_t(&self) -> &'ll Type { + if let Some(class_t) = self.objc_class_t.get() { + return class_t; + } + + assert_eq!(self.objc_abi_version(), 2); + + // struct _class_t { + // struct _class_t* isa; + // struct _class_t* const superclass; + // void* cache; + // IMP* vtable; + // struct class_ro_t* ro; + // } + + let class_t = self.type_named_struct("struct._class_t"); + let els = [self.type_ptr(); 5]; + let packed = false; + self.set_struct_body(class_t, &els, packed); + + self.objc_class_t.set(Some(class_t)); + class_t + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + fn get_objc_classref(&self, classname: Symbol) -> &'ll Value { + let mut classrefs = self.objc_classrefs.borrow_mut(); + if let Some(classref) = classrefs.get(&classname).copied() { + return classref; + } + + let g = match self.objc_abi_version() { + 1 => { + // See Clang's `CGObjCMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5205 + let llval = self.define_objc_classname(classname.as_str()); + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASS_REFERENCES_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__cls_refs,literal_pointers,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + 2 => { + // See Clang's `CGObjCNonFragileABIMac::EmitClassRefFromId`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7423 + let llval = { + let extern_sym = format!("OBJC_CLASS_$_{}", classname.as_str()); + let extern_llty = self.get_objc_class_t(); + self.declare_global(&extern_sym, extern_llty) + }; + let llty = self.type_ptr(); + let sym = self.generate_local_symbol_name("OBJC_CLASSLIST_REFERENCES_$_"); + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::InternalLinkage); + llvm::set_section(g, c"__DATA,__objc_classrefs,regular,no_dead_strip"); + self.add_compiler_used_global(g); + g + } + _ => unreachable!(), + }; + + classrefs.insert(classname, g); + g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. We + // deduplicate references within a CGU, but we need a reference definition in each referencing + // CGU. All attempts at using external references to a single reference definition result in + // linker errors. + // + // Newer versions of Apple Clang generate calls to `@"objc_msgSend$methname"` selector stub + // functions. We don't currently do that. The code we generate is closer to what Apple Clang + // generates with the `-fno-objc-msgsend-selector-stubs` option. + fn get_objc_selref(&self, methname: Symbol) -> &'ll Value { + let mut selrefs = self.objc_selrefs.borrow_mut(); + if let Some(selref) = selrefs.get(&methname).copied() { + return selref; + } + + let abi_version = self.objc_abi_version(); + + // See Clang's `CGObjCCommonMac::CreateCStringLiteral`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134 + let methname_llval = self.null_terminate_const_bytes(methname.as_str().as_bytes()); + let methname_llty = self.val_ty(methname_llval); + let methname_sym = self.generate_local_symbol_name("OBJC_METH_VAR_NAME_"); + let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", methname_sym); + }); + set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align); + llvm::set_initializer(methname_g, methname_llval); + llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage); + llvm::set_section( + methname_g, + match abi_version { + 1 => c"__TEXT,__cstring,cstring_literals", + 2 => c"__TEXT,__objc_methname,cstring_literals", + _ => unreachable!(), + }, + ); + llvm::LLVMSetGlobalConstant(methname_g, llvm::TRUE); + llvm::LLVMSetUnnamedAddress(methname_g, llvm::UnnamedAddr::Global); + self.add_compiler_used_global(methname_g); + + // See Clang's `CGObjCMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5243 + // And Clang's `CGObjCNonFragileABIMac::EmitSelectorAddr`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7586 + let selref_llval = methname_g; + let selref_llty = self.type_ptr(); + let selref_sym = self.generate_local_symbol_name("OBJC_SELECTOR_REFERENCES_"); + let selref_g = self.define_global(&selref_sym, selref_llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", selref_sym); + }); + set_global_alignment(self, selref_g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(selref_g, selref_llval); + llvm::set_externally_initialized(selref_g, true); + llvm::set_linkage( + selref_g, + match abi_version { + 1 => llvm::Linkage::PrivateLinkage, + 2 => llvm::Linkage::InternalLinkage, + _ => unreachable!(), + }, + ); + llvm::set_section( + selref_g, + match abi_version { + 1 => c"__OBJC,__message_refs,literal_pointers,no_dead_strip", + 2 => c"__DATA,__objc_selrefs,literal_pointers,no_dead_strip", + _ => unreachable!(), + }, + ); + self.add_compiler_used_global(selref_g); + + selrefs.insert(methname, selref_g); + selref_g + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `ObjCTypesHelper`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5936 + // And Clang's `CGObjCMac::EmitModuleInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5151 + pub(crate) fn define_objc_module_info(&mut self) { + assert_eq!(self.objc_abi_version(), 1); + + // struct _objc_module { + // long version; // Hardcoded to 7 in Clang. + // long size; // sizeof(struct _objc_module) + // char* name; // Hardcoded to classname "" in Clang. + // struct _objc_symtab* symtab; // Null without class or category definitions. + // } + + let llty = self.type_named_struct("struct._objc_module"); + let i32_llty = self.type_i32(); + let ptr_llty = self.type_ptr(); + let packed = false; + self.set_struct_body(llty, &[i32_llty, i32_llty, ptr_llty, ptr_llty], packed); + + let version = self.const_uint(i32_llty, 7); + let size = self.const_uint(i32_llty, 16); + let name = self.define_objc_classname(""); + let symtab = self.const_null(ptr_llty); + let llval = crate::common::named_struct(llty, &[version, size, name, symtab]); + + let sym = "OBJC_MODULES"; + let g = self.define_global(&sym, llty).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi); + llvm::set_initializer(g, llval); + llvm::set_linkage(g, llvm::Linkage::PrivateLinkage); + llvm::set_section(g, c"__OBJC,__module_info,regular,no_dead_strip"); + + self.add_compiler_used_global(g); } } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index ee77774c6883..808aaceab4d2 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -8,7 +8,6 @@ use std::str; use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx}; use rustc_codegen_ssa::back::versioned_llvm_target; use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh}; -use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::errors as ssa_errors; use rustc_codegen_ssa::traits::*; use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; @@ -27,18 +26,17 @@ use rustc_session::config::{ BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet, }; use rustc_span::source_map::Spanned; -use rustc_span::{DUMMY_SP, Span}; +use rustc_span::{DUMMY_SP, Span, Symbol}; use rustc_symbol_mangling::mangle_internal_symbol; use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel}; use smallvec::SmallVec; +use crate::abi::to_llvm_calling_convention; use crate::back::write::to_llvm_code_model; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; -use crate::llvm::Metadata; -use crate::type_::Type; -use crate::value::Value; -use crate::{attributes, common, coverageinfo, debuginfo, llvm, llvm_util}; +use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; +use crate::{attributes, common, coverageinfo, debuginfo, llvm_util}; /// `TyCtxt` (and related cache datastructures) can't be move between threads. /// However, there are various cx related functions which we want to be available to the builder and @@ -120,7 +118,7 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Statics that will be placed in the llvm.compiler.used variable /// See for details - pub compiler_used_statics: Vec<&'ll Value>, + pub compiler_used_statics: RefCell>, /// Mapping of non-scalar types to llvm types. pub type_lowering: RefCell, Option), &'ll Type>>, @@ -147,6 +145,15 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// `global_asm!` needs to be able to find this new global so that it can /// compute the correct mangled symbol name to insert into the asm. pub renamed_statics: RefCell>, + + /// Cached Objective-C class type + pub objc_class_t: Cell>, + + /// Cache of Objective-C class references + pub objc_classrefs: RefCell>, + + /// Cache of Objective-C selector references + pub objc_selrefs: RefCell>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -173,35 +180,6 @@ pub(crate) unsafe fn create_module<'ll>( let mut target_data_layout = sess.target.data_layout.to_string(); let llvm_version = llvm_util::get_version(); - if llvm_version < (20, 0, 0) { - if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") { - // LLVM 20 defines three additional address spaces for alternate - // pointer kinds used in Windows. - // See https://github.com/llvm/llvm-project/pull/111879 - target_data_layout = - target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", ""); - } - if sess.target.arch.starts_with("sparc") { - // LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/106951 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("mips64") { - // LLVM 20 updates the mips64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/112084 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("powerpc64") { - // LLVM 20 updates the powerpc64 layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/118004 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - if sess.target.arch.starts_with("wasm32") || sess.target.arch.starts_with("wasm64") { - // LLVM 20 updates the wasm(32|64) layout to correctly align 128 bit integers to 128 bit. - // See https://github.com/llvm/llvm-project/pull/119204 - target_data_layout = target_data_layout.replace("-i128:128", ""); - } - } if llvm_version < (21, 0, 0) { if sess.target.arch == "nvptx64" { // LLVM 21 updated the default layout on nvptx: https://github.com/llvm/llvm-project/pull/124961 @@ -213,6 +191,16 @@ pub(crate) unsafe fn create_module<'ll>( target_data_layout = target_data_layout.replace("p8:128:128:128:48", "p8:128:128") } } + if llvm_version < (22, 0, 0) { + if sess.target.arch == "avr" { + // LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010 + target_data_layout = target_data_layout.replace("n8:16", "n8") + } + if sess.target.arch == "nvptx64" { + // LLVM 22 updated the NVPTX layout to indicate 256-bit vector load/store: https://github.com/llvm/llvm-project/pull/155198 + target_data_layout = target_data_layout.replace("-i256:256", ""); + } + } // Ensure the data-layout values hardcoded remain the defaults. { @@ -372,7 +360,17 @@ pub(crate) unsafe fn create_module<'ll>( } } - if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if let Some(regparm_count) = sess.opts.unstable_opts.regparm { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Error, + "NumRegisterParameters", + regparm_count, + ); + } + + if let Some(BranchProtection { bti, pac_ret, gcs }) = sess.opts.unstable_opts.branch_protection + { if sess.target.arch == "aarch64" { llvm::add_module_flag_u32( llmod, @@ -405,6 +403,12 @@ pub(crate) unsafe fn create_module<'ll>( "sign-return-address-with-bkey", u32::from(pac_opts.key == PAuthKey::B), ); + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Min, + "guarded-control-stack", + gcs.into(), + ); } else { bug!( "branch-protection used on non-AArch64 target; \ @@ -457,6 +461,15 @@ pub(crate) unsafe fn create_module<'ll>( } } + if sess.opts.unstable_opts.indirect_branch_cs_prefix { + llvm::add_module_flag_u32( + llmod, + llvm::ModuleFlagMergeBehavior::Override, + "indirect_branch_cs_prefix", + 1, + ); + } + match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support()) { // Set up the small-data optimization limit for architectures that use @@ -481,14 +494,7 @@ pub(crate) unsafe fn create_module<'ll>( format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION")); let name_metadata = cx.create_metadata(rustc_producer.as_bytes()); - - unsafe { - llvm::LLVMAddNamedMetadataOperand( - llmod, - c"llvm.ident".as_ptr(), - &cx.get_metadata_value(llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)), - ); - } + cx.module_add_named_metadata_node(llmod, c"llvm.ident", &[name_metadata]); // Emit RISC-V specific target-abi metadata // to workaround lld as the LTO plugin not @@ -617,7 +623,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { const_globals: Default::default(), statics_to_rauw: RefCell::new(Vec::new()), used_statics: Vec::new(), - compiler_used_statics: Vec::new(), + compiler_used_statics: Default::default(), type_lowering: Default::default(), scalar_lltypes: Default::default(), coverage_cx, @@ -628,6 +634,9 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { intrinsics: Default::default(), local_gen_sym_counter: Cell::new(0), renamed_statics: Default::default(), + objc_class_t: Cell::new(None), + objc_classrefs: Default::default(), + objc_selrefs: Default::default(), }, PhantomData, ) @@ -652,12 +661,71 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { llvm::set_linkage(g, llvm::Linkage::AppendingLinkage); llvm::set_section(g, c"llvm.metadata"); } + + /// The Objective-C ABI that is used. + /// + /// This corresponds to the `-fobjc-abi-version=` flag in Clang / GCC. + pub(crate) fn objc_abi_version(&self) -> u32 { + assert!(self.tcx.sess.target.is_like_darwin); + if self.tcx.sess.target.arch == "x86" && self.tcx.sess.target.os == "macos" { + // 32-bit x86 macOS uses ABI version 1 (a.k.a. the "fragile ABI"). + 1 + } else { + // All other Darwin-like targets we support use ABI version 2 + // (a.k.a the "non-fragile ABI"). + 2 + } + } + + // We do our best here to match what Clang does when compiling Objective-C natively. + // See Clang's `CGObjCCommonMac::EmitImageInfo`: + // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5085 + pub(crate) fn add_objc_module_flags(&self) { + let abi_version = self.objc_abi_version(); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Version", + abi_version, + ); + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Version", + 0, + ); + + llvm::add_module_flag_str( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Image Info Section", + match abi_version { + 1 => "__OBJC,__image_info,regular", + 2 => "__DATA,__objc_imageinfo,regular,no_dead_strip", + _ => unreachable!(), + }, + ); + + if self.tcx.sess.target.env == "sim" { + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Is Simulated", + 1 << 5, + ); + } + + llvm::add_module_flag_u32( + self.llmod, + llvm::ModuleFlagMergeBehavior::Error, + "Objective-C Class Properties", + 1 << 6, + ); + } } impl<'ll> SimpleCx<'ll> { - pub(crate) fn get_return_type(&self, ty: &'ll Type) -> &'ll Type { - assert_eq!(self.type_kind(ty), TypeKind::Function); - unsafe { llvm::LLVMGetReturnType(ty) } - } pub(crate) fn get_type_of_global(&self, val: &'ll Value) -> &'ll Type { unsafe { llvm::LLVMGlobalGetValueType(val) } } @@ -671,7 +739,7 @@ impl<'ll> SimpleCx<'ll> { llcx: &'ll llvm::Context, pointer_size: Size, ) -> Self { - let isize_ty = llvm::Type::ix_llcx(llcx, pointer_size.bits()); + let isize_ty = llvm::LLVMIntTypeInContext(llcx, pointer_size.bits() as c_uint); Self(SCx { llmod, llcx, isize_ty }, PhantomData) } } @@ -682,7 +750,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn get_const_int(&self, ty: &'ll Type, val: u64) -> &'ll Value { - unsafe { llvm::LLVMConstInt(ty, val, llvm::False) } + unsafe { llvm::LLVMConstInt(ty, val, llvm::FALSE) } } pub(crate) fn get_const_i64(&self, n: u64) -> &'ll Value { @@ -721,16 +789,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { llvm::LLVMMDStringInContext2(self.llcx(), name.as_ptr() as *const c_char, name.len()) } } - - pub(crate) fn get_functions(&self) -> Vec<&'ll Value> { - let mut functions = vec![]; - let mut func = unsafe { llvm::LLVMGetFirstFunction(self.llmod()) }; - while let Some(f) = func { - functions.push(f); - func = unsafe { llvm::LLVMGetNextFunction(f) } - } - functions - } } impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { @@ -810,7 +868,7 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } else { let fty = self.type_variadic_func(&[], self.type_i32()); let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); - let target_cpu = attributes::target_cpu_attr(self); + let target_cpu = attributes::target_cpu_attr(self, self.sess()); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); llfn } @@ -825,30 +883,33 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn set_frame_pointer_type(&self, llfn: &'ll Value) { - if let Some(attr) = attributes::frame_pointer_type_attr(self) { + if let Some(attr) = attributes::frame_pointer_type_attr(self, self.sess()) { attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); } } fn apply_target_cpu_attr(&self, llfn: &'ll Value) { let mut attrs = SmallVec::<[_; 2]>::new(); - attrs.push(attributes::target_cpu_attr(self)); - attrs.extend(attributes::tune_cpu_attr(self)); + attrs.push(attributes::target_cpu_attr(self, self.sess())); + attrs.extend(attributes::tune_cpu_attr(self, self.sess())); attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); } fn declare_c_main(&self, fn_type: Self::Type) -> Option { let entry_name = self.sess().target.entry_name.as_ref(); if self.get_declared_value(entry_name).is_none() { - Some(self.declare_entry_fn( + let llfn = self.declare_entry_fn( entry_name, - llvm::CallConv::from_conv( - self.sess().target.entry_abi, - self.sess().target.arch.borrow(), - ), + to_llvm_calling_convention(self.sess(), self.sess().target.entry_abi), llvm::UnnamedAddr::Global, fn_type, - )) + ); + attributes::apply_to_llfn( + llfn, + llvm::AttributePlace::Function, + attributes::target_features_attr(self, self.tcx, vec![]).as_slice(), + ); + Some(llfn) } else { // If the symbol already exists, it is an error: for example, the user wrote // #[no_mangle] extern "C" fn main(..) {..} @@ -930,15 +991,75 @@ impl CodegenCx<'_, '_> { } impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { + /// Wrapper for `LLVMMDNodeInContext2`, i.e. `llvm::MDNode::get`. + pub(crate) fn md_node_in_context(&self, md_list: &[&'ll Metadata]) -> &'ll Metadata { + unsafe { llvm::LLVMMDNodeInContext2(self.llcx(), md_list.as_ptr(), md_list.len()) } + } + /// A wrapper for [`llvm::LLVMSetMetadata`], but it takes `Metadata` as a parameter instead of `Value`. pub(crate) fn set_metadata<'a>( &self, val: &'a Value, - kind_id: impl Into, + kind_id: MetadataKindId, md: &'ll Metadata, ) { let node = self.get_metadata_value(md); - llvm::LLVMSetMetadata(val, kind_id.into(), node); + llvm::LLVMSetMetadata(val, kind_id, node); + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`) + /// - `LLVMSetMetadata` (to set that node as metadata of `kind_id` for `instruction`) + pub(crate) fn set_metadata_node( + &self, + instruction: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + self.set_metadata(instruction, kind_id, md); + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMMetadataAsValue` (to adapt that node to an `llvm::Value`) + /// - `LLVMAddNamedMetadataOperand` (to set that node as metadata of `kind_name` for `module`) + pub(crate) fn module_add_named_metadata_node( + &self, + module: &'ll Module, + kind_name: &CStr, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + let md_as_val = self.get_metadata_value(md); + unsafe { llvm::LLVMAddNamedMetadataOperand(module, kind_name.as_ptr(), md_as_val) }; + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMRustGlobalAddMetadata` (to set that node as metadata of `kind_id` for `global`) + pub(crate) fn global_add_metadata_node( + &self, + global: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + unsafe { llvm::LLVMRustGlobalAddMetadata(global, kind_id, md) }; + } + + /// Helper method for the sequence of calls: + /// - `LLVMMDNodeInContext2` (to create an `llvm::MDNode` from a list of metadata) + /// - `LLVMGlobalSetMetadata` (to set that node as metadata of `kind_id` for `global`) + pub(crate) fn global_set_metadata_node( + &self, + global: &'ll Value, + kind_id: MetadataKindId, + md_list: &[&'ll Metadata], + ) { + let md = self.md_node_in_context(md_list); + unsafe { llvm::LLVMGlobalSetMetadata(global, kind_id, md) }; } } @@ -972,7 +1093,10 @@ impl<'tcx, 'll> HasTypingEnv<'tcx> for CodegenCx<'ll, 'tcx> { impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { - if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + if let LayoutError::SizeOverflow(_) + | LayoutError::ReferencesError(_) + | LayoutError::InvalidSimd { .. } = err + { self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() }) } else { self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) @@ -989,7 +1113,11 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { match err { - FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + FnAbiError::Layout( + LayoutError::SizeOverflow(_) + | LayoutError::Cycle(_) + | LayoutError::InvalidSimd { .. }, + ) => { self.tcx.dcx().emit_fatal(Spanned { span, node: err }); } _ => match fn_abi_request { diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index f6000e728400..caa36c5dd499 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -1,5 +1,3 @@ -use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId}; - /// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] @@ -26,30 +24,12 @@ pub(crate) enum CounterKind { pub(crate) struct Counter { // Important: The layout (order and types of fields) must match its C++ counterpart. pub(crate) kind: CounterKind, - id: u32, + pub(crate) id: u32, } impl Counter { /// A `Counter` of kind `Zero`. For this counter kind, the `id` is not used. pub(crate) const ZERO: Self = Self { kind: CounterKind::Zero, id: 0 }; - - /// Constructs a new `Counter` of kind `CounterValueReference`. - pub(crate) fn counter_value_reference(counter_id: CounterId) -> Self { - Self { kind: CounterKind::CounterValueReference, id: counter_id.as_u32() } - } - - /// Constructs a new `Counter` of kind `Expression`. - pub(crate) fn expression(expression_id: ExpressionId) -> Self { - Self { kind: CounterKind::Expression, id: expression_id.as_u32() } - } - - pub(crate) fn from_term(term: CovTerm) -> Self { - match term { - CovTerm::Zero => Self::ZERO, - CovTerm::Counter(id) => Self::counter_value_reference(id), - CovTerm::Expression(id) => Self::expression(id), - } - } } /// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`. @@ -73,48 +53,6 @@ pub(crate) struct CounterExpression { pub(crate) rhs: Counter, } -pub(crate) mod mcdc { - use rustc_middle::mir::coverage::{ConditionId, ConditionInfo, DecisionInfo}; - - /// Must match the layout of `LLVMRustMCDCDecisionParameters`. - #[repr(C)] - #[derive(Clone, Copy, Debug, Default)] - pub(crate) struct DecisionParameters { - bitmap_idx: u32, - num_conditions: u16, - } - - type LLVMConditionId = i16; - - /// Must match the layout of `LLVMRustMCDCBranchParameters`. - #[repr(C)] - #[derive(Clone, Copy, Debug, Default)] - pub(crate) struct BranchParameters { - condition_id: LLVMConditionId, - condition_ids: [LLVMConditionId; 2], - } - - impl From for BranchParameters { - fn from(value: ConditionInfo) -> Self { - let to_llvm_cond_id = |cond_id: Option| { - cond_id.and_then(|id| LLVMConditionId::try_from(id.as_usize()).ok()).unwrap_or(-1) - }; - let ConditionInfo { condition_id, true_next_id, false_next_id } = value; - Self { - condition_id: to_llvm_cond_id(Some(condition_id)), - condition_ids: [to_llvm_cond_id(false_next_id), to_llvm_cond_id(true_next_id)], - } - } - } - - impl From for DecisionParameters { - fn from(info: DecisionInfo) -> Self { - let DecisionInfo { bitmap_idx, num_conditions } = info; - Self { bitmap_idx, num_conditions } - } - } -} - /// A span of source code coordinates to be embedded in coverage metadata. /// /// Must match the layout of `LLVMRustCoverageSpan`. @@ -136,41 +74,6 @@ pub(crate) struct CoverageSpan { pub(crate) end_col: u32, } -/// Holds tables of the various region types in one struct. -/// -/// Don't pass this struct across FFI; pass the individual region tables as -/// pointer/length pairs instead. -/// -/// Each field name has a `_regions` suffix for improved readability after -/// exhaustive destructing, which ensures that all region types are handled. -#[derive(Clone, Debug, Default)] -pub(crate) struct Regions { - pub(crate) code_regions: Vec, - pub(crate) expansion_regions: Vec, - pub(crate) branch_regions: Vec, - pub(crate) mcdc_branch_regions: Vec, - pub(crate) mcdc_decision_regions: Vec, -} - -impl Regions { - /// Returns true if none of this structure's tables contain any regions. - pub(crate) fn has_no_regions(&self) -> bool { - let Self { - code_regions, - expansion_regions, - branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, - } = self; - - code_regions.is_empty() - && expansion_regions.is_empty() - && branch_regions.is_empty() - && mcdc_branch_regions.is_empty() - && mcdc_decision_regions.is_empty() - } -} - /// Must match the layout of `LLVMRustCoverageCodeRegion`. #[derive(Clone, Debug)] #[repr(C)] @@ -195,21 +98,3 @@ pub(crate) struct BranchRegion { pub(crate) true_counter: Counter, pub(crate) false_counter: Counter, } - -/// Must match the layout of `LLVMRustCoverageMCDCBranchRegion`. -#[derive(Clone, Debug)] -#[repr(C)] -pub(crate) struct MCDCBranchRegion { - pub(crate) cov_span: CoverageSpan, - pub(crate) true_counter: Counter, - pub(crate) false_counter: Counter, - pub(crate) mcdc_branch_params: mcdc::BranchParameters, -} - -/// Must match the layout of `LLVMRustCoverageMCDCDecisionRegion`. -#[derive(Clone, Debug)] -#[repr(C)] -pub(crate) struct MCDCDecisionRegion { - pub(crate) cov_span: CoverageSpan, - pub(crate) mcdc_decision_params: mcdc::DecisionParameters, -} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index 907d6d41a1fb..a58202834cfa 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -2,26 +2,25 @@ use std::ffi::CString; -use crate::common::AsCCharPtr; use crate::coverageinfo::ffi; use crate::llvm; pub(crate) fn covmap_var_name() -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapVarNameToString(s); })) .expect("covmap variable name should not contain NUL") } pub(crate) fn covmap_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovmapSectionNameToString(llmod, s); })) .expect("covmap section name should not contain NUL") } pub(crate) fn covfun_section_name(llmod: &llvm::Module) -> CString { - CString::new(llvm::build_byte_buffer(|s| unsafe { + CString::new(llvm::build_byte_buffer(|s| { llvm::LLVMRustCoverageWriteCovfunSectionNameToString(llmod, s); })) .expect("covfun section name should not contain NUL") @@ -34,7 +33,7 @@ pub(crate) fn create_pgo_func_name_var<'ll>( unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar( llfn, - mangled_fn_name.as_c_char_ptr(), + mangled_fn_name.as_ptr(), mangled_fn_name.len(), ) } @@ -44,7 +43,7 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef]) -> Vec, Vec<_>>(); llvm::build_byte_buffer(|buffer| unsafe { @@ -58,18 +57,35 @@ pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef]) -> Vec, + pub(crate) expansion_regions: Vec, + pub(crate) branch_regions: Vec, +} + +impl Regions { + /// Returns true if none of this structure's tables contain any regions. + pub(crate) fn has_no_regions(&self) -> bool { + let Self { code_regions, expansion_regions, branch_regions } = self; + + code_regions.is_empty() && expansion_regions.is_empty() && branch_regions.is_empty() + } +} + pub(crate) fn write_function_mappings_to_buffer( virtual_file_mapping: &[u32], expressions: &[ffi::CounterExpression], - regions: &ffi::Regions, + regions: &Regions, ) -> Vec { - let ffi::Regions { - code_regions, - expansion_regions, - branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, - } = regions; + let Regions { code_regions, expansion_regions, branch_regions } = regions; // SAFETY: // - All types are FFI-compatible and have matching representations in Rust/C++. @@ -87,10 +103,6 @@ pub(crate) fn write_function_mappings_to_buffer( expansion_regions.len(), branch_regions.as_ptr(), branch_regions.len(), - mcdc_branch_regions.as_ptr(), - mcdc_branch_regions.len(), - mcdc_decision_regions.as_ptr(), - mcdc_decision_regions.len(), buffer, ) }) @@ -99,12 +111,12 @@ pub(crate) fn write_function_mappings_to_buffer( /// Hashes some bytes into a 64-bit hash, via LLVM's `IndexedInstrProf::ComputeHash`, /// as required for parts of the LLVM coverage mapping format. pub(crate) fn hash_bytes(bytes: &[u8]) -> u64 { - unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_c_char_ptr(), bytes.len()) } + unsafe { llvm::LLVMRustCoverageHashBytes(bytes.as_ptr(), bytes.len()) } } /// Returns LLVM's `coverage::CovMapVersion::CurrentVersion` (CoverageMapping.h) /// as a raw numeric value. For historical reasons, the numeric value is 1 less /// than the number in the version's name, so `Version7` is actually `6u32`. pub(crate) fn mapping_version() -> u32 { - unsafe { llvm::LLVMRustCoverageMappingVersion() } + llvm::LLVMRustCoverageMappingVersion() } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs index 8c9dfcfd18c2..b3a11f8b12bf 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -1,3 +1,4 @@ +use std::assert_matches::assert_matches; use std::sync::Arc; use itertools::Itertools; @@ -14,12 +15,29 @@ use tracing::debug; use crate::common::CodegenCx; use crate::coverageinfo::llvm_cov; use crate::coverageinfo::mapgen::covfun::prepare_covfun_record; -use crate::llvm; +use crate::{TryFromU32, llvm}; mod covfun; mod spans; mod unused; +/// Version number that will be included the `__llvm_covmap` section header. +/// Corresponds to LLVM's `llvm::coverage::CovMapVersion` (in `CoverageMapping.h`), +/// or at least the subset that we know and care about. +/// +/// Note that version `n` is encoded as `(n-1)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, TryFromU32)] +enum CovmapVersion { + /// Used by LLVM 18 onwards. + Version7 = 6, +} + +impl CovmapVersion { + fn to_u32(self) -> u32 { + self as u32 + } +} + /// Generates and exports the coverage map, which is embedded in special /// linker sections in the final binary. /// @@ -29,19 +47,13 @@ pub(crate) fn finalize(cx: &mut CodegenCx<'_, '_>) { let tcx = cx.tcx; // Ensure that LLVM is using a version of the coverage mapping format that - // agrees with our Rust-side code. Expected versions (encoded as n-1) are: - // - `CovMapVersion::Version7` (6) used by LLVM 18-19 - let covmap_version = { - let llvm_covmap_version = llvm_cov::mapping_version(); - let expected_versions = 6..=6; - assert!( - expected_versions.contains(&llvm_covmap_version), - "Coverage mapping version exposed by `llvm-wrapper` is out of sync; \ - expected {expected_versions:?} but was {llvm_covmap_version}" - ); - // This is the version number that we will embed in the covmap section: - llvm_covmap_version - }; + // agrees with our Rust-side code. Expected versions are: + // - `Version7` (6) used by LLVM 18 onwards. + let covmap_version = + CovmapVersion::try_from(llvm_cov::mapping_version()).unwrap_or_else(|raw_version: u32| { + panic!("unknown coverage mapping version reported by `llvm-wrapper`: {raw_version}") + }); + assert_matches!(covmap_version, CovmapVersion::Version7); debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); @@ -116,7 +128,7 @@ impl GlobalFileTable { for file in all_files { raw_file_table.entry(file.stable_id).or_insert_with(|| { file.name - .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) + .for_scope(tcx.sess, RemapPathScopeComponents::COVERAGE) .to_string_lossy() .into_owned() }); @@ -135,7 +147,7 @@ impl GlobalFileTable { .sess .opts .working_dir - .for_scope(tcx.sess, RemapPathScopeComponents::MACRO) + .for_scope(tcx.sess, RemapPathScopeComponents::COVERAGE) .to_string_lossy(); table.push(base_dir.as_ref()); @@ -201,7 +213,11 @@ impl VirtualFileMapping { /// Generates the contents of the covmap record for this CGU, which mostly /// consists of a header and a list of filenames. The record is then stored /// as a global variable in the `__llvm_covmap` section. -fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) { +fn generate_covmap_record<'ll>( + cx: &mut CodegenCx<'ll, '_>, + version: CovmapVersion, + filenames_buffer: &[u8], +) { // A covmap record consists of four target-endian u32 values, followed by // the encoded filenames table. Two of the header fields are unused in // modern versions of the LLVM coverage mapping format, and are always 0. @@ -212,7 +228,7 @@ fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filena cx.const_u32(0), // (unused) cx.const_u32(filenames_buffer.len() as u32), cx.const_u32(0), // (unused) - cx.const_u32(version), + cx.const_u32(version.to_u32()), ], /* packed */ false, ); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs index b704cf2b1cd4..7835d1804686 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs @@ -10,8 +10,8 @@ use std::sync::Arc; use rustc_abi::Align; use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods as _, ConstCodegenMethods}; use rustc_middle::mir::coverage::{ - BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, - MappingKind, Op, + BasicCoverageBlock, CounterId, CovTerm, CoverageIdsInfo, Expression, ExpressionId, + FunctionCoverageInfo, Mapping, MappingKind, Op, }; use rustc_middle::ty::{Instance, TyCtxt}; use rustc_span::{SourceFile, Span}; @@ -27,13 +27,16 @@ use crate::llvm; /// the final record that will be embedded in the `__llvm_covfun` section. #[derive(Debug)] pub(crate) struct CovfunRecord<'tcx> { + /// Not used directly, but helpful in debug messages. + _instance: Instance<'tcx>, + mangled_function_name: &'tcx str, source_hash: u64, is_used: bool, virtual_file_mapping: VirtualFileMapping, expressions: Vec, - regions: ffi::Regions, + regions: llvm_cov::Regions, } impl<'tcx> CovfunRecord<'tcx> { @@ -55,12 +58,13 @@ pub(crate) fn prepare_covfun_record<'tcx>( let expressions = prepare_expressions(ids_info); let mut covfun = CovfunRecord { + _instance: instance, mangled_function_name: tcx.symbol_name(instance).name, source_hash: if is_used { fn_cov_info.function_source_hash } else { 0 }, is_used, virtual_file_mapping: VirtualFileMapping::default(), expressions, - regions: ffi::Regions::default(), + regions: llvm_cov::Regions::default(), }; fill_region_tables(tcx, fn_cov_info, ids_info, &mut covfun); @@ -73,10 +77,21 @@ pub(crate) fn prepare_covfun_record<'tcx>( Some(covfun) } +pub(crate) fn counter_for_term(term: CovTerm) -> ffi::Counter { + use ffi::Counter; + match term { + CovTerm::Zero => Counter::ZERO, + CovTerm::Counter(id) => { + Counter { kind: ffi::CounterKind::CounterValueReference, id: CounterId::as_u32(id) } + } + CovTerm::Expression(id) => { + Counter { kind: ffi::CounterKind::Expression, id: ExpressionId::as_u32(id) } + } + } +} + /// Convert the function's coverage-counter expressions into a form suitable for FFI. fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec { - let counter_for_term = ffi::Counter::from_term; - // We know that LLVM will optimize out any unused expressions before // producing the final coverage map, so there's no need to do the same // thing on the Rust side unless we're confident we can do much better. @@ -102,11 +117,21 @@ fn fill_region_tables<'tcx>( ids_info: &'tcx CoverageIdsInfo, covfun: &mut CovfunRecord<'tcx>, ) { + // If this function is unused, replace all counters with zero. + let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { + let term = if covfun.is_used { + ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") + } else { + CovTerm::Zero + }; + counter_for_term(term) + }; + // Currently a function's mappings must all be in the same file, so use the // first mapping's span to determine the file. let source_map = tcx.sess.source_map(); let Some(first_span) = (try { fn_cov_info.mappings.first()?.span }) else { - debug_assert!(false, "function has no mappings: {:?}", covfun.mangled_function_name); + debug_assert!(false, "function has no mappings: {covfun:?}"); return; }; let source_file = source_map.lookup_source_file(first_span.lo()); @@ -117,32 +142,20 @@ fn fill_region_tables<'tcx>( // codegen needs to handle that gracefully to avoid #133606. // It's hard for tests to trigger this organically, so instead we set // `-Zcoverage-options=discard-all-spans-in-codegen` to force it to occur. - let discard_all = tcx.sess.coverage_discard_all_spans_in_codegen(); + let discard_all = tcx.sess.coverage_options().discard_all_spans_in_codegen; let make_coords = |span: Span| { if discard_all { None } else { spans::make_coords(source_map, &source_file, span) } }; - let ffi::Regions { + let llvm_cov::Regions { code_regions, expansion_regions: _, // FIXME(Zalathar): Fill out support for expansion regions branch_regions, - mcdc_branch_regions, - mcdc_decision_regions, } = &mut covfun.regions; // For each counter/region pair in this function+file, convert it to a // form suitable for FFI. for &Mapping { ref kind, span } in &fn_cov_info.mappings { - // If this function is unused, replace all counters with zero. - let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter { - let term = if covfun.is_used { - ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term") - } else { - CovTerm::Zero - }; - ffi::Counter::from_term(term) - }; - let Some(coords) = make_coords(span) else { continue }; let cov_span = coords.make_coverage_span(local_file_id); @@ -157,20 +170,6 @@ fn fill_region_tables<'tcx>( false_counter: counter_for_bcb(false_bcb), }); } - MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => { - mcdc_branch_regions.push(ffi::MCDCBranchRegion { - cov_span, - true_counter: counter_for_bcb(true_bcb), - false_counter: counter_for_bcb(false_bcb), - mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params), - }); - } - MappingKind::MCDCDecision(mcdc_decision_params) => { - mcdc_decision_regions.push(ffi::MCDCDecisionRegion { - cov_span, - mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params), - }); - } } } } @@ -184,6 +183,7 @@ pub(crate) fn generate_covfun_record<'tcx>( covfun: &CovfunRecord<'tcx>, ) { let &CovfunRecord { + _instance, mangled_function_name, source_hash, is_used, diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs index 39a59560c9d3..574463be7ffe 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs @@ -39,7 +39,10 @@ impl Coords { /// or other expansions), and if it does happen then skipping a span or function is /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option { - let span = ensure_non_empty_span(source_map, span)?; + if span.is_empty() { + debug_assert!(false, "can't make coords from empty span: {span:?}"); + return None; + } let lo = span.lo(); let hi = span.hi(); @@ -70,29 +73,6 @@ pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) }) } -fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option { - if !span.is_empty() { - return Some(span); - } - - // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'. - source_map - .span_to_source(span, |src, start, end| try { - // Adjusting span endpoints by `BytePos(1)` is normally a bug, - // but in this case we have specifically checked that the character - // we're skipping over is one of two specific ASCII characters, so - // adjusting by exactly 1 byte is correct. - if src.as_bytes().get(end).copied() == Some(b'{') { - Some(span.with_hi(span.hi() + BytePos(1))) - } else if start > 0 && src.as_bytes()[start - 1] == b'}' { - Some(span.with_lo(span.lo() - BytePos(1))) - } else { - None - } - }) - .ok()? -} - /// If `llvm-cov` sees a source region that is improperly ordered (end < start), /// it will immediately exit with a fatal error. To prevent that from happening, /// discard regions that are improperly ordered, or might be interpreted in a diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 119237abd6b8..6a58f495c9d8 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -1,11 +1,10 @@ use std::cell::{OnceCell, RefCell}; use std::ffi::{CStr, CString}; -use rustc_abi::Size; use rustc_codegen_ssa::traits::{ - BuilderMethods, ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, + ConstCodegenMethods, CoverageInfoBuilderMethods, MiscCodegenMethods, }; -use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; +use rustc_data_structures::fx::FxIndexMap; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; use tracing::{debug, instrument}; @@ -28,34 +27,13 @@ pub(crate) struct CguCoverageContext<'ll, 'tcx> { /// symbol name, and `llvm-cov` will exit fatally if it can't resolve that /// hash back to an entry in the binary's `__llvm_prf_names` linker section. pub(crate) pgo_func_name_var_map: RefCell, &'ll llvm::Value>>, - pub(crate) mcdc_condition_bitmap_map: RefCell, Vec<&'ll llvm::Value>>>, covfun_section_name: OnceCell, } impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { pub(crate) fn new() -> Self { - Self { - pgo_func_name_var_map: Default::default(), - mcdc_condition_bitmap_map: Default::default(), - covfun_section_name: Default::default(), - } - } - - /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is - /// called condition bitmap. In order to handle nested decisions, several condition bitmaps can - /// be allocated for a function body. These values are named `mcdc.addr.{i}` and are a 32-bit - /// integers. They respectively hold the condition bitmaps for decisions with a depth of `i`. - fn try_get_mcdc_condition_bitmap( - &self, - instance: &Instance<'tcx>, - decision_depth: u16, - ) -> Option<&'ll llvm::Value> { - self.mcdc_condition_bitmap_map - .borrow() - .get(instance) - .and_then(|bitmap_map| bitmap_map.get(decision_depth as usize)) - .copied() // Dereference Option<&&Value> to Option<&Value> + Self { pgo_func_name_var_map: Default::default(), covfun_section_name: Default::default() } } /// Returns the list of instances considered "used" in this CGU, as @@ -105,38 +83,6 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { - fn init_coverage(&mut self, instance: Instance<'tcx>) { - let Some(function_coverage_info) = - self.tcx.instance_mir(instance.def).function_coverage_info.as_deref() - else { - return; - }; - - // If there are no MC/DC bitmaps to set up, return immediately. - if function_coverage_info.mcdc_bitmap_bits == 0 { - return; - } - - let fn_name = self.ensure_pgo_func_name_var(instance); - let hash = self.const_u64(function_coverage_info.function_source_hash); - let bitmap_bits = self.const_u32(function_coverage_info.mcdc_bitmap_bits as u32); - self.mcdc_parameters(fn_name, hash, bitmap_bits); - - // Create pointers named `mcdc.addr.{i}` to stack-allocated condition bitmaps. - let mut cond_bitmaps = vec![]; - for i in 0..function_coverage_info.mcdc_num_condition_bitmaps { - // MC/DC intrinsics will perform loads/stores that use the ABI default - // alignment for i32, so our variable declaration should match. - let align = self.tcx.data_layout.i32_align.abi; - let cond_bitmap = self.alloca(Size::from_bytes(4), align); - llvm::set_value_name(cond_bitmap, format!("mcdc.addr.{i}").as_bytes()); - self.store(self.const_i32(0), cond_bitmap, align); - cond_bitmaps.push(cond_bitmap); - } - - self.coverage_cx().mcdc_condition_bitmap_map.borrow_mut().insert(instance, cond_bitmaps); - } - #[instrument(level = "debug", skip(self))] fn add_coverage(&mut self, instance: Instance<'tcx>, kind: &CoverageKind) { // Our caller should have already taken care of inlining subtleties, @@ -153,7 +99,7 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { // When that happens, we currently just discard those statements, so // the corresponding code will be undercounted. // FIXME(Zalathar): Find a better solution for mixed-coverage builds. - let Some(coverage_cx) = &bx.cx.coverage_cx else { return }; + let Some(_coverage_cx) = &bx.cx.coverage_cx else { return }; let Some(function_coverage_info) = bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref() @@ -185,30 +131,6 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { } // If a BCB doesn't have an associated physical counter, there's nothing to codegen. CoverageKind::VirtualCounter { .. } => {} - CoverageKind::CondBitmapUpdate { index, decision_depth } => { - let cond_bitmap = coverage_cx - .try_get_mcdc_condition_bitmap(&instance, decision_depth) - .expect("mcdc cond bitmap should have been allocated for updating"); - let cond_index = bx.const_i32(index as i32); - bx.mcdc_condbitmap_update(cond_index, cond_bitmap); - } - CoverageKind::TestVectorBitmapUpdate { bitmap_idx, decision_depth } => { - let cond_bitmap = - coverage_cx.try_get_mcdc_condition_bitmap(&instance, decision_depth).expect( - "mcdc cond bitmap should have been allocated for merging \ - into the global bitmap", - ); - assert!( - bitmap_idx as usize <= function_coverage_info.mcdc_bitmap_bits, - "bitmap index of the decision out of range" - ); - - let fn_name = bx.ensure_pgo_func_name_var(instance); - let hash = bx.const_u64(function_coverage_info.function_source_hash); - let bitmap_index = bx.const_u32(bitmap_idx); - bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_index, cond_bitmap); - bx.mcdc_condbitmap_reset(cond_bitmap); - } } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs b/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs new file mode 100644 index 000000000000..5d874631ca1a --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs @@ -0,0 +1,69 @@ +use libc::c_uint; +use rustc_abi::Align; + +use crate::llvm::debuginfo::DIBuilder; +use crate::llvm::{self, ToLlvmBool}; + +/// Extension trait for defining safe wrappers and helper methods on +/// `&DIBuilder<'ll>`, without requiring it to be defined in the same crate. +pub(crate) trait DIBuilderExt<'ll> { + fn as_di_builder(&self) -> &DIBuilder<'ll>; + + fn create_expression(&self, addr_ops: &[u64]) -> &'ll llvm::Metadata { + let this = self.as_di_builder(); + unsafe { llvm::LLVMDIBuilderCreateExpression(this, addr_ops.as_ptr(), addr_ops.len()) } + } + + fn create_static_variable( + &self, + scope: Option<&'ll llvm::Metadata>, + name: &str, + linkage_name: &str, + file: &'ll llvm::Metadata, + line_number: c_uint, + ty: &'ll llvm::Metadata, + is_local_to_unit: bool, + val: &'ll llvm::Value, + decl: Option<&'ll llvm::Metadata>, + align: Option, + ) -> &'ll llvm::Metadata { + let this = self.as_di_builder(); + let align_in_bits = align.map_or(0, |align| align.bits() as u32); + + // `LLVMDIBuilderCreateGlobalVariableExpression` would assert if we + // gave it a null `Expr` pointer, so give it an empty expression + // instead, which is what the C++ `createGlobalVariableExpression` + // method would do if given a null `DIExpression` pointer. + let expr = self.create_expression(&[]); + + let global_var_expr = unsafe { + llvm::LLVMDIBuilderCreateGlobalVariableExpression( + this, + scope, + name.as_ptr(), + name.len(), + linkage_name.as_ptr(), + linkage_name.len(), + file, + line_number, + ty, + is_local_to_unit.to_llvm_bool(), + expr, + decl, + align_in_bits, + ) + }; + + unsafe { llvm::LLVMGlobalSetMetadata(val, llvm::MD_dbg, global_var_expr) }; + + global_var_expr + } +} + +impl<'ll> DIBuilderExt<'ll> for &DIBuilder<'ll> { + fn as_di_builder(&self) -> &DIBuilder<'ll> { + self + } + + // All other methods have default bodies that rely on `as_di_builder`. +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs index 408429152223..52d04625749b 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/dwarf_const.rs @@ -35,3 +35,6 @@ declare_constant!(DW_OP_plus_uconst: u64); /// Double-checked by a static assertion in `RustWrapper.cpp`. #[allow(non_upper_case_globals)] pub(crate) const DW_OP_LLVM_fragment: u64 = 0x1000; +// It describes the actual value of a source variable which might not exist in registers or in memory. +#[allow(non_upper_case_globals)] +pub(crate) const DW_OP_stack_value: u64 = 0x9f; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs index 61555ac2f6f6..3a750b6b53eb 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -1,17 +1,15 @@ // .debug_gdb_scripts binary section. -use rustc_attr_data_structures::{AttributeKind, find_attr}; use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::DebuggerVisualizerType; use rustc_hir::def_id::LOCAL_CRATE; use rustc_middle::bug; -use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerType; use rustc_session::config::{CrateType, DebugInfo}; use crate::builder::Builder; use crate::common::CodegenCx; -use crate::llvm; -use crate::value::Value; +use crate::llvm::{self, Value}; /// Inserts a side-effect free instruction sequence that makes sure that the /// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. @@ -73,7 +71,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); llvm::set_section(section_var, c".debug_gdb_scripts"); llvm::set_initializer(section_var, cx.const_bytes(section_contents)); - llvm::LLVMSetGlobalConstant(section_var, llvm::True); + llvm::LLVMSetGlobalConstant(section_var, llvm::TRUE); llvm::set_unnamed_address(section_var, llvm::UnnamedAddr::Global); llvm::set_linkage(section_var, llvm::Linkage::LinkOnceODRLinkage); // This should make sure that the whole section is not larger than @@ -85,9 +83,6 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>( } pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { - let omit_gdb_pretty_printer_section = - find_attr!(cx.tcx.hir_krate_attrs(), AttributeKind::OmitGdbPrettyPrinterSection); - // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create // ODR violations at link time, this section will not be emitted for rlibs since // each rlib could produce a different set of visualizers that would be embedded @@ -116,8 +111,7 @@ pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { } }); - !omit_gdb_pretty_printer_section - && cx.sess().opts.debuginfo != DebugInfo::None + cx.sess().opts.debuginfo != DebugInfo::None && cx.sess().target.emit_debug_gdb_scripts && embed_visualizers } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 0e9dbfba658d..5535ebe65859 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -19,7 +19,9 @@ use rustc_middle::ty::{ self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; -use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene}; +use rustc_span::{ + DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Span, Symbol, hygiene, +}; use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::spec::DebuginfoKind; use smallvec::smallvec; @@ -30,19 +32,16 @@ use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId}; use super::CodegenUnitDebugContext; use super::namespace::mangled_name_of_instance; use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; -use super::utils::{ - DIB, create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, -}; +use super::utils::{DIB, debug_context, get_namespace_for_item, is_node_local_to_unit}; use crate::common::{AsCCharPtr, CodegenCx}; -use crate::debuginfo::dwarf_const; use crate::debuginfo::metadata::type_map::build_type_with_children; use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind}; -use crate::llvm; +use crate::debuginfo::{DIBuilderExt, dwarf_const}; use crate::llvm::debuginfo::{ DIBasicType, DIBuilder, DICompositeType, DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, DebugNameTableKind, }; -use crate::value::Value; +use crate::llvm::{self, FromGeneric, Value}; impl PartialEq for llvm::Metadata { fn eq(&self, other: &Self) -> bool { @@ -101,32 +100,33 @@ fn build_fixed_size_array_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, array_type: Ty<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let ty::Array(element_type, len) = array_type.kind() else { bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) }; - let element_type_di_node = type_di_node(cx, *element_type); + let element_type_di_node = spanned_type_di_node(cx, *element_type, span); return_if_di_node_created_in_meantime!(cx, unique_type_id); - let (size, align) = cx.size_and_align_of(array_type); + let (size, align) = cx.spanned_size_and_align_of(array_type, span); let upper_bound = len .try_to_target_usize(cx.tcx) .expect("expected monomorphic const in codegen") as c_longlong; - let subrange = - unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; + let subrange = unsafe { llvm::LLVMDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound) }; + let subscripts = &[subrange]; - let subscripts = create_DIArray(DIB(cx), &[subrange]); let di_node = unsafe { - llvm::LLVMRustDIBuilderCreateArrayType( + llvm::LLVMDIBuilderCreateArrayType( DIB(cx), size.bits(), align.bits() as u32, element_type_di_node, - subscripts, + subscripts.as_ptr(), + subscripts.len() as c_uint, ) }; @@ -172,17 +172,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( "ptr_type={ptr_type}, pointee_type={pointee_type}", ); - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - pointer_size.bits(), - pointer_align.abi.bits() as u32, - 0, // Ignore DWARF address space. - ptr_type_debuginfo_name.as_c_char_ptr(), - ptr_type_debuginfo_name.len(), - ) - }; + let di_node = create_pointer_type( + cx, + pointee_type_di_node, + pointer_size, + pointer_align.abi, + &ptr_type_debuginfo_name, + ); DINodeCreationResult { di_node, already_stored_in_typemap: false } } @@ -230,17 +226,13 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>( // The data pointer type is a regular, thin pointer, regardless of whether this // is a slice or a trait object. - let data_ptr_type_di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - pointee_type_di_node, - addr_field.size.bits(), - addr_field.align.abi.bits() as u32, - 0, // Ignore DWARF address space. - std::ptr::null(), - 0, - ) - }; + let data_ptr_type_di_node = create_pointer_type( + cx, + pointee_type_di_node, + addr_field.size, + addr_field.align.abi, + "", + ); smallvec![ build_field_di_node( @@ -315,7 +307,7 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); - let fn_di_node = create_subroutine_type(cx, create_DIArray(DIB(cx), &signature_di_nodes[..])); + let fn_di_node = create_subroutine_type(cx, &signature_di_nodes[..]); // This is actually a function pointer, so wrap it in pointer DI. let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); @@ -326,26 +318,44 @@ fn build_subroutine_type_di_node<'ll, 'tcx>( } _ => unreachable!(), }; - let di_node = unsafe { - llvm::LLVMRustDIBuilderCreatePointerType( - DIB(cx), - fn_di_node, - size.bits(), - align.bits() as u32, - 0, // Ignore DWARF address space. - name.as_c_char_ptr(), - name.len(), - ) - }; + let di_node = create_pointer_type(cx, fn_di_node, size, align, &name); DINodeCreationResult::new(di_node, false) } pub(super) fn create_subroutine_type<'ll>( cx: &CodegenCx<'ll, '_>, - signature: &'ll DICompositeType, + signature: &[Option<&'ll llvm::Metadata>], ) -> &'ll DICompositeType { - unsafe { llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(cx), signature) } + unsafe { + llvm::LLVMDIBuilderCreateSubroutineType( + DIB(cx), + None, // ("File" is ignored and has no effect) + signature.as_ptr(), + signature.len() as c_uint, + DIFlags::FlagZero, // (default value) + ) + } +} + +fn create_pointer_type<'ll>( + cx: &CodegenCx<'ll, '_>, + pointee_ty: &'ll llvm::Metadata, + size: Size, + align: Align, + name: &str, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreatePointerType( + DIB(cx), + pointee_ty, + size.bits(), + align.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr(), + name.len(), + ) + } } /// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs @@ -423,6 +433,14 @@ fn build_slice_type_di_node<'ll, 'tcx>( /// This function will look up the debuginfo node in the TypeMap. If it can't find it, it /// will create the node by dispatching to the corresponding `build_*_di_node()` function. pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + spanned_type_di_node(cx, t, DUMMY_SP) +} + +pub(crate) fn spanned_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, + span: Span, +) -> &'ll DIType { let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) @@ -437,7 +455,7 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> build_basic_type_di_node(cx, t) } ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), - ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t, span), ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), @@ -458,9 +476,9 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), ty::Adt(def, ..) => match def.adt_kind() { - AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), - AdtKind::Union => build_union_type_di_node(cx, unique_type_id), - AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id, span), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id, span), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id, span), }, ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), @@ -802,14 +820,15 @@ fn build_basic_type_di_node<'ll, 'tcx>( }; let typedef_di_node = unsafe { - llvm::LLVMRustDIBuilderCreateTypedef( + llvm::LLVMDIBuilderCreateTypedef( DIB(cx), ty_di_node, - typedef_name.as_c_char_ptr(), + typedef_name.as_ptr(), typedef_name.len(), unknown_file_metadata(cx), - 0, - None, + 0, // (no line number) + None, // (no scope) + 0u32, // (no alignment specified) ) }; @@ -823,12 +842,13 @@ fn create_basic_type<'ll, 'tcx>( encoding: u32, ) -> &'ll DIBasicType { unsafe { - llvm::LLVMRustDIBuilderCreateBasicType( + llvm::LLVMDIBuilderCreateBasicType( DIB(cx), - name.as_c_char_ptr(), + name.as_ptr(), name.len(), size.bits(), encoding, + DIFlags::FlagZero, ) } } @@ -1014,15 +1034,15 @@ fn create_member_type<'ll, 'tcx>( type_di_node: &'ll DIType, ) -> &'ll DIType { unsafe { - llvm::LLVMRustDIBuilderCreateMemberType( + llvm::LLVMDIBuilderCreateMemberType( DIB(cx), owner, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, layout.size.bits(), - layout.align.abi.bits() as u32, + layout.align.bits() as u32, offset.bits(), flags, type_di_node, @@ -1055,6 +1075,7 @@ fn visibility_di_flags<'ll, 'tcx>( fn build_struct_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let struct_type = unique_type_id.expect_ty(); let ty::Adt(adt_def, _) = struct_type.kind() else { @@ -1062,7 +1083,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( }; assert!(adt_def.is_struct()); let containing_scope = get_namespace_for_item(cx, adt_def.did()); - let struct_type_and_layout = cx.layout_of(struct_type); + let struct_type_and_layout = cx.spanned_layout_of(struct_type, span); let variant_def = adt_def.non_enum_variant(); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) @@ -1255,6 +1276,7 @@ fn build_closure_env_di_node<'ll, 'tcx>( fn build_union_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let union_type = unique_type_id.expect_ty(); let (union_def_id, variant_def) = match union_type.kind() { @@ -1262,7 +1284,7 @@ fn build_union_type_di_node<'ll, 'tcx>( _ => bug!("build_union_type_di_node on a non-ADT"), }; let containing_scope = get_namespace_for_item(cx, union_def_id); - let union_ty_and_layout = cx.layout_of(union_type); + let union_ty_and_layout = cx.spanned_layout_of(union_type, span); let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { Some(file_metadata_from_def_id(cx, Some(union_def_id))) @@ -1388,23 +1410,18 @@ pub(crate) fn build_global_var_di_node<'ll>( let global_align = cx.align_of(variable_type); - unsafe { - llvm::LLVMRustDIBuilderCreateStaticVariable( - DIB(cx), - Some(var_scope), - var_name.as_c_char_ptr(), - var_name.len(), - linkage_name.as_c_char_ptr(), - linkage_name.len(), - file_metadata, - line_number, - type_di_node, - is_local_to_unit, - global, - None, - global_align.bits() as u32, - ); - } + DIB(cx).create_static_variable( + Some(var_scope), + var_name, + linkage_name, + file_metadata, + line_number, + type_di_node, + is_local_to_unit, + global, // (value) + None, // (decl) + Some(global_align), + ); } /// Generates LLVM debuginfo for a vtable. @@ -1425,7 +1442,7 @@ fn build_vtable_type_di_node<'ll, 'tcx>( let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); - let trait_ref = tcx.erase_regions(trait_ref); + let trait_ref = tcx.erase_and_anonymize_regions(trait_ref); tcx.vtable_entries(trait_ref) } else { @@ -1552,7 +1569,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( // Unwrap potential addrspacecast let vtable = find_vtable_behind_cast(vtable); let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); - let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_ref_self = cx.tcx.erase_and_anonymize_regions(trait_ref_self); let trait_def_id = trait_ref_self.def_id; let trait_vis = cx.tcx.visibility(trait_def_id); @@ -1584,21 +1601,11 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>( let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); let typeid = cx.create_metadata(trait_ref_typeid.as_bytes()); - unsafe { - let v = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; - llvm::LLVMRustGlobalAddMetadata( - vtable, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(cx.llcx, v.as_ptr(), v.len()), - ); - let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); - let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); - llvm::LLVMGlobalSetMetadata( - vtable, - llvm::MetadataType::MD_vcall_visibility as c_uint, - vcall_visibility_metadata, - ); - } + let type_ = [llvm::LLVMValueAsMetadata(cx.const_usize(0)), typeid]; + cx.global_add_metadata_node(vtable, llvm::MD_type, &type_); + + let vcall_visibility = [llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64))]; + cx.global_set_metadata_node(vtable, llvm::MD_vcall_visibility, &vcall_visibility); } /// Creates debug information for the given vtable, which is for the @@ -1631,25 +1638,19 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>( let vtable_name = compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); - let linkage_name = ""; - unsafe { - llvm::LLVMRustDIBuilderCreateStaticVariable( - DIB(cx), - NO_SCOPE_METADATA, - vtable_name.as_c_char_ptr(), - vtable_name.len(), - linkage_name.as_c_char_ptr(), - linkage_name.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - vtable_type_di_node, - true, - vtable, - None, - 0, - ); - } + DIB(cx).create_static_variable( + NO_SCOPE_METADATA, + &vtable_name, + "", // (linkage_name) + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + vtable_type_di_node, + true, // (is_local_to_unit) + vtable, // (value) + None, // (decl) + None::, + ); } /// Creates an "extension" of an existing `DIScope` into another file. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index a5c808957410..4ecc3086e1bd 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -11,7 +11,7 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty}; use smallvec::smallvec; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::dwarf_const::DW_TAG_const_type; use crate::debuginfo::metadata::enums::DiscrResult; use crate::debuginfo::metadata::type_map::{self, Stub, UniqueTypeId}; @@ -378,20 +378,17 @@ fn build_single_variant_union_fields<'ll, 'tcx>( variant_struct_type_wrapper_di_node, None, ), - unsafe { - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), - enum_type_di_node, - TAG_FIELD_NAME.as_c_char_ptr(), - TAG_FIELD_NAME.len(), - unknown_file_metadata(cx), - UNKNOWN_LINE_NUMBER, - variant_names_type_di_node, - visibility_flags, - Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), - tag_base_type_align.bits() as u32, - ) - } + create_static_member_type( + cx, + enum_type_di_node, + TAG_FIELD_NAME, + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + variant_names_type_di_node, + visibility_flags, + Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)), + tag_base_type_align, + ), ] } @@ -570,27 +567,28 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>( let build_assoc_const = |name: &str, type_di_node_: &'ll DIType, value: u64, - align: Align| unsafe { + align: Align| + -> &'ll llvm::Metadata { // FIXME: Currently we force all DISCR_* values to be u64's as LLDB seems to have // problems inspecting other value types. Since DISCR_* is typically only going to be // directly inspected via the debugger visualizer - which compares it to the `tag` value // (whose type is not modified at all) it shouldn't cause any real problems. let (t_di, align) = if name == ASSOC_CONST_DISCR_NAME { - (type_di_node_, align.bits() as u32) + (type_di_node_, align) } else { let ty_u64 = Ty::new_uint(cx.tcx, ty::UintTy::U64); - (type_di_node(cx, ty_u64), Align::EIGHT.bits() as u32) + (type_di_node(cx, ty_u64), Align::EIGHT) }; // must wrap type in a `const` modifier for LLDB to be able to inspect the value of the member - let field_type = - llvm::LLVMRustDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di); + let field_type = unsafe { + llvm::LLVMDIBuilderCreateQualifiedType(DIB(cx), DW_TAG_const_type, t_di) + }; - llvm::LLVMRustDIBuilderCreateStaticMemberType( - DIB(cx), + create_static_member_type( + cx, wrapper_struct_type_di_node, - name.as_c_char_ptr(), - name.len(), + name, unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER, field_type, @@ -975,3 +973,30 @@ fn variant_struct_wrapper_type_name(variant_index: VariantIdx) -> Cow<'static, s .map(|&s| Cow::from(s)) .unwrap_or_else(|| format!("Variant{}", variant_index.as_usize()).into()) } + +fn create_static_member_type<'ll>( + cx: &CodegenCx<'ll, '_>, + scope: &'ll llvm::Metadata, + name: &str, + file: &'ll llvm::Metadata, + line_number: c_uint, + ty: &'ll llvm::Metadata, + flags: DIFlags, + value: Option<&'ll llvm::Value>, + align: Align, +) -> &'ll llvm::Metadata { + unsafe { + llvm::LLVMDIBuilderCreateStaticMemberType( + DIB(cx), + scope, + name.as_ptr(), + name.len(), + file, + line_number, + ty, + flags, + value, + align.bits() as c_uint, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index 7c701926d2c5..caff35860797 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -10,7 +10,7 @@ use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::Symbol; +use rustc_span::{Span, Symbol}; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -30,13 +30,14 @@ mod native; pub(super) fn build_enum_type_di_node<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, unique_type_id: UniqueTypeId<'tcx>, + span: Span, ) -> DINodeCreationResult<'ll> { let enum_type = unique_type_id.expect_ty(); let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) }; - let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_and_layout = cx.spanned_layout_of(enum_type, span); if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) { return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index 62d38d463aba..1ae6e6e5eeca 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -289,7 +289,7 @@ fn build_enum_variant_part_di_node<'ll, 'tcx>( file_metadata, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, DIFlags::FlagZero, tag_member_di_node, create_DIArray(DIB(cx), &[]), @@ -449,7 +449,7 @@ fn build_enum_variant_member_di_node<'ll, 'tcx>( file_di_node, line_number, enum_type_and_layout.size.bits(), - enum_type_and_layout.align.abi.bits() as u32, + enum_type_and_layout.align.bits() as u32, Size::ZERO.bits(), discr, DIFlags::FlagZero, diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs index d1502d2b1e62..37200fdc41af 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -1,5 +1,6 @@ use std::cell::RefCell; +use libc::c_uint; use rustc_abi::{Align, Size, VariantIdx}; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; @@ -9,7 +10,7 @@ use rustc_middle::bug; use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; -use crate::common::{AsCCharPtr, CodegenCx}; +use crate::common::CodegenCx; use crate::debuginfo::utils::{DIB, create_DIArray, debug_context}; use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; use crate::llvm::{self}; @@ -191,7 +192,7 @@ pub(super) fn stub<'ll, 'tcx>( containing_scope: Option<&'ll DIScope>, flags: DIFlags, ) -> StubInfo<'ll, 'tcx> { - let empty_array = create_DIArray(DIB(cx), &[]); + let no_elements: &[Option<&llvm::Metadata>] = &[]; let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); let (file_metadata, line_number) = if let Some(def_location) = def_location { @@ -207,10 +208,10 @@ pub(super) fn stub<'ll, 'tcx>( _ => None, }; unsafe { - llvm::LLVMRustDIBuilderCreateStructType( + llvm::LLVMDIBuilderCreateStructType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, @@ -218,28 +219,30 @@ pub(super) fn stub<'ll, 'tcx>( align.bits() as u32, flags, None, - empty_array, - 0, + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) vtable_holder, - unique_type_id_str.as_c_char_ptr(), + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) } } Stub::Union => unsafe { - llvm::LLVMRustDIBuilderCreateUnionType( + llvm::LLVMDIBuilderCreateUnionType( DIB(cx), containing_scope, - name.as_c_char_ptr(), + name.as_ptr(), name.len(), file_metadata, line_number, size.bits(), align.bits() as u32, flags, - Some(empty_array), - 0, - unique_type_id_str.as_c_char_ptr(), + no_elements.as_ptr(), + no_elements.len() as c_uint, + 0u32, // (Objective-C runtime version; default is 0) + unique_type_id_str.as_ptr(), unique_type_id_str.len(), ) }, @@ -276,7 +279,7 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( && let ty::Adt(adt_def, args) = ty.kind() { let def_id = adt_def.did(); - // If any sub type reference the original type definition and the sub type has a type + // If any child type references the original type definition and the child type has a type // parameter that strictly contains the original parameter, the original type is a recursive // type that can expanding indefinitely. Example, // ``` @@ -285,21 +288,43 @@ pub(super) fn build_type_with_children<'ll, 'tcx>( // Item(T), // } // ``` - let is_expanding_recursive = adt_def.is_enum() - && debug_context(cx).adt_stack.borrow().iter().any(|(parent_def_id, parent_args)| { - if def_id == *parent_def_id { - args.iter().zip(parent_args.iter()).any(|(arg, parent_arg)| { - if let (Some(arg), Some(parent_arg)) = (arg.as_type(), parent_arg.as_type()) - { - arg != parent_arg && arg.contains(parent_arg) - } else { - false - } - }) - } else { - false - } - }); + let is_expanding_recursive = { + let stack = debug_context(cx).adt_stack.borrow(); + stack + .iter() + .enumerate() + .rev() + .skip(1) + .filter(|(_, (ancestor_def_id, _))| def_id == *ancestor_def_id) + .any(|(ancestor_index, (_, ancestor_args))| { + args.iter() + .zip(ancestor_args.iter()) + .filter_map(|(arg, ancestor_arg)| arg.as_type().zip(ancestor_arg.as_type())) + .any(|(arg, ancestor_arg)| + // Strictly contains. + (arg != ancestor_arg && arg.contains(ancestor_arg)) + // Check all types between current and ancestor use the + // ancestor_arg. + // Otherwise, duplicate wrappers in normal recursive type may be + // regarded as expanding. + // ``` + // struct Recursive { + // a: Box>, + // } + // ``` + // It can produce an ADT stack like this, + // - Box + // - Recursive + // - Box> + && stack[ancestor_index + 1..stack.len()].iter().all( + |(_, intermediate_args)| + intermediate_args + .iter() + .filter_map(|arg| arg.as_type()) + .any(|mid_arg| mid_arg.contains(ancestor_arg)) + )) + }) + }; if is_expanding_recursive { // FIXME: indicate that this is an expanding recursive type in stub metadata? return DINodeCreationResult::new(stub_info.metadata, false); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index 6cbf2dbf7d3f..b95ad03b70e0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -28,37 +28,30 @@ use rustc_target::spec::DebuginfoKind; use smallvec::SmallVec; use tracing::debug; -use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, type_di_node}; +use self::create_scope_map::compute_mir_scopes; +pub(crate) use self::di_builder::DIBuilderExt; +pub(crate) use self::metadata::build_global_var_di_node; +use self::metadata::{ + UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, spanned_type_di_node, type_di_node, +}; use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::builder::Builder; use crate::common::{AsCCharPtr, CodegenCx}; -use crate::llvm; use crate::llvm::debuginfo::{ DIArray, DIBuilderBox, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DITemplateTypeParameter, DIType, DIVariable, }; -use crate::value::Value; +use crate::llvm::{self, Value}; mod create_scope_map; +mod di_builder; mod dwarf_const; mod gdb; pub(crate) mod metadata; mod namespace; mod utils; -use self::create_scope_map::compute_mir_scopes; -pub(crate) use self::metadata::build_global_var_di_node; - -// FIXME(Zalathar): These `DW_TAG_*` constants are fake values that were -// removed from LLVM in 2015, and are only used by our own `RustWrapper.cpp` -// to decide which C++ API to call. Instead, we should just have two separate -// FFI functions and choose the correct one on the Rust side. -#[allow(non_upper_case_globals)] -const DW_TAG_auto_variable: c_uint = 0x100; -#[allow(non_upper_case_globals)] -const DW_TAG_arg_variable: c_uint = 0x101; - /// A context object for maintaining all state needed by the debuginfo module. pub(crate) struct CodegenUnitDebugContext<'ll, 'tcx> { llmod: &'ll llvm::Module, @@ -163,16 +156,64 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { variable_alloca: Self::Value, direct_offset: Size, indirect_offsets: &[Size], - fragment: Option>, + fragment: &Option>, ) { use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst}; // Convert the direct and indirect offsets and fragment byte range to address ops. let mut addr_ops = SmallVec::<[u64; 8]>::new(); + if direct_offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(direct_offset.bytes()); + } + for &offset in indirect_offsets { + addr_ops.push(DW_OP_deref); + if offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst); + addr_ops.push(offset.bytes()); + } + } + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(DW_OP_LLVM_fragment); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); + } + + let di_builder = DIB(self.cx()); + let addr_expr = di_builder.create_expression(&addr_ops); + unsafe { + llvm::LLVMDIBuilderInsertDeclareRecordAtEnd( + di_builder, + variable_alloca, + dbg_var, + addr_expr, + dbg_loc, + self.llbb(), + ) + }; + } + + fn dbg_var_value( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + value: Self::Value, + direct_offset: Size, + indirect_offsets: &[Size], + fragment: &Option>, + ) { + use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst, DW_OP_stack_value}; + + // Convert the direct and indirect offsets and fragment byte range to address ops. + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + if direct_offset.bytes() > 0 { addr_ops.push(DW_OP_plus_uconst); addr_ops.push(direct_offset.bytes() as u64); + addr_ops.push(DW_OP_stack_value); } for &offset in indirect_offsets { addr_ops.push(DW_OP_deref); @@ -189,14 +230,16 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { addr_ops.push((fragment.end - fragment.start).bits() as u64); } + let di_builder = DIB(self.cx()); + let addr_expr = unsafe { + llvm::LLVMDIBuilderCreateExpression(di_builder, addr_ops.as_ptr(), addr_ops.len()) + }; unsafe { - // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. - llvm::LLVMRustDIBuilderInsertDeclareAtEnd( - DIB(self.cx()), - variable_alloca, + llvm::LLVMDIBuilderInsertDbgValueRecordAtEnd( + di_builder, + value, dbg_var, - addr_ops.as_ptr(), - addr_ops.len() as c_uint, + addr_expr, dbg_loc, self.llbb(), ); @@ -347,7 +390,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let file_metadata = file_metadata(self, &loc.file); let function_type_metadata = - create_subroutine_type(self, get_function_signature(self, fn_abi)); + create_subroutine_type(self, &get_function_signature(self, fn_abi)); let mut name = String::with_capacity(64); type_names::push_item_name(tcx, def_id, false, &mut name); @@ -439,9 +482,9 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn get_function_signature<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, - ) -> &'ll DIArray { + ) -> Vec> { if cx.sess().opts.debuginfo != DebugInfo::Full { - return create_DIArray(DIB(cx), &[]); + return vec![]; } let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); @@ -482,7 +525,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); } - create_DIArray(DIB(cx), &signature[..]) + signature } fn get_template_parameters<'ll, 'tcx>( @@ -533,31 +576,26 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { // First, let's see if this is a method within an inherent impl. Because // if yes, we want to make the result subroutine DIE a child of the // subroutine's self-type. - if let Some(impl_def_id) = cx.tcx.impl_of_assoc(instance.def_id()) { - // If the method does *not* belong to a trait, proceed - if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { - let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( - instance.args, - cx.typing_env(), - cx.tcx.type_of(impl_def_id), - ); + // For trait method impls we still use the "parallel namespace" + // strategy + if let Some(imp_def_id) = cx.tcx.inherent_impl_of_assoc(instance.def_id()) { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(imp_def_id), + ); - // Only "class" methods are generally understood by LLVM, - // so avoid methods on other types (e.g., `<*mut T>::null`). - if let ty::Adt(def, ..) = impl_self_ty.kind() - && !def.is_box() - { - // Again, only create type information if full debuginfo is enabled - if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() - { - return (type_di_node(cx, impl_self_ty), true); - } else { - return (namespace::item_namespace(cx, def.did()), false); - } + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); } - } else { - // For trait method impls we still use the "parallel namespace" - // strategy } } @@ -631,30 +669,41 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { let loc = self.lookup_debug_loc(span.lo()); let file_metadata = file_metadata(self, &loc.file); - let type_metadata = type_di_node(self, variable_type); + let type_metadata = spanned_type_di_node(self, variable_type, span); - let (argument_index, dwarf_tag) = match variable_kind { - ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), - LocalVariable => (0, DW_TAG_auto_variable), - }; let align = self.align_of(variable_type); let name = variable_name.as_str(); - unsafe { - llvm::LLVMRustDIBuilderCreateVariable( - DIB(self), - dwarf_tag, - scope_metadata, - name.as_c_char_ptr(), - name.len(), - file_metadata, - loc.line, - type_metadata, - true, - DIFlags::FlagZero, - argument_index, - align.bits() as u32, - ) + + match variable_kind { + ArgumentVariable(arg_index) => unsafe { + llvm::LLVMDIBuilderCreateParameterVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + arg_index as c_uint, + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + ) + }, + LocalVariable => unsafe { + llvm::LLVMDIBuilderCreateAutoVariable( + DIB(self), + scope_metadata, + name.as_ptr(), + name.len(), + file_metadata, + loc.line, + type_metadata, + llvm::Bool::TRUE, // (preserve descriptor during optimizations) + DIFlags::FlagZero, + align.bits() as u32, + ) + }, } } } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs index b4d639368b00..1dcf4ff3062a 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -38,7 +38,7 @@ pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'l parent_scope, namespace_name_string.as_ptr(), namespace_name_string.len(), - llvm::False, // ExportSymbols (only relevant for C++ anonymous namespaces) + llvm::FALSE, // ExportSymbols (only relevant for C++ anonymous namespaces) ) }; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs index cc1d504b4301..7e1e49310f61 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -28,7 +28,7 @@ pub(crate) fn create_DIArray<'ll>( builder: &DIBuilder<'ll>, arr: &[Option<&'ll DIDescriptor>], ) -> &'ll DIArray { - unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } + unsafe { llvm::LLVMDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len()) } } #[inline] diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index 960a895a2031..8f69f176138c 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -23,13 +23,11 @@ use smallvec::SmallVec; use tracing::debug; use crate::abi::FnAbiLlvmExt; +use crate::attributes; use crate::common::AsCCharPtr; use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx}; use crate::llvm::AttributePlace::Function; -use crate::llvm::Visibility; -use crate::type_::Type; -use crate::value::Value; -use crate::{attributes, llvm}; +use crate::llvm::{self, FromGeneric, Type, Value, Visibility}; /// Declare a function with a SimpleCx. /// @@ -76,7 +74,7 @@ pub(crate) fn declare_raw_fn<'ll, 'tcx>( attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); } - attrs.extend(attributes::non_lazy_bind_attr(cx)); + attrs.extend(attributes::non_lazy_bind_attr(cx, cx.tcx.sess)); attributes::apply_to_llfn(llfn, Function, &attrs); @@ -232,13 +230,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } } - /// Declare a private global - /// - /// Use this function when you intend to define a global without a name. - pub(crate) fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { - unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod(), ty) } - } - /// Gets declared value by name. pub(crate) fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { debug!("get_declared_value(name={:?})", name); diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 7b27e496986a..14b3f3626efe 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -3,29 +3,31 @@ use std::cmp::Ordering; use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; +use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::*; -use rustc_hir as hir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_hir::{self as hir}; use rustc_middle::mir::BinOp; use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf}; -use rustc_middle::ty::{self, GenericArgsRef, Ty}; +use rustc_middle::ty::{self, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv}; use rustc_middle::{bug, span_bug}; use rustc_span::{Span, Symbol, sym}; -use rustc_symbol_mangling::mangle_internal_symbol; -use rustc_target::spec::PanicStrategy; +use rustc_symbol_mangling::{mangle_internal_symbol, symbol_name_for_instance_in_crate}; +use rustc_target::callconv::PassMode; use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; +use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::context::CodegenCx; -use crate::llvm::{self, Metadata}; -use crate::type_::Type; +use crate::errors::AutoDiffWithoutEnable; +use crate::llvm::{self, Metadata, Type, Value}; use crate::type_of::LayoutLlvmExt; use crate::va_arg::emit_va_arg; -use crate::value::Value; fn call_simple_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, @@ -189,6 +191,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &[ptr, args[1].immediate()], ) } + sym::autodiff => { + codegen_autodiff(self, tcx, instance, args, result); + return Ok(()); + } sym::is_val_statically_known => { if let OperandValue::Immediate(imm) = args[0].val { self.call_intrinsic( @@ -289,7 +295,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let align = if name == sym::unaligned_volatile_load { 1 } else { - result.layout.align.abi.bytes() as u32 + result.layout.align.bytes() as u32 }; unsafe { llvm::LLVMSetAlignment(load, align); @@ -321,10 +327,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); + let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_i32(); self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], - &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], + &[ + ptr, + self.const_i32(rw), + self.const_i32(locality), + self.const_i32(cache_type), + ], ) } sym::carrying_mul_add => { @@ -368,7 +380,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::rotate_left | sym::rotate_right | sym::saturating_add - | sym::saturating_sub => { + | sym::saturating_sub + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { let ty = args[0].layout.ty; if !ty.is_integral() { tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { @@ -409,18 +423,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::bitreverse => { self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()]) } - sym::rotate_left | sym::rotate_right => { - let is_left = name == sym::rotate_left; - let val = args[0].immediate(); - let raw_shift = args[1].immediate(); - // rotate = funnel shift with first two args the same + sym::rotate_left + | sym::rotate_right + | sym::unchecked_funnel_shl + | sym::unchecked_funnel_shr => { + let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl; + let lhs = args[0].immediate(); + let (rhs, raw_shift) = + if name == sym::rotate_left || name == sym::rotate_right { + // rotate = funnel shift with first two args the same + (lhs, args[1].immediate()) + } else { + (args[1].immediate(), args[2].immediate()) + }; let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' }); // llvm expects shift to be the same type as the values, but rust // always uses `u32`. - let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false); - self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift]) + self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; @@ -649,7 +671,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( catch_func: &'ll Value, dest: PlaceRef<'tcx, &'ll Value>, ) { - if bx.sess().panic_strategy() == PanicStrategy::Abort { + if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); bx.call(try_func_ty, None, None, try_func, &[data], None, None); // Return 0 unconditionally from the intrinsic call; @@ -1023,7 +1045,7 @@ fn codegen_emcc_try<'ll, 'tcx>( // create an alloca and pass a pointer to that. let ptr_size = bx.tcx().data_layout.pointer_size(); let ptr_align = bx.tcx().data_layout.pointer_align().abi; - let i8_align = bx.tcx().data_layout.i8_align.abi; + let i8_align = bx.tcx().data_layout.i8_align; // Required in order for there to be no padding between the fields. assert!(i8_align <= ptr_align); let catch_data = bx.alloca(2 * ptr_size, ptr_align); @@ -1113,6 +1135,148 @@ fn get_rust_try_fn<'a, 'll, 'tcx>( rust_try } +fn codegen_autodiff<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tcx: TyCtxt<'tcx>, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + result: PlaceRef<'tcx, &'ll Value>, +) { + if !tcx.sess.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable) { + let _ = tcx.dcx().emit_almost_fatal(AutoDiffWithoutEnable); + } + + let fn_args = instance.args; + let callee_ty = instance.ty(tcx, bx.typing_env()); + + let sig = callee_ty.fn_sig(tcx).skip_binder(); + + let ret_ty = sig.output(); + let llret_ty = bx.layout_of(ret_ty).llvm_type(bx); + + // Get source, diff, and attrs + let (source_id, source_args) = match fn_args.into_type_list(tcx)[0].kind() { + ty::FnDef(def_id, source_params) => (def_id, source_params), + _ => bug!("invalid autodiff intrinsic args"), + }; + + let fn_source = match Instance::try_resolve(tcx, bx.cx.typing_env(), *source_id, source_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + source_id, + source_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let source_symbol = symbol_name_for_instance_in_crate(tcx, fn_source.clone(), LOCAL_CRATE); + let Some(fn_to_diff) = bx.cx.get_function(&source_symbol) else { + bug!("could not find source function") + }; + + let (diff_id, diff_args) = match fn_args.into_type_list(tcx)[1].kind() { + ty::FnDef(def_id, diff_args) => (def_id, diff_args), + _ => bug!("invalid args"), + }; + + let fn_diff = match Instance::try_resolve(tcx, bx.cx.typing_env(), *diff_id, diff_args) { + Ok(Some(instance)) => instance, + Ok(None) => bug!( + "could not resolve ({:?}, {:?}) to a specific autodiff instance", + diff_id, + diff_args + ), + Err(_) => { + // An error has already been emitted + return; + } + }; + + let val_arr = get_args_from_tuple(bx, args[2], fn_diff); + let diff_symbol = symbol_name_for_instance_in_crate(tcx, fn_diff.clone(), LOCAL_CRATE); + + let Some(mut diff_attrs) = autodiff_attrs(tcx, fn_diff.def_id()) else { + bug!("could not find autodiff attrs") + }; + + adjust_activity_to_abi( + tcx, + fn_source, + TypingEnv::fully_monomorphized(), + &mut diff_attrs.input_activity, + ); + + let fnc_tree = + rustc_middle::ty::fnc_typetrees(tcx, fn_source.ty(tcx, TypingEnv::fully_monomorphized())); + + // Build body + generate_enzyme_call( + bx, + bx.cx, + fn_to_diff, + &diff_symbol, + llret_ty, + &val_arr, + diff_attrs.clone(), + result, + fnc_tree, + ); +} + +fn get_args_from_tuple<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + tuple_op: OperandRef<'tcx, &'ll Value>, + fn_instance: Instance<'tcx>, +) -> Vec<&'ll Value> { + let cx = bx.cx; + let fn_abi = cx.fn_abi_of_instance(fn_instance, ty::List::empty()); + + match tuple_op.val { + OperandValue::Immediate(val) => vec![val], + OperandValue::Pair(v1, v2) => vec![v1, v2], + OperandValue::Ref(ptr) => { + let tuple_place = PlaceRef { val: ptr, layout: tuple_op.layout }; + + let mut result = Vec::with_capacity(fn_abi.args.len()); + let mut tuple_index = 0; + + for arg in &fn_abi.args { + match arg.mode { + PassMode::Ignore => {} + PassMode::Direct(_) | PassMode::Cast { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(val); + tuple_index += 1; + } + PassMode::Pair(_, _) => { + let field = tuple_place.project_field(bx, tuple_index); + let llvm_ty = field.layout.llvm_type(bx.cx); + let pair_val = bx.load(llvm_ty, field.val.llval, field.val.align); + result.push(bx.extract_value(pair_val, 0)); + result.push(bx.extract_value(pair_val, 1)); + tuple_index += 1; + } + PassMode::Indirect { .. } => { + let field = tuple_place.project_field(bx, tuple_index); + result.push(field.val.llval); + tuple_index += 1; + } + } + } + + result + } + + OperandValue::ZeroSized => vec![], + } +} + fn generic_simd_intrinsic<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, name: Symbol, @@ -1161,6 +1325,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>( }; } + let llvm_version = crate::llvm_util::get_version(); + /// Converts a vector mask, where each element has a bit width equal to the data elements it is used with, /// down to an i1 based mask that can be used by llvm intrinsics. /// @@ -1644,7 +1810,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); // Alignment of T, must be a constant integer value: - let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + let alignment = bx.align_of(in_elem).bytes(); // Truncate the mask vector to a vector of i1s: let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len); @@ -1655,11 +1821,23 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); - return Ok(bx.call_intrinsic( - "llvm.masked.gather", - &[llvm_elem_vec_ty, llvm_pointer_vec_ty], - &[args[1].immediate(), alignment, mask, args[0].immediate()], - )); + let args: &[&'ll Value] = if llvm_version < (22, 0, 0) { + let alignment = bx.const_i32(alignment as i32); + &[args[1].immediate(), alignment, mask, args[0].immediate()] + } else { + &[args[1].immediate(), mask, args[0].immediate()] + }; + + let call = + bx.call_intrinsic("llvm.masked.gather", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], args); + if llvm_version >= (22, 0, 0) { + crate::attributes::apply_to_callsite( + call, + crate::llvm::AttributePlace::Argument(0), + &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)], + ) + } + return Ok(call); } if name == sym::simd_masked_load { @@ -1727,18 +1905,30 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); // Alignment of T, must be a constant integer value: - let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + let alignment = bx.align_of(values_elem).bytes(); let llvm_pointer = bx.type_ptr(); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); - return Ok(bx.call_intrinsic( - "llvm.masked.load", - &[llvm_elem_vec_ty, llvm_pointer], - &[args[1].immediate(), alignment, mask, args[2].immediate()], - )); + let args: &[&'ll Value] = if llvm_version < (22, 0, 0) { + let alignment = bx.const_i32(alignment as i32); + + &[args[1].immediate(), alignment, mask, args[2].immediate()] + } else { + &[args[1].immediate(), mask, args[2].immediate()] + }; + + let call = bx.call_intrinsic("llvm.masked.load", &[llvm_elem_vec_ty, llvm_pointer], args); + if llvm_version >= (22, 0, 0) { + crate::attributes::apply_to_callsite( + call, + crate::llvm::AttributePlace::Argument(0), + &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)], + ) + } + return Ok(call); } if name == sym::simd_masked_store { @@ -1800,18 +1990,29 @@ fn generic_simd_intrinsic<'ll, 'tcx>( let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len); // Alignment of T, must be a constant integer value: - let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32); + let alignment = bx.align_of(values_elem).bytes(); let llvm_pointer = bx.type_ptr(); // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len); - return Ok(bx.call_intrinsic( - "llvm.masked.store", - &[llvm_elem_vec_ty, llvm_pointer], - &[args[2].immediate(), args[1].immediate(), alignment, mask], - )); + let args: &[&'ll Value] = if llvm_version < (22, 0, 0) { + let alignment = bx.const_i32(alignment as i32); + &[args[2].immediate(), args[1].immediate(), alignment, mask] + } else { + &[args[2].immediate(), args[1].immediate(), mask] + }; + + let call = bx.call_intrinsic("llvm.masked.store", &[llvm_elem_vec_ty, llvm_pointer], args); + if llvm_version >= (22, 0, 0) { + crate::attributes::apply_to_callsite( + call, + crate::llvm::AttributePlace::Argument(1), + &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)], + ) + } + return Ok(call); } if name == sym::simd_scatter { @@ -1876,7 +2077,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); // Alignment of T, must be a constant integer value: - let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + let alignment = bx.align_of(in_elem).bytes(); // Truncate the mask vector to a vector of i1s: let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len); @@ -1886,12 +2087,25 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // Type of the vector of elements: let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); - - return Ok(bx.call_intrinsic( + let args: &[&'ll Value] = if llvm_version < (22, 0, 0) { + let alignment = bx.const_i32(alignment as i32); + &[args[0].immediate(), args[1].immediate(), alignment, mask] + } else { + &[args[0].immediate(), args[1].immediate(), mask] + }; + let call = bx.call_intrinsic( "llvm.masked.scatter", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], - &[args[0].immediate(), args[1].immediate(), alignment, mask], - )); + args, + ); + if llvm_version >= (22, 0, 0) { + crate::attributes::apply_to_callsite( + call, + crate::llvm::AttributePlace::Argument(1), + &[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)], + ) + } + return Ok(call); } macro_rules! arith_red { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index ca84b6de8b11..982d5cd3ac41 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -14,6 +14,7 @@ #![feature(if_let_guard)] #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] +#![feature(macro_derive)] #![feature(rustdoc_internals)] #![feature(slice_as_array)] #![feature(try_blocks)] @@ -29,8 +30,7 @@ use back::write::{create_informational_target_machine, create_target_machine}; use context::SimpleCx; use errors::ParseTargetMachineConfig; use llvm_util::target_config; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; +use rustc_ast::expand::allocator::AllocatorMethod; use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule}; use rustc_codegen_ssa::back::write::{ CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, @@ -38,7 +38,7 @@ use rustc_codegen_ssa::back::write::{ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetConfig}; use rustc_data_structures::fx::FxIndexMap; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; @@ -46,19 +46,15 @@ use rustc_middle::util::Providers; use rustc_session::Session; use rustc_session::config::{OptLevel, OutputFilenames, PrintKind, PrintRequest}; use rustc_span::Symbol; +use rustc_target::spec::{RelocModel, TlsModel}; -mod back { - pub(crate) mod archive; - pub(crate) mod lto; - pub(crate) mod owned_target_machine; - mod profiling; - pub(crate) mod write; -} +use crate::llvm::ToLlvmBool; mod abi; mod allocator; mod asm; mod attributes; +mod back; mod base; mod builder; mod callee; @@ -72,14 +68,18 @@ mod errors; mod intrinsic; mod llvm; mod llvm_util; +mod macros; mod mono_item; mod type_; mod type_of; +mod typetree; mod va_arg; mod value; rustc_fluent_macro::fluent_messages! { "../messages.ftl" } +pub(crate) use macros::TryFromU32; + #[derive(Clone)] pub struct LlvmCodegenBackend(()); @@ -109,14 +109,13 @@ impl ExtraBackendMethods for LlvmCodegenBackend { &self, tcx: TyCtxt<'tcx>, module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) -> ModuleLlvm { let module_llvm = ModuleLlvm::new_metadata(tcx, module_name); let cx = SimpleCx::new(module_llvm.llmod(), &module_llvm.llcx, tcx.data_layout.pointer_size()); unsafe { - allocator::codegen(tcx, cx, module_name, kind, alloc_error_handler_kind); + allocator::codegen(tcx, cx, module_name, methods); } module_llvm } @@ -173,20 +172,15 @@ impl WriteBackendMethods for LlvmCodegenBackend { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_fncs: Vec, - ) -> Result, FatalError> { + ) -> ModuleCodegen { let mut module = - back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?; - - if !diff_fncs.is_empty() { - builder::autodiff::differentiate(&module, cgcx, diff_fncs)?; - } + back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules); let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - back::lto::run_pass_manager(cgcx, dcx, &mut module, false)?; + back::lto::run_pass_manager(cgcx, dcx, &mut module, false); - Ok(module) + module } fn run_thin_lto( cgcx: &CodegenContext, @@ -194,7 +188,7 @@ impl WriteBackendMethods for LlvmCodegenBackend { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError> { + ) -> (Vec>, Vec) { back::lto::run_thin( cgcx, exported_symbols_for_lto, @@ -208,27 +202,24 @@ impl WriteBackendMethods for LlvmCodegenBackend { dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError> { + ) { back::write::optimize(cgcx, dcx, module, config) } fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError> { + ) -> ModuleCodegen { back::lto::optimize_thin_module(thin, cgcx) } fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result { + ) -> CompiledModule { back::write::codegen(cgcx, module, config) } - fn prepare_thin( - module: ModuleCodegen, - emit_summary: bool, - ) -> (String, Self::ThinBuffer) { - back::lto::prepare_thin(module, emit_summary) + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) } fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) @@ -246,6 +237,10 @@ impl CodegenBackend for LlvmCodegenBackend { crate::DEFAULT_LOCALE_RESOURCE } + fn name(&self) -> &'static str { + "llvm" + } + fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited } @@ -260,16 +255,7 @@ impl CodegenBackend for LlvmCodegenBackend { match req.kind { PrintKind::RelocationModels => { writeln!(out, "Available relocation models:").unwrap(); - for name in &[ - "static", - "pic", - "pie", - "dynamic-no-pic", - "ropi", - "rwpi", - "ropi-rwpi", - "default", - ] { + for name in RelocModel::ALL.iter().map(RelocModel::desc).chain(["default"]) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -283,9 +269,7 @@ impl CodegenBackend for LlvmCodegenBackend { } PrintKind::TlsModels => { writeln!(out, "Available TLS models:").unwrap(); - for name in - &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"] - { + for name in TlsModel::ALL.iter().map(TlsModel::desc) { writeln!(out, " {name}").unwrap(); } writeln!(out).unwrap(); @@ -375,7 +359,14 @@ impl CodegenBackend for LlvmCodegenBackend { // Run the linker on any artifacts that resulted from the LLVM run. // This should produce either a finished executable or library. - link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, metadata, outputs); + link_binary( + sess, + &LlvmArchiveBuilderBuilder, + codegen_results, + metadata, + outputs, + self.name(), + ); } } @@ -394,7 +385,8 @@ unsafe impl Sync for ModuleLlvm {} impl ModuleLlvm { fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self { unsafe { - let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llcx = llvm::LLVMContextCreate(); + llvm::LLVMContextSetDiscardValueNames(llcx, tcx.sess.fewer_names().to_llvm_bool()); let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; ModuleLlvm { llmod_raw, @@ -406,7 +398,8 @@ impl ModuleLlvm { fn new_metadata(tcx: TyCtxt<'_>, mod_name: &str) -> Self { unsafe { - let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llcx = llvm::LLVMContextCreate(); + llvm::LLVMContextSetDiscardValueNames(llcx, tcx.sess.fewer_names().to_llvm_bool()); let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; ModuleLlvm { llmod_raw, @@ -420,12 +413,12 @@ impl ModuleLlvm { cgcx: &CodegenContext, name: &str, dcx: DiagCtxtHandle<'_>, - ) -> Result { + ) -> OwnedTargetMachine { let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name); match (cgcx.tm_factory)(tm_factory_config) { - Ok(m) => Ok(m), + Ok(m) => m, Err(e) => { - return Err(dcx.emit_almost_fatal(ParseTargetMachineConfig(e))); + dcx.emit_fatal(ParseTargetMachineConfig(e)); } } } @@ -435,13 +428,14 @@ impl ModuleLlvm { name: &CStr, buffer: &[u8], dcx: DiagCtxtHandle<'_>, - ) -> Result { + ) -> Self { unsafe { - let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); - let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx)?; - let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx)?; + let llcx = llvm::LLVMContextCreate(); + llvm::LLVMContextSetDiscardValueNames(llcx, cgcx.fewer_names.to_llvm_bool()); + let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx); + let tm = ModuleLlvm::tm_from_cgcx(cgcx, name.to_str().unwrap(), dcx); - Ok(ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) }) + ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) } } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs deleted file mode 100644 index 51bcc4d123d3..000000000000 --- a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! A wrapper around LLVM's archive (.a) code - -use std::path::Path; -use std::{slice, str}; - -use rustc_fs_util::path_to_c_string; - -pub(crate) struct ArchiveRO { - pub raw: &'static mut super::Archive, -} - -unsafe impl Send for ArchiveRO {} - -pub(crate) struct Iter<'a> { - raw: &'a mut super::ArchiveIterator<'a>, -} - -pub(crate) struct Child<'a> { - pub raw: &'a mut super::ArchiveChild<'a>, -} - -impl ArchiveRO { - /// Opens a static archive for read-only purposes. This is more optimized - /// than the `open` method because it uses LLVM's internal `Archive` class - /// rather than shelling out to `ar` for everything. - /// - /// If this archive is used with a mutable method, then an error will be - /// raised. - pub(crate) fn open(dst: &Path) -> Result { - unsafe { - let s = path_to_c_string(dst); - let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { - super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) - })?; - Ok(ArchiveRO { raw: ar }) - } - } - - pub(crate) fn iter(&self) -> Iter<'_> { - unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } - } -} - -impl Drop for ArchiveRO { - fn drop(&mut self) { - unsafe { - super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Iterator for Iter<'a> { - type Item = Result, String>; - - fn next(&mut self) -> Option, String>> { - unsafe { - match super::LLVMRustArchiveIteratorNext(self.raw) { - Some(raw) => Some(Ok(Child { raw })), - None => super::last_error().map(Err), - } - } - } -} - -impl<'a> Drop for Iter<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); - } - } -} - -impl<'a> Child<'a> { - pub(crate) fn name(&self) -> Option<&'a str> { - unsafe { - let mut name_len = 0; - let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); - if name_ptr.is_null() { - None - } else { - let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); - str::from_utf8(name).ok().map(|s| s.trim()) - } - } - } -} - -impl<'a> Drop for Child<'a> { - fn drop(&mut self) { - unsafe { - super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); - } - } -} diff --git a/compiler/rustc_codegen_llvm/src/llvm/conversions.rs b/compiler/rustc_codegen_llvm/src/llvm/conversions.rs new file mode 100644 index 000000000000..9e9f9339ade8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/conversions.rs @@ -0,0 +1,115 @@ +//! Conversions from backend-independent data types to/from LLVM FFI types. + +use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate}; +use rustc_middle::ty::AtomicOrdering; +use rustc_session::config::DebugInfo; +use rustc_target::spec::SymbolVisibility; + +use crate::llvm; + +/// Helper trait for converting backend-independent types to LLVM-specific +/// types, for FFI purposes. +pub(crate) trait FromGeneric { + fn from_generic(other: T) -> Self; +} + +impl FromGeneric for llvm::Visibility { + fn from_generic(visibility: SymbolVisibility) -> Self { + match visibility { + SymbolVisibility::Hidden => Self::Hidden, + SymbolVisibility::Protected => Self::Protected, + SymbolVisibility::Interposable => Self::Default, + } + } +} + +impl FromGeneric for llvm::IntPredicate { + fn from_generic(int_pred: IntPredicate) -> Self { + match int_pred { + IntPredicate::IntEQ => Self::IntEQ, + IntPredicate::IntNE => Self::IntNE, + IntPredicate::IntUGT => Self::IntUGT, + IntPredicate::IntUGE => Self::IntUGE, + IntPredicate::IntULT => Self::IntULT, + IntPredicate::IntULE => Self::IntULE, + IntPredicate::IntSGT => Self::IntSGT, + IntPredicate::IntSGE => Self::IntSGE, + IntPredicate::IntSLT => Self::IntSLT, + IntPredicate::IntSLE => Self::IntSLE, + } + } +} + +impl FromGeneric for llvm::RealPredicate { + fn from_generic(real_pred: RealPredicate) -> Self { + match real_pred { + RealPredicate::RealPredicateFalse => Self::RealPredicateFalse, + RealPredicate::RealOEQ => Self::RealOEQ, + RealPredicate::RealOGT => Self::RealOGT, + RealPredicate::RealOGE => Self::RealOGE, + RealPredicate::RealOLT => Self::RealOLT, + RealPredicate::RealOLE => Self::RealOLE, + RealPredicate::RealONE => Self::RealONE, + RealPredicate::RealORD => Self::RealORD, + RealPredicate::RealUNO => Self::RealUNO, + RealPredicate::RealUEQ => Self::RealUEQ, + RealPredicate::RealUGT => Self::RealUGT, + RealPredicate::RealUGE => Self::RealUGE, + RealPredicate::RealULT => Self::RealULT, + RealPredicate::RealULE => Self::RealULE, + RealPredicate::RealUNE => Self::RealUNE, + RealPredicate::RealPredicateTrue => Self::RealPredicateTrue, + } + } +} + +impl FromGeneric for llvm::AtomicRmwBinOp { + fn from_generic(op: AtomicRmwBinOp) -> Self { + match op { + AtomicRmwBinOp::AtomicXchg => Self::AtomicXchg, + AtomicRmwBinOp::AtomicAdd => Self::AtomicAdd, + AtomicRmwBinOp::AtomicSub => Self::AtomicSub, + AtomicRmwBinOp::AtomicAnd => Self::AtomicAnd, + AtomicRmwBinOp::AtomicNand => Self::AtomicNand, + AtomicRmwBinOp::AtomicOr => Self::AtomicOr, + AtomicRmwBinOp::AtomicXor => Self::AtomicXor, + AtomicRmwBinOp::AtomicMax => Self::AtomicMax, + AtomicRmwBinOp::AtomicMin => Self::AtomicMin, + AtomicRmwBinOp::AtomicUMax => Self::AtomicUMax, + AtomicRmwBinOp::AtomicUMin => Self::AtomicUMin, + } + } +} + +impl FromGeneric for llvm::AtomicOrdering { + fn from_generic(ordering: AtomicOrdering) -> Self { + match ordering { + AtomicOrdering::Relaxed => Self::Monotonic, + AtomicOrdering::Acquire => Self::Acquire, + AtomicOrdering::Release => Self::Release, + AtomicOrdering::AcqRel => Self::AcquireRelease, + AtomicOrdering::SeqCst => Self::SequentiallyConsistent, + } + } +} + +impl FromGeneric for llvm::debuginfo::DebugEmissionKind { + fn from_generic(kind: DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. + match kind { + DebugInfo::None => Self::NoDebug, + DebugInfo::LineDirectivesOnly => Self::DebugDirectivesOnly, + DebugInfo::LineTablesOnly => Self::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => Self::FullDebug, + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs index 0e0f2b0eab01..59b2cd329ae7 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -6,7 +6,7 @@ use rustc_span::InnerSpan; pub(crate) use self::Diagnostic::*; use self::OptimizationDiagnosticKind::*; use super::{DiagnosticInfo, SMDiagnostic}; -use crate::value::Value; +use crate::llvm::Value; #[derive(Copy, Clone, Debug)] pub(crate) enum OptimizationDiagnosticKind { diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 56d756e52cce..e63043b21227 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -3,9 +3,36 @@ use libc::{c_char, c_uint}; use super::MetadataKindId; -use super::ffi::{AttributeKind, BasicBlock, Metadata, Module, Type, Value}; +use super::ffi::{AttributeKind, BasicBlock, Context, Metadata, Module, Type, Value}; use crate::llvm::{Bool, Builder}; +// TypeTree types +pub(crate) type CTypeTreeRef = *mut EnzymeTypeTree; + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct EnzymeTypeTree { + _unused: [u8; 0], +} + +#[repr(u32)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +#[allow(non_camel_case_types)] +pub(crate) enum CConcreteType { + DT_Anything = 0, + DT_Integer = 1, + DT_Pointer = 2, + DT_Half = 3, + DT_Float = 4, + DT_Double = 5, + DT_Unknown = 6, + DT_FP128 = 9, +} + +pub(crate) struct TypeTree { + pub(crate) inner: CTypeTreeRef, +} + #[link(name = "llvm-wrapper", kind = "static")] unsafe extern "C" { // Enzyme @@ -59,19 +86,49 @@ pub(crate) enum LLVMRustVerifierFailureAction { LLVMReturnStatusAction = 2, } -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) use self::Enzyme_AD::*; -#[cfg(llvm_enzyme)] +#[cfg(feature = "llvm_enzyme")] pub(crate) mod Enzyme_AD { use std::ffi::{CString, c_char}; use libc::c_void; + use super::{CConcreteType, CTypeTreeRef, Context}; + unsafe extern "C" { pub(crate) fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8); pub(crate) fn EnzymeSetCLString(arg1: *mut ::std::os::raw::c_void, arg2: *const c_char); } + + // TypeTree functions + unsafe extern "C" { + pub(crate) fn EnzymeNewTypeTree() -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef; + pub(crate) fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef; + pub(crate) fn EnzymeFreeTypeTree(CTT: CTypeTreeRef); + pub(crate) fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool; + pub(crate) fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64); + pub(crate) fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef); + pub(crate) fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ); + pub(crate) fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ); + pub(crate) fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char; + pub(crate) fn EnzymeTypeTreeToStringFree(arg1: *const c_char); + } + unsafe extern "C" { static mut EnzymePrintPerf: c_void; static mut EnzymePrintActivity: c_void; @@ -134,13 +191,74 @@ pub(crate) mod Enzyme_AD { } } -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) use self::Fallback_AD::*; -#[cfg(not(llvm_enzyme))] +#[cfg(not(feature = "llvm_enzyme"))] pub(crate) mod Fallback_AD { #![allow(unused_variables)] + use libc::c_char; + + use super::{CConcreteType, CTypeTreeRef, Context}; + + // TypeTree function fallbacks + pub(crate) unsafe fn EnzymeNewTypeTree() -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeCT(arg1: CConcreteType, ctx: &Context) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeNewTypeTreeTR(arg1: CTypeTreeRef) -> CTypeTreeRef { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeFreeTypeTree(CTT: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeMergeTypeTree(arg1: CTypeTreeRef, arg2: CTypeTreeRef) -> bool { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeOnlyEq(arg1: CTypeTreeRef, pos: i64) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeData0Eq(arg1: CTypeTreeRef) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeShiftIndiciesEq( + arg1: CTypeTreeRef, + data_layout: *const c_char, + offset: i64, + max_size: i64, + add_offset: u64, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeInsertEq( + CTT: CTypeTreeRef, + indices: *const i64, + len: usize, + ct: CConcreteType, + ctx: &Context, + ) { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToString(arg1: CTypeTreeRef) -> *const c_char { + unimplemented!() + } + + pub(crate) unsafe fn EnzymeTypeTreeToStringFree(arg1: *const c_char) { + unimplemented!() + } + pub(crate) fn set_inline(val: bool) { unimplemented!() } @@ -169,3 +287,89 @@ pub(crate) mod Fallback_AD { unimplemented!() } } + +impl TypeTree { + pub(crate) fn new() -> TypeTree { + let inner = unsafe { EnzymeNewTypeTree() }; + TypeTree { inner } + } + + pub(crate) fn from_type(t: CConcreteType, ctx: &Context) -> TypeTree { + let inner = unsafe { EnzymeNewTypeTreeCT(t, ctx) }; + TypeTree { inner } + } + + pub(crate) fn merge(self, other: Self) -> Self { + unsafe { + EnzymeMergeTypeTree(self.inner, other.inner); + } + drop(other); + self + } + + #[must_use] + pub(crate) fn shift( + self, + layout: &str, + offset: isize, + max_size: isize, + add_offset: usize, + ) -> Self { + let layout = std::ffi::CString::new(layout).unwrap(); + + unsafe { + EnzymeTypeTreeShiftIndiciesEq( + self.inner, + layout.as_ptr(), + offset as i64, + max_size as i64, + add_offset as u64, + ); + } + + self + } + + pub(crate) fn insert(&mut self, indices: &[i64], ct: CConcreteType, ctx: &Context) { + unsafe { + EnzymeTypeTreeInsertEq(self.inner, indices.as_ptr(), indices.len(), ct, ctx); + } + } +} + +impl Clone for TypeTree { + fn clone(&self) -> Self { + let inner = unsafe { EnzymeNewTypeTreeTR(self.inner) }; + TypeTree { inner } + } +} + +impl std::fmt::Display for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let ptr = unsafe { EnzymeTypeTreeToString(self.inner) }; + let cstr = unsafe { std::ffi::CStr::from_ptr(ptr) }; + match cstr.to_str() { + Ok(x) => write!(f, "{}", x)?, + Err(err) => write!(f, "could not parse: {}", err)?, + } + + // delete C string pointer + unsafe { + EnzymeTypeTreeToStringFree(ptr); + } + + Ok(()) + } +} + +impl std::fmt::Debug for TypeTree { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + ::fmt(self, f) + } +} + +impl Drop for TypeTree { + fn drop(&mut self) { + unsafe { EnzymeFreeTypeTree(self.inner) } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 0d0cb5f139ee..9a391d57d6fb 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -11,32 +11,79 @@ //! the need for an extra cast from `*const u8` on the Rust side. #![allow(non_camel_case_types)] -#![allow(non_upper_case_globals)] -use std::fmt::Debug; +use std::fmt::{self, Debug}; use std::marker::PhantomData; use std::num::NonZero; use std::ptr; use bitflags::bitflags; use libc::{c_char, c_int, c_uchar, c_uint, c_ulonglong, c_void, size_t}; -use rustc_macros::TryFromU32; -use rustc_target::spec::SymbolVisibility; use super::RustString; use super::debuginfo::{ - DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, - DIFile, DIFlags, DIGlobalVariableExpression, DILocation, DISPFlags, DIScope, DISubprogram, - DISubrange, DITemplateTypeParameter, DIType, DIVariable, DebugEmissionKind, DebugNameTableKind, + DIArray, DIBuilder, DIDerivedType, DIDescriptor, DIEnumerator, DIFile, DIFlags, DILocation, + DISPFlags, DIScope, DISubprogram, DITemplateTypeParameter, DIType, DebugEmissionKind, + DebugNameTableKind, }; -use crate::llvm; +use crate::llvm::MetadataKindId; +use crate::{TryFromU32, llvm}; /// In the LLVM-C API, boolean values are passed as `typedef int LLVMBool`, /// which has a different ABI from Rust or C++ `bool`. -pub(crate) type Bool = c_int; +/// +/// This wrapper does not implement `PartialEq`. +/// To test the underlying boolean value, use [`Self::is_true`]. +#[derive(Clone, Copy)] +#[repr(transparent)] +pub(crate) struct Bool { + value: c_int, +} -pub(crate) const True: Bool = 1 as Bool; -pub(crate) const False: Bool = 0 as Bool; +pub(crate) const TRUE: Bool = Bool::TRUE; +pub(crate) const FALSE: Bool = Bool::FALSE; + +impl Bool { + pub(crate) const TRUE: Self = Self { value: 1 }; + pub(crate) const FALSE: Self = Self { value: 0 }; + + pub(crate) const fn from_bool(rust_bool: bool) -> Self { + if rust_bool { Self::TRUE } else { Self::FALSE } + } + + /// Converts this LLVM-C boolean to a Rust `bool` + pub(crate) fn is_true(self) -> bool { + // Since we're interacting with a C API, follow the C convention of + // treating any nonzero value as true. + self.value != Self::FALSE.value + } +} + +impl Debug for Bool { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.value { + 0 => f.write_str("FALSE"), + 1 => f.write_str("TRUE"), + // As with `Self::is_true`, treat any nonzero value as true. + v => write!(f, "TRUE ({v})"), + } + } +} + +/// Convenience trait to convert `bool` to `llvm::Bool` with an explicit method call. +/// +/// Being able to write `b.to_llvm_bool()` is less noisy than `llvm::Bool::from(b)`, +/// while being more explicit and less mistake-prone than something like `b.into()`. +pub(crate) trait ToLlvmBool: Copy { + fn to_llvm_bool(self) -> llvm::Bool; +} + +impl ToLlvmBool for bool { + #[inline(always)] + fn to_llvm_bool(self) -> llvm::Bool { + llvm::Bool::from_bool(self) + } +} /// Wrapper for a raw enum value returned from LLVM's C APIs. /// @@ -97,6 +144,17 @@ pub(crate) enum ModuleFlagMergeBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. +/// Must match the layout of `LLVMTailCallKind`. +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) enum TailCallKind { + None = 0, + Tail = 1, + MustTail = 2, + NoTail = 3, +} + /// LLVM CallingConv::ID. Should we wrap this? /// /// See @@ -160,16 +218,6 @@ pub(crate) enum Visibility { Protected = 2, } -impl Visibility { - pub(crate) fn from_generic(visibility: SymbolVisibility) -> Self { - match visibility { - SymbolVisibility::Hidden => Visibility::Hidden, - SymbolVisibility::Protected => Visibility::Protected, - SymbolVisibility::Interposable => Visibility::Default, - } - } -} - /// LLVMUnnamedAddr #[repr(C)] pub(crate) enum UnnamedAddr { @@ -204,7 +252,7 @@ pub(crate) enum AttributeKind { MinSize = 4, Naked = 5, NoAlias = 6, - NoCapture = 7, + CapturesAddress = 7, NoInline = 8, NonNull = 9, NoRedZone = 10, @@ -239,6 +287,9 @@ pub(crate) enum AttributeKind { FnRetThunkExtern = 41, Writable = 42, DeadOnUnwind = 43, + DeadOnReturn = 44, + CapturesReadOnly = 45, + CapturesNone = 46, } /// LLVMIntPredicate @@ -257,24 +308,6 @@ pub(crate) enum IntPredicate { IntSLE = 41, } -impl IntPredicate { - pub(crate) fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { - use rustc_codegen_ssa::common::IntPredicate as Common; - match intpre { - Common::IntEQ => Self::IntEQ, - Common::IntNE => Self::IntNE, - Common::IntUGT => Self::IntUGT, - Common::IntUGE => Self::IntUGE, - Common::IntULT => Self::IntULT, - Common::IntULE => Self::IntULE, - Common::IntSGT => Self::IntSGT, - Common::IntSGE => Self::IntSGE, - Common::IntSLT => Self::IntSLT, - Common::IntSLE => Self::IntSLE, - } - } -} - /// LLVMRealPredicate #[derive(Copy, Clone)] #[repr(C)] @@ -297,34 +330,15 @@ pub(crate) enum RealPredicate { RealPredicateTrue = 15, } -impl RealPredicate { - pub(crate) fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { - use rustc_codegen_ssa::common::RealPredicate as Common; - match realp { - Common::RealPredicateFalse => Self::RealPredicateFalse, - Common::RealOEQ => Self::RealOEQ, - Common::RealOGT => Self::RealOGT, - Common::RealOGE => Self::RealOGE, - Common::RealOLT => Self::RealOLT, - Common::RealOLE => Self::RealOLE, - Common::RealONE => Self::RealONE, - Common::RealORD => Self::RealORD, - Common::RealUNO => Self::RealUNO, - Common::RealUEQ => Self::RealUEQ, - Common::RealUGT => Self::RealUGT, - Common::RealUGE => Self::RealUGE, - Common::RealULT => Self::RealULT, - Common::RealULE => Self::RealULE, - Common::RealUNE => Self::RealUNE, - Common::RealPredicateTrue => Self::RealPredicateTrue, - } - } -} - -/// LLVMTypeKind -#[derive(Copy, Clone, PartialEq, Debug)] +/// Must match the layout of `LLVMTypeKind`. +/// +/// Use [`RawEnum`] for values of `LLVMTypeKind` returned from LLVM, +/// to avoid risk of UB if LLVM adds new enum values. +/// +/// All of LLVM's variants should be declared here, even if no Rust-side code refers +/// to them, because unknown variants will cause [`RawEnum::to_rust`] to panic. +#[derive(Copy, Clone, PartialEq, Debug, TryFromU32)] #[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] pub(crate) enum TypeKind { Void = 0, Half = 1, @@ -391,25 +405,6 @@ pub(crate) enum AtomicRmwBinOp { AtomicUMin = 10, } -impl AtomicRmwBinOp { - pub(crate) fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { - use rustc_codegen_ssa::common::AtomicRmwBinOp as Common; - match op { - Common::AtomicXchg => Self::AtomicXchg, - Common::AtomicAdd => Self::AtomicAdd, - Common::AtomicSub => Self::AtomicSub, - Common::AtomicAnd => Self::AtomicAnd, - Common::AtomicNand => Self::AtomicNand, - Common::AtomicOr => Self::AtomicOr, - Common::AtomicXor => Self::AtomicXor, - Common::AtomicMax => Self::AtomicMax, - Common::AtomicMin => Self::AtomicMin, - Common::AtomicUMax => Self::AtomicUMax, - Common::AtomicUMin => Self::AtomicUMin, - } - } -} - /// LLVMAtomicOrdering #[derive(Copy, Clone)] #[repr(C)] @@ -426,19 +421,6 @@ pub(crate) enum AtomicOrdering { SequentiallyConsistent = 7, } -impl AtomicOrdering { - pub(crate) fn from_generic(ao: rustc_middle::ty::AtomicOrdering) -> Self { - use rustc_middle::ty::AtomicOrdering as Common; - match ao { - Common::Relaxed => Self::Monotonic, - Common::Acquire => Self::Acquire, - Common::Release => Self::Release, - Common::AcqRel => Self::AcquireRelease, - Common::SeqCst => Self::SequentiallyConsistent, - } - } -} - /// LLVMRustFileType #[derive(Copy, Clone)] #[repr(C)] @@ -447,31 +429,6 @@ pub(crate) enum FileType { ObjectFile, } -/// LLVMMetadataType -#[derive(Copy, Clone)] -#[repr(C)] -#[expect(dead_code, reason = "Some variants are unused, but are kept to match LLVM-C")] -pub(crate) enum MetadataType { - MD_dbg = 0, - MD_tbaa = 1, - MD_prof = 2, - MD_fpmath = 3, - MD_range = 4, - MD_tbaa_struct = 5, - MD_invariant_load = 6, - MD_alias_scope = 7, - MD_noalias = 8, - MD_nontemporal = 9, - MD_mem_parallel_loop_access = 10, - MD_nonnull = 11, - MD_unpredictable = 15, - MD_align = 17, - MD_type = 19, - MD_vcall_visibility = 28, - MD_noundef = 29, - MD_kcfi_type = 36, -} - /// Must match the layout of `LLVMInlineAsmDialect`. #[derive(Copy, Clone, PartialEq)] #[repr(C)] @@ -599,17 +556,6 @@ pub(crate) enum DiagnosticLevel { Remark, } -/// LLVMRustArchiveKind -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) enum ArchiveKind { - K_GNU, - K_BSD, - K_DARWIN, - K_COFF, - K_AIXBIG, -} - unsafe extern "C" { // LLVMRustThinLTOData pub(crate) type ThinLTOData; @@ -655,6 +601,7 @@ pub(crate) enum MemoryEffects { None, ReadOnly, InaccessibleMemOnly, + ReadOnlyNotPure, } /// LLVMOpcode @@ -731,6 +678,15 @@ pub(crate) enum Opcode { CatchSwitch = 65, } +/// Must match the layout of `LLVMRustCompressionKind`. +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) enum CompressionKind { + None = 0, + Zlib = 1, + Zstd = 2, +} + unsafe extern "C" { type Opaque; } @@ -751,6 +707,8 @@ unsafe extern "C" { pub(crate) type Metadata; pub(crate) type BasicBlock; pub(crate) type Comdat; + /// `&'ll DbgRecord` represents `LLVMDbgRecordRef`. + pub(crate) type DbgRecord; } #[repr(C)] pub(crate) struct Builder<'a>(InvariantOpaque<'a>); @@ -758,19 +716,12 @@ pub(crate) struct Builder<'a>(InvariantOpaque<'a>); pub(crate) struct PassManager<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub type TargetMachine; - pub(crate) type Archive; } -#[repr(C)] -pub(crate) struct ArchiveIterator<'a>(InvariantOpaque<'a>); -#[repr(C)] -pub(crate) struct ArchiveChild<'a>(InvariantOpaque<'a>); unsafe extern "C" { pub(crate) type Twine; pub(crate) type DiagnosticInfo; pub(crate) type SMDiagnostic; } -#[repr(C)] -pub(crate) struct RustArchiveMember<'a>(InvariantOpaque<'a>); /// Opaque pointee of `LLVMOperandBundleRef`. #[repr(C)] pub(crate) struct OperandBundle<'a>(InvariantOpaque<'a>); @@ -840,9 +791,7 @@ pub(crate) mod debuginfo { pub(crate) type DIDerivedType = DIType; pub(crate) type DICompositeType = DIDerivedType; pub(crate) type DIVariable = DIDescriptor; - pub(crate) type DIGlobalVariableExpression = DIDescriptor; pub(crate) type DIArray = DIDescriptor; - pub(crate) type DISubrange = DIDescriptor; pub(crate) type DIEnumerator = DIDescriptor; pub(crate) type DITemplateTypeParameter = DIDescriptor; @@ -914,28 +863,6 @@ pub(crate) mod debuginfo { DebugDirectivesOnly, } - impl DebugEmissionKind { - pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { - // We should be setting LLVM's emission kind to `LineTablesOnly` if - // we are compiling with "limited" debuginfo. However, some of the - // existing tools relied on slightly more debuginfo being generated than - // would be the case with `LineTablesOnly`, and we did not want to break - // these tools in a "drive-by fix", without a good idea or plan about - // what limited debuginfo should exactly look like. So for now we are - // instead adding a new debuginfo option "line-tables-only" so as to - // not break anything and to allow users to have 'limited' debug info. - // - // See https://github.com/rust-lang/rust/issues/60020 for details. - use rustc_session::config::DebugInfo; - match kind { - DebugInfo::None => DebugEmissionKind::NoDebug, - DebugInfo::LineDirectivesOnly => DebugEmissionKind::DebugDirectivesOnly, - DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, - DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, - } - } - } - /// LLVMRustDebugNameTableKind #[derive(Clone, Copy)] #[repr(C)] @@ -985,25 +912,19 @@ pub(crate) type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -> *mut c_void; pub(crate) type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void; -#[derive(Copy, Clone)] -#[repr(transparent)] -pub(crate) struct MetadataKindId(c_uint); - -impl From for MetadataKindId { - fn from(value: MetadataType) -> Self { - Self(value as c_uint) - } -} - unsafe extern "C" { // Create and destroy contexts. + pub(crate) fn LLVMContextCreate() -> &'static mut Context; pub(crate) fn LLVMContextDispose(C: &'static mut Context); + pub(crate) fn LLVMContextSetDiscardValueNames(C: &Context, Discard: Bool); pub(crate) fn LLVMGetMDKindIDInContext( C: &Context, Name: *const c_char, SLen: c_uint, ) -> MetadataKindId; + pub(crate) fn LLVMDisposeTargetMachine(T: ptr::NonNull); + // Create modules. pub(crate) fn LLVMModuleCreateWithNameInContext( ModuleID: *const c_char, @@ -1035,13 +956,15 @@ unsafe extern "C" { CanThrow: llvm::Bool, ) -> &'ll Value; + pub(crate) safe fn LLVMGetTypeKind(Ty: &Type) -> RawEnum; + // Operations on integer types pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt16TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt32TypeInContext(C: &Context) -> &Type; pub(crate) fn LLVMInt64TypeInContext(C: &Context) -> &Type; - pub(crate) fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + pub(crate) safe fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; @@ -1070,7 +993,7 @@ unsafe extern "C" { ) -> &'a Type; // Operations on array, pointer, and vector types (sequence types) - pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; + pub(crate) safe fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type; @@ -1085,7 +1008,11 @@ unsafe extern "C" { pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub(crate) safe fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: MetadataKindId, Node: &'a Value); - pub(crate) fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub(crate) fn LLVMGlobalSetMetadata<'a>( + Val: &'a Value, + KindID: MetadataKindId, + Metadata: &'a Metadata, + ); pub(crate) safe fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1186,6 +1113,8 @@ unsafe extern "C" { pub(crate) safe fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; pub(crate) safe fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub(crate) safe fn LLVMSetTailCallKind(CallInst: &Value, kind: TailCallKind); + pub(crate) safe fn LLVMSetExternallyInitialized(GlobalVar: &Value, IsExtInit: Bool); // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( @@ -1198,6 +1127,7 @@ unsafe extern "C" { // Operations on functions pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub(crate) fn LLVMDeleteFunction(Fn: &Value); // Operations about llvm intrinsics pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint; @@ -1227,12 +1157,15 @@ unsafe extern "C" { pub(crate) fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; pub(crate) fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; pub(crate) fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>; + pub(crate) fn LLVMGetNextInstruction(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMInstructionEraseFromParent(Val: &Value); // Operations on call sites pub(crate) fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); // Operations on load/store instructions (only) pub(crate) fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + pub(crate) fn LLVMSetOrdering(MemoryAccessInst: &Value, Ordering: AtomicOrdering); // Operations on phi nodes pub(crate) fn LLVMAddIncoming<'a>( @@ -1819,6 +1752,202 @@ unsafe extern "C" { Scope: &'ll Metadata, InlinedAt: Option<&'ll Metadata>, ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateSubroutineType<'ll>( + Builder: &DIBuilder<'ll>, + File: Option<&'ll Metadata>, // (ignored and has no effect) + ParameterTypes: *const Option<&'ll Metadata>, + NumParameterTypes: c_uint, + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateUnionType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateArrayType<'ll>( + Builder: &DIBuilder<'ll>, + Size: u64, + Align: u32, + Ty: &'ll Metadata, + Subscripts: *const &'ll Metadata, + NumSubscripts: c_uint, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateBasicType<'ll>( + Builder: &DIBuilder<'ll>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, // (`LLVMDWARFTypeEncoding`) + Flags: DIFlags, // (default is `DIFlags::DIFlagZero`) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreatePointerType<'ll>( + Builder: &DIBuilder<'ll>, + PointeeTy: &'ll Metadata, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, // (optional DWARF address space; default is 0) + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStructType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'ll Metadata>, + Elements: *const Option<&'ll Metadata>, + NumElements: c_uint, + RunTimeLang: c_uint, // (optional Objective-C runtime version; default is 0) + VTableHolder: Option<&'ll Metadata>, + UniqueId: *const c_uchar, // See "PTR_LEN_STR". + UniqueIdLen: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateStaticMemberType<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNumber: c_uint, + Type: &'ll Metadata, + Flags: DIFlags, + ConstantVal: Option<&'ll Value>, + AlignInBits: u32, + ) -> &'ll Metadata; + + /// Creates a "qualified type" in the C/C++ sense, by adding modifiers + /// like `const` or `volatile`. + pub(crate) fn LLVMDIBuilderCreateQualifiedType<'ll>( + Builder: &DIBuilder<'ll>, + Tag: c_uint, // (DWARF tag, e.g. `DW_TAG_const_type`) + Type: &'ll Metadata, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateTypedef<'ll>( + Builder: &DIBuilder<'ll>, + Type: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Scope: Option<&'ll Metadata>, + AlignInBits: u32, // (optional; default is 0) + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateSubrange<'ll>( + Builder: &DIBuilder<'ll>, + LowerBound: i64, + Count: i64, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderGetOrCreateArray<'ll>( + Builder: &DIBuilder<'ll>, + Data: *const Option<&'ll Metadata>, + NumElements: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateExpression<'ll>( + Builder: &DIBuilder<'ll>, + Addr: *const u64, + Length: size_t, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateGlobalVariableExpression<'ll>( + Builder: &DIBuilder<'ll>, + Scope: Option<&'ll Metadata>, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + Linkage: *const c_uchar, // See "PTR_LEN_STR". + LinkLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + LocalToUnit: llvm::Bool, + Expr: &'ll Metadata, + Decl: Option<&'ll Metadata>, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderInsertDeclareRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Storage: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderInsertDbgValueRecordAtEnd<'ll>( + Builder: &DIBuilder<'ll>, + Val: &'ll Value, + VarInfo: &'ll Metadata, + Expr: &'ll Metadata, + DebugLoc: &'ll Metadata, + Block: &'ll BasicBlock, + ) -> &'ll DbgRecord; + + pub(crate) fn LLVMDIBuilderCreateAutoVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + AlignInBits: u32, + ) -> &'ll Metadata; + + pub(crate) fn LLVMDIBuilderCreateParameterVariable<'ll>( + Builder: &DIBuilder<'ll>, + Scope: &'ll Metadata, + Name: *const c_uchar, // See "PTR_LEN_STR". + NameLen: size_t, + ArgNo: c_uint, + File: &'ll Metadata, + LineNo: c_uint, + Ty: &'ll Metadata, + AlwaysPreserve: llvm::Bool, // "If true, this descriptor will survive optimizations." + Flags: DIFlags, + ) -> &'ll Metadata; } #[link(name = "llvm-wrapper", kind = "static")] @@ -1826,16 +1955,10 @@ unsafe extern "C" { pub(crate) fn LLVMRustInstallErrorHandlers(); pub(crate) fn LLVMRustDisableSystemDialogsOnCrash(); - // Create and destroy contexts. - pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; - - /// See llvm::LLVMTypeKind::getTypeID. - pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; - // Operations on all values pub(crate) fn LLVMRustGlobalAddMetadata<'a>( Val: &'a Value, - KindID: c_uint, + KindID: MetadataKindId, Metadata: &'a Metadata, ); pub(crate) fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool; @@ -1859,7 +1982,6 @@ unsafe extern "C" { NameLen: size_t, T: &'a Type, ) -> &'a Value; - pub(crate) fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; pub(crate) fn LLVMRustGetNamedValue( M: &Module, Name: *const c_char, @@ -1881,11 +2003,17 @@ unsafe extern "C" { C: &Context, effects: MemoryEffects, ) -> &Attribute; + /// ## Safety + /// - Each of `LowerWords` and `UpperWords` must point to an array that is + /// long enough to fully define an integer of size `NumBits`, i.e. each + /// pointer must point to `NumBits.div_ceil(64)` elements or more. + /// - The implementation will make its own copy of the pointed-to `u64` + /// values, so the pointers only need to outlive this function call. pub(crate) fn LLVMRustCreateRangeAttribute( C: &Context, - num_bits: c_uint, - lower_words: *const u64, - upper_words: *const u64, + NumBits: c_uint, + LowerWords: *const u64, + UpperWords: *const u64, ) -> &Attribute; // Operations on functions @@ -1942,69 +2070,6 @@ unsafe extern "C" { IsVolatile: bool, ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceFAdd<'a>( - B: &Builder<'a>, - Acc: &'a Value, - Src: &'a Value, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceFMul<'a>( - B: &Builder<'a>, - Acc: &'a Value, - Src: &'a Value, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceMin<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsSigned: bool, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceMax<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsSigned: bool, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceFMin<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsNaN: bool, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildVectorReduceFMax<'a>( - B: &Builder<'a>, - Src: &'a Value, - IsNaN: bool, - ) -> &'a Value; - - pub(crate) fn LLVMRustBuildMinNum<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - ) -> &'a Value; - pub(crate) fn LLVMRustBuildMaxNum<'a>( - B: &Builder<'a>, - LHS: &'a Value, - RHS: &'a Value, - ) -> &'a Value; - - // Atomic Operations - pub(crate) fn LLVMRustBuildAtomicLoad<'a>( - B: &Builder<'a>, - ElementType: &'a Type, - PointerVal: &'a Value, - Name: *const c_char, - Order: AtomicOrdering, - ) -> &'a Value; - - pub(crate) fn LLVMRustBuildAtomicStore<'a>( - B: &Builder<'a>, - Val: &'a Value, - Ptr: &'a Value, - Order: AtomicOrdering, - ) -> &'a Value; - pub(crate) fn LLVMRustTimeTraceProfilerInitialize(); pub(crate) fn LLVMRustTimeTraceProfilerFinishThread(); @@ -2026,8 +2091,11 @@ unsafe extern "C" { ConstraintsLen: size_t, ) -> bool; + /// A list of pointer-length strings is passed as two pointer-length slices, + /// one slice containing pointers and one slice containing their corresponding + /// lengths. The implementation will check that both slices have the same length. pub(crate) fn LLVMRustCoverageWriteFilenamesToBuffer( - Filenames: *const *const c_char, + Filenames: *const *const c_uchar, // See "PTR_LEN_STR". FilenamesLen: size_t, Lengths: *const size_t, LengthsLen: size_t, @@ -2045,27 +2113,30 @@ unsafe extern "C" { NumExpansionRegions: size_t, BranchRegions: *const crate::coverageinfo::ffi::BranchRegion, NumBranchRegions: size_t, - MCDCBranchRegions: *const crate::coverageinfo::ffi::MCDCBranchRegion, - NumMCDCBranchRegions: size_t, - MCDCDecisionRegions: *const crate::coverageinfo::ffi::MCDCDecisionRegion, - NumMCDCDecisionRegions: size_t, BufferOut: &RustString, ); pub(crate) fn LLVMRustCoverageCreatePGOFuncNameVar( F: &Value, - FuncName: *const c_char, + FuncName: *const c_uchar, // See "PTR_LEN_STR". FuncNameLen: size_t, ) -> &Value; - pub(crate) fn LLVMRustCoverageHashBytes(Bytes: *const c_char, NumBytes: size_t) -> u64; + pub(crate) fn LLVMRustCoverageHashBytes( + Bytes: *const c_uchar, // See "PTR_LEN_STR". + NumBytes: size_t, + ) -> u64; - pub(crate) fn LLVMRustCoverageWriteCovmapSectionNameToString(M: &Module, OutStr: &RustString); + pub(crate) safe fn LLVMRustCoverageWriteCovmapSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovfunSectionNameToString( + M: &Module, + OutStr: &RustString, + ); + pub(crate) safe fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); - pub(crate) fn LLVMRustCoverageWriteCovfunSectionNameToString(M: &Module, OutStr: &RustString); - - pub(crate) fn LLVMRustCoverageWriteCovmapVarNameToString(OutStr: &RustString); - - pub(crate) fn LLVMRustCoverageMappingVersion() -> u32; + pub(crate) safe fn LLVMRustCoverageMappingVersion() -> u32; pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; pub(crate) fn LLVMRustVersionMajor() -> u32; pub(crate) fn LLVMRustVersionMinor() -> u32; @@ -2122,11 +2193,6 @@ unsafe extern "C" { SourceLen: size_t, ) -> &'a DIFile; - pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( - Builder: &DIBuilder<'a>, - ParameterTypes: &'a DIArray, - ) -> &'a DICompositeType; - pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIDescriptor, @@ -2160,66 +2226,6 @@ unsafe extern "C" { TParam: &'a DIArray, ) -> &'a DISubprogram; - pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( - Builder: &DIBuilder<'a>, - Name: *const c_char, - NameLen: size_t, - SizeInBits: u64, - Encoding: c_uint, - ) -> &'a DIBasicType; - - pub(crate) fn LLVMRustDIBuilderCreateTypedef<'a>( - Builder: &DIBuilder<'a>, - Type: &'a DIBasicType, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Scope: Option<&'a DIScope>, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( - Builder: &DIBuilder<'a>, - PointeeTy: &'a DIType, - SizeInBits: u64, - AlignInBits: u32, - AddressSpace: c_uint, - Name: *const c_char, - NameLen: size_t, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIDescriptor>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - DerivedFrom: Option<&'a DIType>, - Elements: &'a DIArray, - RunTimeLang: c_uint, - VTableHolder: Option<&'a DIType>, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DICompositeType; - - pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - SizeInBits: u64, - AlignInBits: u32, - OffsetInBits: u64, - Flags: DIFlags, - Ty: &'a DIType, - ) -> &'a DIDerivedType; - pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2235,86 +2241,6 @@ unsafe extern "C" { Ty: &'a DIType, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateStaticMemberType<'a>( - Builder: &DIBuilder<'a>, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - Flags: DIFlags, - val: Option<&'a Value>, - AlignInBits: u32, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateQualifiedType<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Type: &'a DIType, - ) -> &'a DIDerivedType; - - pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( - Builder: &DIBuilder<'a>, - Context: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - LinkageName: *const c_char, - LinkageNameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - isLocalToUnit: bool, - Val: &'a Value, - Decl: Option<&'a DIDescriptor>, - AlignInBits: u32, - ) -> &'a DIGlobalVariableExpression; - - pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( - Builder: &DIBuilder<'a>, - Tag: c_uint, - Scope: &'a DIDescriptor, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNo: c_uint, - Ty: &'a DIType, - AlwaysPreserve: bool, - Flags: DIFlags, - ArgNo: c_uint, - AlignInBits: u32, - ) -> &'a DIVariable; - - pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( - Builder: &DIBuilder<'a>, - Size: u64, - AlignInBits: u32, - Ty: &'a DIType, - Subscripts: &'a DIArray, - ) -> &'a DIType; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( - Builder: &DIBuilder<'a>, - Lo: i64, - Count: i64, - ) -> &'a DISubrange; - - pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( - Builder: &DIBuilder<'a>, - Ptr: *const Option<&'a DIDescriptor>, - Count: c_uint, - ) -> &'a DIArray; - - pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( - Builder: &DIBuilder<'a>, - Val: &'a Value, - VarInfo: &'a DIVariable, - AddrOps: *const u64, - AddrOpsCount: c_uint, - DL: &'a DILocation, - InsertAtEnd: &'a BasicBlock, - ); - pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( Builder: &DIBuilder<'a>, Name: *const c_char, @@ -2338,22 +2264,6 @@ unsafe extern "C" { IsScoped: bool, ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( - Builder: &DIBuilder<'a>, - Scope: Option<&'a DIScope>, - Name: *const c_char, - NameLen: size_t, - File: &'a DIFile, - LineNumber: c_uint, - SizeInBits: u64, - AlignInBits: u32, - Flags: DIFlags, - Elements: Option<&'a DIArray>, - RunTimeLang: c_uint, - UniqueId: *const c_char, - UniqueIdLen: size_t, - ) -> &'a DIType; - pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( Builder: &DIBuilder<'a>, Scope: &'a DIScope, @@ -2428,14 +2338,11 @@ unsafe extern "C" { UseInitArray: bool, SplitDwarfFile: *const c_char, OutputObjFile: *const c_char, - DebugInfoCompression: *const c_char, + DebugInfoCompression: CompressionKind, UseEmulatedTls: bool, - ArgsCstrBuff: *const c_char, - ArgsCstrBuffLen: usize, UseWasmEH: bool, ) -> *mut TargetMachine; - pub(crate) fn LLVMRustDisposeTargetMachine(T: *mut TargetMachine); pub(crate) fn LLVMRustAddLibraryInfo<'a>( PM: &PassManager<'a>, M: &'a Module, @@ -2497,19 +2404,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); pub(crate) fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); - pub(crate) fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; - pub(crate) fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; - pub(crate) fn LLVMRustArchiveIteratorNext<'a>( - AIR: &ArchiveIterator<'a>, - ) -> Option<&'a mut ArchiveChild<'a>>; - pub(crate) fn LLVMRustArchiveChildName( - ACR: &ArchiveChild<'_>, - size: &mut size_t, - ) -> *const c_char; - pub(crate) fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); - pub(crate) fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); - pub(crate) fn LLVMRustDestroyArchive(AR: &'static mut Archive); - pub(crate) fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); pub(crate) fn LLVMRustUnpackOptimizationDiagnostic<'a>( @@ -2547,21 +2441,6 @@ unsafe extern "C" { num_ranges: &mut usize, ) -> bool; - pub(crate) fn LLVMRustWriteArchive( - Dst: *const c_char, - NumMembers: size_t, - Members: *const &RustArchiveMember<'_>, - WriteSymbtab: bool, - Kind: ArchiveKind, - isEC: bool, - ) -> LLVMRustResult; - pub(crate) fn LLVMRustArchiveMemberNew<'a>( - Filename: *const c_char, - Name: *const c_char, - Child: Option<&ArchiveChild<'a>>, - ) -> &'a mut RustArchiveMember<'a>; - pub(crate) fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); - pub(crate) fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); pub(crate) fn LLVMRustPositionBuilderPastAllocas<'a>(B: &Builder<'a>, Fn: &'a Value); @@ -2580,7 +2459,6 @@ unsafe extern "C" { pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, is_thin: bool, - emit_summary: bool, ) -> &'static mut ThinLTOBuffer; pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; @@ -2648,9 +2526,8 @@ unsafe extern "C" { pub(crate) fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; - pub(crate) fn LLVMRustLLVMHasZlibCompressionForDebugSymbols() -> bool; - - pub(crate) fn LLVMRustLLVMHasZstdCompressionForDebugSymbols() -> bool; + pub(crate) safe fn LLVMRustLLVMHasZlibCompression() -> bool; + pub(crate) safe fn LLVMRustLLVMHasZstdCompression() -> bool; pub(crate) fn LLVMRustGetSymbols( buf_ptr: *const u8, @@ -2664,6 +2541,8 @@ unsafe extern "C" { pub(crate) fn LLVMRustIsECObject(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustIsAnyArm64Coff(buf_ptr: *const u8, buf_len: usize) -> bool; + pub(crate) fn LLVMRustSetNoSanitizeAddress(Global: &Value); pub(crate) fn LLVMRustSetNoSanitizeHWAddress(Global: &Value); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs new file mode 100644 index 000000000000..a8a671b5c85f --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/metadata_kind.rs @@ -0,0 +1,71 @@ +use libc::c_uint; + +pub(crate) use self::fixed_kinds::*; + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub(crate) struct MetadataKindId(c_uint); + +macro_rules! declare_fixed_metadata_kinds { + ( + $( + FIXED_MD_KIND($variant:ident, $value:literal) + )* + ) => { + // Use a submodule to group all declarations into one `#[expect(..)]`. + #[expect(dead_code)] + mod fixed_kinds { + use super::MetadataKindId; + $( + #[expect(non_upper_case_globals)] + pub(crate) const $variant: MetadataKindId = MetadataKindId($value); + )* + } + }; +} + +// Must be kept in sync with the corresponding static assertions in `RustWrapper.cpp`. +declare_fixed_metadata_kinds! { + FIXED_MD_KIND(MD_dbg, 0) + FIXED_MD_KIND(MD_tbaa, 1) + FIXED_MD_KIND(MD_prof, 2) + FIXED_MD_KIND(MD_fpmath, 3) + FIXED_MD_KIND(MD_range, 4) + FIXED_MD_KIND(MD_tbaa_struct, 5) + FIXED_MD_KIND(MD_invariant_load, 6) + FIXED_MD_KIND(MD_alias_scope, 7) + FIXED_MD_KIND(MD_noalias, 8) + FIXED_MD_KIND(MD_nontemporal, 9) + FIXED_MD_KIND(MD_mem_parallel_loop_access, 10) + FIXED_MD_KIND(MD_nonnull, 11) + FIXED_MD_KIND(MD_dereferenceable, 12) + FIXED_MD_KIND(MD_dereferenceable_or_null, 13) + FIXED_MD_KIND(MD_make_implicit, 14) + FIXED_MD_KIND(MD_unpredictable, 15) + FIXED_MD_KIND(MD_invariant_group, 16) + FIXED_MD_KIND(MD_align, 17) + FIXED_MD_KIND(MD_loop, 18) + FIXED_MD_KIND(MD_type, 19) + FIXED_MD_KIND(MD_section_prefix, 20) + FIXED_MD_KIND(MD_absolute_symbol, 21) + FIXED_MD_KIND(MD_associated, 22) + FIXED_MD_KIND(MD_callees, 23) + FIXED_MD_KIND(MD_irr_loop, 24) + FIXED_MD_KIND(MD_access_group, 25) + FIXED_MD_KIND(MD_callback, 26) + FIXED_MD_KIND(MD_preserve_access_index, 27) + FIXED_MD_KIND(MD_vcall_visibility, 28) + FIXED_MD_KIND(MD_noundef, 29) + FIXED_MD_KIND(MD_annotation, 30) + FIXED_MD_KIND(MD_nosanitize, 31) + FIXED_MD_KIND(MD_func_sanitize, 32) + FIXED_MD_KIND(MD_exclude, 33) + FIXED_MD_KIND(MD_memprof, 34) + FIXED_MD_KIND(MD_callsite, 35) + FIXED_MD_KIND(MD_kcfi_type, 36) + FIXED_MD_KIND(MD_pcsections, 37) + FIXED_MD_KIND(MD_DIAssignID, 38) + FIXED_MD_KIND(MD_coro_outside_frame, 39) + FIXED_MD_KIND(MD_mmra, 40) + FIXED_MD_KIND(MD_noalias_addrspace, 41) +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index 154ba4fd6901..4c58a92106d5 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -3,7 +3,6 @@ use std::ffi::{CStr, CString}; use std::num::NonZero; use std::ptr; -use std::str::FromStr; use std::string::FromUtf8Error; use libc::c_uint; @@ -12,14 +11,16 @@ use rustc_llvm::RustString; pub(crate) use self::CallConv::*; pub(crate) use self::CodeGenOptSize::*; -pub(crate) use self::MetadataType::*; +pub(crate) use self::conversions::*; pub(crate) use self::ffi::*; +pub(crate) use self::metadata_kind::*; use crate::common::AsCCharPtr; -pub(crate) mod archive_ro; +mod conversions; pub(crate) mod diagnostic; pub(crate) mod enzyme_ffi; mod ffi; +mod metadata_kind; pub(crate) use self::enzyme_ffi::*; @@ -42,32 +43,6 @@ pub(crate) fn AddFunctionAttributes<'ll>( } } -pub(crate) fn HasAttributeAtIndex<'ll>( - llfn: &'ll Value, - idx: AttributePlace, - kind: AttributeKind, -) -> bool { - unsafe { LLVMRustHasAttributeAtIndex(llfn, idx.as_uint(), kind) } -} - -pub(crate) fn HasStringAttribute<'ll>(llfn: &'ll Value, name: &str) -> bool { - unsafe { LLVMRustHasFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveStringAttrFromFn<'ll>(llfn: &'ll Value, name: &str) { - unsafe { LLVMRustRemoveFnAttribute(llfn, name.as_c_char_ptr(), name.len()) } -} - -pub(crate) fn RemoveRustEnumAttributeAtIndex( - llfn: &Value, - place: AttributePlace, - kind: AttributeKind, -) { - unsafe { - LLVMRustRemoveEnumAttributeAtIndex(llfn, place.as_uint(), kind); - } -} - pub(crate) fn AddCallSiteAttributes<'ll>( callsite: &'ll Value, idx: AttributePlace, @@ -140,16 +115,26 @@ pub(crate) fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> & pub(crate) fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute { let lower = range.start; + // LLVM treats the upper bound as exclusive, but allows wrapping. let upper = range.end.wrapping_add(1); - let lower_words = [lower as u64, (lower >> 64) as u64]; - let upper_words = [upper as u64, (upper >> 64) as u64]; + + // Pass each `u128` endpoint value as a `[u64; 2]` array, least-significant part first. + let as_u64_array = |x: u128| [x as u64, (x >> 64) as u64]; + let lower_words: [u64; 2] = as_u64_array(lower); + let upper_words: [u64; 2] = as_u64_array(upper); + + // To ensure that LLVM doesn't try to read beyond the `[u64; 2]` arrays, + // we must explicitly check that `size_bits` does not exceed 128. + let size_bits = size.bits(); + assert!(size_bits <= 128); + // More robust assertions that are redundant with `size_bits <= 128` and + // should be optimized away. + assert!(size_bits.div_ceil(64) <= u64::try_from(lower_words.len()).unwrap()); + assert!(size_bits.div_ceil(64) <= u64::try_from(upper_words.len()).unwrap()); + let size_bits = c_uint::try_from(size_bits).unwrap(); + unsafe { - LLVMRustCreateRangeAttribute( - llcx, - size.bits().try_into().unwrap(), - lower_words.as_ptr(), - upper_words.as_ptr(), - ) + LLVMRustCreateRangeAttribute(llcx, size_bits, lower_words.as_ptr(), upper_words.as_ptr()) } } @@ -178,21 +163,6 @@ pub(crate) enum CodeGenOptSize { CodeGenOptSizeAggressive = 2, } -impl FromStr for ArchiveKind { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "gnu" => Ok(ArchiveKind::K_GNU), - "bsd" => Ok(ArchiveKind::K_BSD), - "darwin" => Ok(ArchiveKind::K_DARWIN), - "coff" => Ok(ArchiveKind::K_COFF), - "aix_big" => Ok(ArchiveKind::K_AIXBIG), - _ => Err(()), - } - } -} - pub(crate) fn SetInstructionCallConv(instr: &Value, cc: CallConv) { unsafe { LLVMSetInstructionCallConv(instr, cc as c_uint); @@ -258,7 +228,7 @@ pub(crate) fn set_initializer(llglobal: &Value, constant_val: &Value) { } pub(crate) fn set_global_constant(llglobal: &Value, is_constant: bool) { - LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + LLVMSetGlobalConstant(llglobal, is_constant.to_llvm_bool()); } pub(crate) fn get_linkage(llglobal: &Value) -> Linkage { @@ -272,7 +242,7 @@ pub(crate) fn set_linkage(llglobal: &Value, linkage: Linkage) { } pub(crate) fn is_declaration(llglobal: &Value) -> bool { - unsafe { LLVMIsDeclaration(llglobal) == ffi::True } + unsafe { LLVMIsDeclaration(llglobal) }.is_true() } pub(crate) fn get_visibility(llglobal: &Value) -> Visibility { @@ -291,6 +261,10 @@ pub(crate) fn set_alignment(llglobal: &Value, align: Align) { } } +pub(crate) fn set_externally_initialized(llglobal: &Value, is_ext_init: bool) { + LLVMSetExternallyInitialized(llglobal, is_ext_init.to_llvm_bool()); +} + /// Get the `name`d comdat from `llmod` and assign it to `llglobal`. /// /// Inserts the comdat into `llmod` if it does not exist. diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 53899da183a3..3b920168e06d 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -26,7 +26,7 @@ static INIT: Once = Once::new(); pub(crate) fn init(sess: &Session) { unsafe { // Before we touch LLVM, make sure that multithreading is enabled. - if llvm::LLVMIsMultithreaded() != 1 { + if !llvm::LLVMIsMultithreaded().is_true() { bug!("LLVM compiled without support for threads"); } INIT.call_once(|| { @@ -106,7 +106,7 @@ unsafe fn configure_llvm(sess: &Session) { if sess.target.os == "emscripten" && !sess.opts.unstable_opts.emscripten_wasm_eh - && sess.panic_strategy() == PanicStrategy::Unwind + && sess.panic_strategy().unwinds() { add("-enable-emscripten-cxx-exceptions", false); } @@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> { /// Rust can also be build with an external precompiled version of LLVM which might lead to failures /// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics. pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option> { - let arch = if sess.target.arch == "x86_64" { - "x86" - } else if sess.target.arch == "arm64ec" { - "aarch64" - } else if sess.target.arch == "sparc64" { - "sparc" - } else if sess.target.arch == "powerpc64" { - "powerpc" - } else { - &*sess.target.arch + let raw_arch = &*sess.target.arch; + let arch = match raw_arch { + "x86_64" => "x86", + "arm64ec" => "aarch64", + "sparc64" => "sparc", + "powerpc64" => "powerpc", + _ => raw_arch, }; + let (major, _, _) = get_version(); match (arch, s) { - ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( - "sse4.2", - smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], - )), - ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), - ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), - ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), - ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), - ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")), ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")), ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")), @@ -246,9 +235,6 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option Some(LLVMFeature::new("perfmon")), ("aarch64", "paca") => Some(LLVMFeature::new("pauth")), ("aarch64", "pacg") => Some(LLVMFeature::new("pauth")), - // Before LLVM 20 those two features were packaged together as b16b16 - ("aarch64", "sve-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), - ("aarch64", "sme-b16b16") if get_version().0 < 20 => Some(LLVMFeature::new("b16b16")), ("aarch64", "flagm2") => Some(LLVMFeature::new("altnzcv")), // Rust ties fp and neon together. ("aarch64", "neon") => Some(LLVMFeature::with_dependencies( @@ -263,46 +249,25 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option None, // only existed in 18 ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")), // Filter out features that are not supported by the current LLVM version - ("loongarch64", "div32" | "lam-bh" | "lamcas" | "ld-seq-sa" | "scq") - if get_version().0 < 20 => - { - None - } - // Filter out features that are not supported by the current LLVM version - ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None, - ( - "s390x", - "message-security-assist-extension12" - | "concurrent-functions" - | "miscellaneous-extensions-4" - | "vector-enhancements-3" - | "vector-packed-decimal-enhancement-3", - ) if get_version().0 < 20 => None, + ("loongarch32" | "loongarch64", "32s") if major < 21 => None, + ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), + ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), + ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies( + "sse4.2", + smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")], + )), + ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")), + ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")), + ("x86", "bmi1") => Some(LLVMFeature::new("bmi")), + ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")), + ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")), // Enable the evex512 target feature if an avx512 target feature is enabled. ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies( s, smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")], )), - // Support for `wide-arithmetic` will first land in LLVM 20 as part of - // llvm/llvm-project#111598 - ("wasm32" | "wasm64", "wide-arithmetic") if get_version() < (20, 0, 0) => None, - ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")), - // In LLVM 19, there is no `v8plus` feature and `v9` means "SPARC-V9 instruction available and SPARC-V8+ ABI used". - // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L27-L28 - // Before LLVM 19, there was no `v8plus` feature and `v9` means "SPARC-V9 instruction available". - // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.0/llvm/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp#L26 - ("sparc", "v8plus") if get_version().0 == 19 => Some(LLVMFeature::new("v9")), - ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")), - // These new `amx` variants and `movrs` were introduced in LLVM20 - ("x86", "amx-avx512" | "amx-fp8" | "amx-movrs" | "amx-tf32" | "amx-transpose") - if get_version().0 < 20 => - { - None - } - ("x86", "movrs") if get_version().0 < 20 => None, ("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")), - ("x86", "avx10.2") if get_version().0 < 20 => None, - ("x86", "avx10.2") if get_version().0 >= 20 => Some(LLVMFeature::new("avx10.2-512")), + ("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")), ("x86", "apxf") => Some(LLVMFeature::with_dependencies( "egpr", smallvec![ @@ -324,15 +289,12 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option TargetConfig { - // Add base features for the target. - // We do *not* add the -Ctarget-features there, and instead duplicate the logic for that below. - // The reason is that if LLVM considers a feature implied but we do not, we don't want that to - // show up in `cfg`. That way, `cfg` is entirely under our control -- except for the handling of - // the target CPU, that is still expanded to target features (with all their implied features) - // by LLVM. let target_machine = create_informational_target_machine(sess, true); let (unstable_target_features, target_features) = cfg_target_feature(sess, |feature| { + // This closure determines whether the target CPU has the feature according to LLVM. We do + // *not* consider the `-Ctarget-feature`s here, as that will be handled later in + // `cfg_target_feature`. if let Some(feat) = to_llvm_features(sess, feature) { // All the LLVM features this expands to must be enabled. for llvm_feature in feat { @@ -371,24 +333,25 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { let target_abi = sess.target.options.abi.as_ref(); let target_pointer_width = sess.target.pointer_width; let version = get_version(); + let lt_20_1_1 = version < (20, 1, 1); + let lt_21_0_0 = version < (21, 0, 0); cfg.has_reliable_f16 = match (target_arch, target_os) { - // Selection failure - ("s390x", _) => false, - // LLVM crash without neon (now fixed) + // LLVM crash without neon (fixed in llvm20) ("aarch64", _) - if !cfg.target_features.iter().any(|f| f.as_str() == "neon") - && version < (20, 1, 1) => + if !cfg.target_features.iter().any(|f| f.as_str() == "neon") && lt_20_1_1 => { false } // Unsupported ("arm64ec", _) => false, + // Selection failure (fixed in llvm21) + ("s390x", _) if lt_21_0_0 => false, // MinGW ABI bugs ("x86_64", "windows") if target_env == "gnu" && target_abi != "llvm" => false, // Infinite recursion ("csky", _) => false, - ("hexagon", _) => false, + ("hexagon", _) if lt_21_0_0 => false, // (fixed in llvm21) ("powerpc" | "powerpc64", _) => false, ("sparc" | "sparc64", _) => false, ("wasm32" | "wasm64", _) => false, @@ -401,9 +364,10 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { cfg.has_reliable_f128 = match (target_arch, target_os) { // Unsupported ("arm64ec", _) => false, - // Selection bug - ("mips64" | "mips64r6", _) => false, - // Selection bug + // Selection bug (fixed in llvm20) + ("mips64" | "mips64r6", _) if lt_20_1_1 => false, + // Selection bug . This issue is closed + // but basic math still does not work. ("nvptx64", _) => false, // Unsupported https://github.com/llvm/llvm-project/issues/121122 ("amdgpu", _) => false, @@ -413,8 +377,8 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { // ABI unsupported ("sparc", _) => false, // Stack alignment bug . NB: tests may - // not fail if our compiler-builtins is linked. - ("x86", _) => false, + // not fail if our compiler-builtins is linked. (fixed in llvm21) + ("x86", _) if lt_21_0_0 => false, // MinGW ABI bugs ("x86_64", "windows") if target_env == "gnu" && target_abi != "llvm" => false, // There are no known problems on other platforms, so the only requirement is that symbols @@ -707,17 +671,7 @@ pub(crate) fn global_llvm_features( }; // Features implied by an implicit or explicit `--target`. - features.extend( - sess.target - .features - .split(',') - .filter(|v| !v.is_empty()) - // Drop +v8plus feature introduced in LLVM 20. - // (Hard-coded target features do not go through `to_llvm_feature` since they already - // are LLVM feature names, hence we need a special case here.) - .filter(|v| *v != "+v8plus" || get_version() >= (20, 0, 0)) - .map(String::from), - ); + features.extend(sess.target.features.split(',').filter(|v| !v.is_empty()).map(String::from)); if wants_wasm_eh(sess) && sess.panic_strategy() == PanicStrategy::Unwind { features.push("+exception-handling".into()); diff --git a/compiler/rustc_codegen_llvm/src/macros.rs b/compiler/rustc_codegen_llvm/src/macros.rs new file mode 100644 index 000000000000..fddc428ca273 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/macros.rs @@ -0,0 +1,22 @@ +macro_rules! TryFromU32 { + derive() ( + $(#[$meta:meta])* + $vis:vis enum $Type:ident { + $( + $(#[$varmeta:meta])* + $Variant:ident $(= $discr:expr)? + ),* $(,)? + } + ) => { + impl ::core::convert::TryFrom for $Type { + type Error = u32; + #[allow(deprecated)] // Don't warn about deprecated variants. + fn try_from(value: u32) -> ::core::result::Result<$Type, Self::Error> { + $( if value == const { $Type::$Variant as u32 } { return Ok($Type::$Variant) } )* + Err(value) + } + } + } +} + +pub(crate) use TryFromU32; diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index f9edaded60de..52eefe2d4d24 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -1,8 +1,9 @@ use rustc_codegen_ssa::traits::*; +use rustc_hir::attrs::Linkage; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; use rustc_session::config::CrateType; @@ -132,7 +133,7 @@ impl CodegenCx<'_, '_> { // Thread-local variables generally don't support copy relocations. let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) - .is_some_and(|v| llvm::LLVMIsThreadLocal(v) == llvm::True); + .is_some_and(|v| llvm::LLVMIsThreadLocal(v).is_true()); if is_thread_local_var { return false; } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 893655031388..81bb70c95879 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -13,12 +13,10 @@ use rustc_middle::ty::{self, Ty}; use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; +use crate::common; use crate::context::{CodegenCx, GenericCx, SCx}; -pub(crate) use crate::llvm::Type; -use crate::llvm::{Bool, False, Metadata, True}; +use crate::llvm::{self, FALSE, Metadata, TRUE, ToLlvmBool, Type, Value}; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; -use crate::{common, llvm}; impl PartialEq for Type { fn eq(&self, other: &Self) -> bool { @@ -53,7 +51,9 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { - unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + unsafe { + llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed.to_llvm_bool()) + } } pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } @@ -61,7 +61,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { ///x Creates an integer type with the given number of bits, e.g., i24 pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { - unsafe { llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint) } + llvm::LLVMIntTypeInContext(self.llcx(), num_bits as c_uint) } pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { @@ -139,7 +139,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { } pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, TRUE) } } pub(crate) fn type_i1(&self) -> &'ll Type { @@ -152,7 +152,7 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { self.llcx(), els.as_ptr(), els.len() as c_uint, - packed as Bool, + packed.to_llvm_bool(), ) } } @@ -176,7 +176,7 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_i128(&self) -> &'ll Type { - unsafe { llvm::LLVMIntTypeInContext(self.llcx(), 128) } + self.type_ix(128) } fn type_isize(&self) -> &'ll Type { @@ -200,19 +200,19 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { - unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, FALSE) } } fn type_kind(&self, ty: &'ll Type) -> TypeKind { - unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + llvm::LLVMGetTypeKind(ty).to_rust().to_generic() } fn type_ptr(&self) -> &'ll Type { - self.type_ptr_ext(AddressSpace::ZERO) + llvm_type_ptr(self.llcx()) } fn type_ptr_ext(&self, address_space: AddressSpace) -> &'ll Type { - unsafe { llvm::LLVMPointerTypeInContext(self.llcx(), address_space.0) } + llvm_type_ptr_in_address_space(self.llcx(), address_space) } fn element_type(&self, ty: &'ll Type) -> &'ll Type { @@ -251,15 +251,15 @@ impl<'ll, CX: Borrow>> BaseTypeCodegenMethods for GenericCx<'ll, CX> { } } -impl Type { - /// Creates an integer type with the given number of bits, e.g., i24 - pub(crate) fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { - unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } - } +pub(crate) fn llvm_type_ptr(llcx: &llvm::Context) -> &Type { + llvm_type_ptr_in_address_space(llcx, AddressSpace::ZERO) +} - pub(crate) fn ptr_llcx(llcx: &llvm::Context) -> &Type { - unsafe { llvm::LLVMPointerTypeInContext(llcx, AddressSpace::ZERO.0) } - } +pub(crate) fn llvm_type_ptr_in_address_space<'ll>( + llcx: &'ll llvm::Context, + addr_space: AddressSpace, +) -> &'ll Type { + llvm::LLVMPointerTypeInContext(llcx, addr_space.0) } impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { @@ -300,26 +300,14 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn add_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { let typeid_metadata = self.create_metadata(typeid); - unsafe { - let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; - llvm::LLVMRustGlobalAddMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), - ) - } + let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; + self.global_add_metadata_node(function, llvm::MD_type, &v); } fn set_type_metadata(&self, function: &'ll Value, typeid: &[u8]) { let typeid_metadata = self.create_metadata(typeid); - unsafe { - let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_type as c_uint, - llvm::LLVMMDNodeInContext2(self.llcx, v.as_ptr(), v.len()), - ) - } + let v = [llvm::LLVMValueAsMetadata(self.const_usize(0)), typeid_metadata]; + self.global_set_metadata_node(function, llvm::MD_type, &v); } fn typeid_metadata(&self, typeid: &[u8]) -> Option<&'ll Metadata> { @@ -327,32 +315,12 @@ impl<'ll, 'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { } fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { - let kcfi_type_metadata = self.const_u32(kcfi_typeid); - unsafe { - llvm::LLVMRustGlobalAddMetadata( - function, - llvm::MD_kcfi_type as c_uint, - llvm::LLVMMDNodeInContext2( - self.llcx, - &llvm::LLVMValueAsMetadata(kcfi_type_metadata), - 1, - ), - ) - } + let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))]; + self.global_add_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata); } fn set_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) { - let kcfi_type_metadata = self.const_u32(kcfi_typeid); - unsafe { - llvm::LLVMGlobalSetMetadata( - function, - llvm::MD_kcfi_type as c_uint, - llvm::LLVMMDNodeInContext2( - self.llcx, - &llvm::LLVMValueAsMetadata(kcfi_type_metadata), - 1, - ), - ) - } + let kcfi_type_metadata = [llvm::LLVMValueAsMetadata(self.const_u32(kcfi_typeid))]; + self.global_set_metadata_node(function, llvm::MD_kcfi_type, &kcfi_type_metadata); } } diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 4e7096da502d..2b2ac1c6eb29 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -7,10 +7,11 @@ use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; +use rustc_span::{DUMMY_SP, Span}; use tracing::debug; use crate::common::*; -use crate::type_::Type; +use crate::llvm::Type; fn uncached_llvm_type<'a, 'tcx>( cx: &CodegenCx<'a, 'tcx>, @@ -149,7 +150,11 @@ impl<'a, 'tcx> CodegenCx<'a, 'tcx> { } pub(crate) fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { - let layout = self.layout_of(ty); + self.spanned_size_and_align_of(ty, DUMMY_SP) + } + + pub(crate) fn spanned_size_and_align_of(&self, ty: Ty<'tcx>, span: Span) -> (Size, Align) { + let layout = self.spanned_layout_of(ty, span); (layout.size, layout.align.abi) } } @@ -226,7 +231,7 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { // Make sure lifetimes are erased, to avoid generating distinct LLVM // types for Rust types that only differ in the choice of lifetimes. - let normal_ty = cx.tcx.erase_regions(self.ty); + let normal_ty = cx.tcx.erase_and_anonymize_regions(self.ty); let mut defer = None; let llty = if self.ty != normal_ty { diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs new file mode 100644 index 000000000000..7e2635037008 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -0,0 +1,122 @@ +use rustc_ast::expand::typetree::FncTree; +#[cfg(feature = "llvm_enzyme")] +use { + crate::attributes, + rustc_ast::expand::typetree::TypeTree as RustTypeTree, + std::ffi::{CString, c_char, c_uint}, +}; + +use crate::llvm::{self, Value}; + +#[cfg(feature = "llvm_enzyme")] +fn to_enzyme_typetree( + rust_typetree: RustTypeTree, + _data_layout: &str, + llcx: &llvm::Context, +) -> llvm::TypeTree { + let mut enzyme_tt = llvm::TypeTree::new(); + process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); + enzyme_tt +} +#[cfg(feature = "llvm_enzyme")] +fn process_typetree_recursive( + enzyme_tt: &mut llvm::TypeTree, + rust_typetree: &RustTypeTree, + parent_indices: &[i64], + llcx: &llvm::Context, +) { + for rust_type in &rust_typetree.0 { + let concrete_type = match rust_type.kind { + rustc_ast::expand::typetree::Kind::Anything => llvm::CConcreteType::DT_Anything, + rustc_ast::expand::typetree::Kind::Integer => llvm::CConcreteType::DT_Integer, + rustc_ast::expand::typetree::Kind::Pointer => llvm::CConcreteType::DT_Pointer, + rustc_ast::expand::typetree::Kind::Half => llvm::CConcreteType::DT_Half, + rustc_ast::expand::typetree::Kind::Float => llvm::CConcreteType::DT_Float, + rustc_ast::expand::typetree::Kind::Double => llvm::CConcreteType::DT_Double, + rustc_ast::expand::typetree::Kind::F128 => llvm::CConcreteType::DT_FP128, + rustc_ast::expand::typetree::Kind::Unknown => llvm::CConcreteType::DT_Unknown, + }; + + let mut indices = parent_indices.to_vec(); + if !parent_indices.is_empty() { + indices.push(rust_type.offset as i64); + } else if rust_type.offset == -1 { + indices.push(-1); + } else { + indices.push(rust_type.offset as i64); + } + + enzyme_tt.insert(&indices, concrete_type, llcx); + + if rust_type.kind == rustc_ast::expand::typetree::Kind::Pointer + && !rust_type.child.0.is_empty() + { + process_typetree_recursive(enzyme_tt, &rust_type.child, &indices, llcx); + } + } +} + +#[cfg(feature = "llvm_enzyme")] +pub(crate) fn add_tt<'ll>( + llmod: &'ll llvm::Module, + llcx: &'ll llvm::Context, + fn_def: &'ll Value, + tt: FncTree, +) { + let inputs = tt.args; + let ret_tt: RustTypeTree = tt.ret; + + let llvm_data_layout: *const c_char = unsafe { llvm::LLVMGetDataLayoutStr(&*llmod) }; + let llvm_data_layout = + std::str::from_utf8(unsafe { std::ffi::CStr::from_ptr(llvm_data_layout) }.to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + let attr_name = "enzyme_type"; + let c_attr_name = CString::new(attr_name).unwrap(); + + for (i, input) in inputs.iter().enumerate() { + unsafe { + let enzyme_tt = to_enzyme_typetree(input.clone(), llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::Argument(i as u32), &[attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } + } + + unsafe { + let enzyme_tt = to_enzyme_typetree(ret_tt, llvm_data_layout, llcx); + let c_str = llvm::EnzymeTypeTreeToString(enzyme_tt.inner); + let c_str = std::ffi::CStr::from_ptr(c_str); + + let ret_attr = llvm::LLVMCreateStringAttribute( + llcx, + c_attr_name.as_ptr(), + c_attr_name.as_bytes().len() as c_uint, + c_str.as_ptr(), + c_str.to_bytes().len() as c_uint, + ); + + attributes::apply_to_llfn(fn_def, llvm::AttributePlace::ReturnValue, &[ret_attr]); + llvm::EnzymeTypeTreeToStringFree(c_str.as_ptr()); + } +} + +#[cfg(not(feature = "llvm_enzyme"))] +pub(crate) fn add_tt<'ll>( + _llmod: &'ll llvm::Module, + _llcx: &'ll llvm::Context, + _fn_def: &'ll Value, + _tt: FncTree, +) { + unimplemented!() +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index ce079f3cb0af..2d9abb412d4f 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -9,9 +9,8 @@ use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use crate::builder::Builder; -use crate::type_::Type; +use crate::llvm::{Type, Value}; use crate::type_of::LayoutLlvmExt; -use crate::value::Value; fn round_up_to_alignment<'ll>( bx: &mut Builder<'_, 'll, '_>, @@ -28,9 +27,12 @@ fn round_pointer_up_to_alignment<'ll>( align: Align, ptr_ty: &'ll Type, ) -> &'ll Value { - let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); - ptr_as_int = round_up_to_alignment(bx, ptr_as_int, align); - bx.inttoptr(ptr_as_int, ptr_ty) + let ptr = bx.inbounds_ptradd(addr, bx.const_i32(align.bytes() as i32 - 1)); + bx.call_intrinsic( + "llvm.ptrmask", + &[ptr_ty, bx.type_i32()], + &[ptr, bx.const_int(bx.isize_ty, -(align.bytes() as isize) as i64)], + ) } fn emit_direct_ptr_va_arg<'ll, 'tcx>( @@ -190,7 +192,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>( // the offset again. bx.switch_to_block(maybe_reg); - if gr_type && layout.align.abi.bytes() > 8 { + if gr_type && layout.align.bytes() > 8 { reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); } @@ -288,7 +290,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr }; - let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi); + let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align); // "Align" the register count when the type is passed as `i64`. if is_i64 || (is_f64 && is_soft_float_abi) { @@ -326,7 +328,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( // Increase the used-register count. let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 }; let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr)); - bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi); + bx.store(new_num_regs, num_regs_addr, dl.i8_align); bx.br(end); @@ -336,7 +338,7 @@ fn emit_powerpc_va_arg<'ll, 'tcx>( let mem_addr = { bx.switch_to_block(in_mem); - bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi); + bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align); // Everything in the overflow area is rounded up to a size of at least 4. let overflow_area_align = Align::from_bytes(4).unwrap(); @@ -735,6 +737,7 @@ fn copy_to_temporary_if_more_aligned<'ll, 'tcx>( src_align, bx.const_u32(layout.layout.size().bytes() as u32), MemFlags::empty(), + None, ); tmp } else { @@ -757,7 +760,7 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( // byte boundary if alignment needed by type exceeds 8 byte boundary. // It isn't stated explicitly in the standard, but in practice we use // alignment greater than 16 where necessary. - if layout.layout.align.abi.bytes() > 8 { + if layout.layout.align.bytes() > 8 { unreachable!("all instances of VaArgSafe have an alignment <= 8"); } @@ -810,7 +813,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>( let va_ndx_offset = va_reg_offset + 4; let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset)); - let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi); + let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align); let offset = round_up_to_alignment(bx, offset, layout.align.abi); let slot_size = layout.size.align_to(Align::from_bytes(4).unwrap()).bytes() as i32; @@ -905,6 +908,21 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( ) } "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + "arm" => { + // Types wider than 16 bytes are not currently supported. Clang has special logic for + // such types, but `VaArgSafe` is not implemented for any type that is this large. + assert!(bx.cx.size_of(target_ty).bytes() <= 16); + + emit_ptr_va_arg( + bx, + addr, + target_ty, + PassMode::Direct, + SlotSize::Bytes4, + AllowHigherAlign::Yes, + ForceRightAdjust::No, + ) + } "s390x" => emit_s390x_va_arg(bx, addr, target_ty), "powerpc" => emit_powerpc_va_arg(bx, addr, target_ty), "powerpc64" | "powerpc64le" => emit_ptr_va_arg( diff --git a/compiler/rustc_codegen_llvm/src/value.rs b/compiler/rustc_codegen_llvm/src/value.rs index 2eabac3be8c5..37f05f10c50c 100644 --- a/compiler/rustc_codegen_llvm/src/value.rs +++ b/compiler/rustc_codegen_llvm/src/value.rs @@ -1,8 +1,7 @@ use std::hash::{Hash, Hasher}; use std::{fmt, ptr}; -use crate::llvm; -pub(crate) use crate::llvm::Value; +use crate::llvm::{self, Value}; impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 94501da69a76..9c5a3d839ceb 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -5,19 +5,16 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -ar_archive_writer = "0.4.2" +ar_archive_writer = "0.5" bitflags = "2.4.1" bstr = "1.11.3" -# `cc` updates often break things, so we pin it here. Cargo enforces "max 1 semver-compat version -# per crate", so if you change this, you need to also change it in `rustc_llvm`. -cc = "=1.2.16" +find-msvc-tools = "0.1.2" itertools = "0.12" pathdiff = "0.2.0" regex = "1.4" rustc_abi = { path = "../rustc_abi" } rustc_arena = { path = "../rustc_arena" } rustc_ast = { path = "../rustc_ast" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_attr_parsing = { path = "../rustc_attr_parsing" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } @@ -27,6 +24,7 @@ rustc_hashes = { path = "../rustc_hashes" } rustc_hir = { path = "../rustc_hir" } rustc_incremental = { path = "../rustc_incremental" } rustc_index = { path = "../rustc_index" } +rustc_lint_defs = { path = "../rustc_lint_defs" } rustc_macros = { path = "../rustc_macros" } rustc_metadata = { path = "../rustc_metadata" } rustc_middle = { path = "../rustc_middle" } diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index a70d0011d161..e321b0773ec3 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -8,8 +8,6 @@ codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error} -codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto - codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument codegen_ssa_binary_output_to_tty = option `-o` or `--emit` is used to write binary output type `{$shorthand}` to stdout, but stdout is a tty @@ -31,7 +29,7 @@ codegen_ssa_cpu_required = target requires explicitly specifying a cpu with `-C codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error} codegen_ssa_dlltool_fail_import_library = - Dlltool could not create import library with {$dlltool_path} {$dlltool_args}: + dlltool could not create import library with {$dlltool_path} {$dlltool_args}: {$stdout} {$stderr} @@ -40,15 +38,15 @@ codegen_ssa_dynamic_linking_with_lto = .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO codegen_ssa_error_calling_dlltool = - Error calling dlltool '{$dlltool_path}': {$error} + error calling dlltool '{$dlltool_path}': {$error} codegen_ssa_error_creating_import_library = - Error creating import library for {$lib_name}: {$error} + error creating import library for {$lib_name}: {$error} codegen_ssa_error_creating_remark_dir = failed to create remark directory: {$error} codegen_ssa_error_writing_def_file = - Error writing .DEF file: {$error} + error writing .DEF file: {$error} codegen_ssa_expected_name_value_pair = expected name value pair @@ -99,6 +97,8 @@ codegen_ssa_invalid_literal_value = invalid literal value codegen_ssa_invalid_monomorphization_basic_float_type = invalid monomorphization of `{$name}` intrinsic: expected basic float type, found `{$ty}` +codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer or pointer type, found `{$ty}` + codegen_ssa_invalid_monomorphization_basic_integer_type = invalid monomorphization of `{$name}` intrinsic: expected basic integer type, found `{$ty}` codegen_ssa_invalid_monomorphization_cannot_return = invalid monomorphization of `{$name}` intrinsic: cannot return `{$ret_ty}`, expected `u{$expected_int_bits}` or `[u8; {$expected_bytes}]` @@ -169,15 +169,16 @@ codegen_ssa_invalid_monomorphization_unsupported_symbol = invalid monomorphizati codegen_ssa_invalid_monomorphization_unsupported_symbol_of_size = invalid monomorphization of `{$name}` intrinsic: unsupported {$symbol} from `{$in_ty}` with element `{$in_elem}` of size `{$size}` to `{$ret_ty}` -codegen_ssa_invalid_no_sanitize = invalid argument for `no_sanitize` - .note = expected one of: `address`, `cfi`, `hwaddress`, `kcfi`, `memory`, `memtag`, `shadow-call-stack`, or `thread` - codegen_ssa_invalid_windows_subsystem = invalid windows subsystem `{$subsystem}`, only `windows` and `console` are allowed codegen_ssa_ld64_unimplemented_modifier = `as-needed` modifier not implemented yet for ld64 codegen_ssa_lib_def_write_failure = failed to write lib.def file: {$error} +codegen_ssa_link_exe_status_stack_buffer_overrun = 0xc0000409 is `STATUS_STACK_BUFFER_OVERRUN` + .abort_note = this may have been caused by a program abort and not a stack buffer overrun + .event_log_note = consider checking the Application Event Log for Windows Error Reporting events to see the fail fast error code + codegen_ssa_link_exe_unexpected_error = `link.exe` returned an unexpected error codegen_ssa_link_script_unavailable = can only use link script when linking with GNU-like linker @@ -222,8 +223,6 @@ codegen_ssa_multiple_main_functions = entry symbol `main` declared multiple time codegen_ssa_no_field = no field `{$name}` -codegen_ssa_no_mangle_nameless = `#[no_mangle]` cannot be used on {$definition} as it has no name - codegen_ssa_no_module_named = no module named `{$user_path}` (mangled: {$cgu_name}). available modules: {$cgu_names} @@ -261,9 +260,9 @@ codegen_ssa_shuffle_indices_evaluation = could not evaluate shuffle_indices at c codegen_ssa_specify_libraries_to_link = use the `-l` flag to specify native libraries to link -codegen_ssa_static_library_native_artifacts = Link against the following native artifacts when linking against this static library. The order and any duplication can be significant on some platforms. +codegen_ssa_static_library_native_artifacts = link against the following native artifacts when linking against this static library. The order and any duplication can be significant on some platforms. -codegen_ssa_static_library_native_artifacts_to_file = Native artifacts to link against have been written to {$path}. The order and any duplication can be significant on some platforms. +codegen_ssa_static_library_native_artifacts_to_file = native artifacts to link against have been written to {$path}. The order and any duplication can be significant on some platforms. codegen_ssa_stripping_debug_info_failed = stripping debug info with `{$util}` failed: {$status} .note = {$output} @@ -361,13 +360,13 @@ codegen_ssa_unable_to_run = unable to run `{$util}`: {$error} codegen_ssa_unable_to_run_dsymutil = unable to run `dsymutil`: {$error} -codegen_ssa_unable_to_write_debugger_visualizer = Unable to write debugger visualizer file `{$path}`: {$error} +codegen_ssa_unable_to_write_debugger_visualizer = unable to write debugger visualizer file `{$path}`: {$error} codegen_ssa_unexpected_parameter_name = unexpected parameter name .label = expected `{$prefix_nops}` or `{$entry_nops}` codegen_ssa_unknown_archive_kind = - Don't know how to build archive of type: {$kind} + don't know how to build archive of type: {$kind} codegen_ssa_unknown_ctarget_feature = unknown and unstable feature specified for `-Ctarget-feature`: `{$feature}` @@ -395,6 +394,9 @@ codegen_ssa_version_script_write_failure = failed to write version script: {$err codegen_ssa_visual_studio_not_installed = you may need to install Visual Studio build tools with the "C++ build tools" workload +codegen_ssa_xcrun_about = + the SDK is needed by the linker to know where to find symbols in system libraries and for embedding the SDK version in the final object file + codegen_ssa_xcrun_command_line_tools_insufficient = when compiling for iOS, tvOS, visionOS or watchOS, you need a full installation of Xcode diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs index 3710625ac12d..43e1e135a666 100644 --- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs +++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs @@ -69,6 +69,15 @@ pub fn assert_module_sources(tcx: TyCtxt<'_>, set_reuse: &dyn Fn(&mut CguReuseTr set_reuse(&mut ams.cgu_reuse_tracker); + if tcx.sess.opts.unstable_opts.print_mono_items + && let Some(data) = &ams.cgu_reuse_tracker.data + { + data.actual_reuse.items().all(|(cgu, reuse)| { + println!("CGU_REUSE {cgu} {reuse}"); + true + }); + } + ams.cgu_reuse_tracker.check_expected_reuse(tcx.sess); }); } diff --git a/compiler/rustc_codegen_ssa/src/back/apple.rs b/compiler/rustc_codegen_ssa/src/back/apple.rs index d242efaf4fd4..b1d646d9265f 100644 --- a/compiler/rustc_codegen_ssa/src/back/apple.rs +++ b/compiler/rustc_codegen_ssa/src/back/apple.rs @@ -17,7 +17,7 @@ mod tests; /// The canonical name of the desired SDK for a given target. pub(super) fn sdk_name(target: &Target) -> &'static str { - match (&*target.os, &*target.abi) { + match (&*target.os, &*target.env) { ("macos", "") => "MacOSX", ("ios", "") => "iPhoneOS", ("ios", "sim") => "iPhoneSimulator", @@ -34,7 +34,7 @@ pub(super) fn sdk_name(target: &Target) -> &'static str { } pub(super) fn macho_platform(target: &Target) -> u32 { - match (&*target.os, &*target.abi) { + match (&*target.os, &*target.env) { ("macos", _) => object::macho::PLATFORM_MACOS, ("ios", "macabi") => object::macho::PLATFORM_MACCATALYST, ("ios", "sim") => object::macho::PLATFORM_IOSSIMULATOR, @@ -160,7 +160,11 @@ pub(super) fn add_version_to_llvm_target( pub(super) fn get_sdk_root(sess: &Session) -> Option { let sdk_name = sdk_name(&sess.target); - match xcrun_show_sdk_path(sdk_name, sess.verbose_internals()) { + // Attempt to invoke `xcrun` to find the SDK. + // + // Note that when cross-compiling from e.g. Linux, the `xcrun` binary may sometimes be provided + // as a shim by a cross-compilation helper tool. It usually isn't, but we still try nonetheless. + match xcrun_show_sdk_path(sdk_name, false) { Ok((path, stderr)) => { // Emit extra stderr, such as if `-verbose` was passed, or if `xcrun` emitted a warning. if !stderr.is_empty() { @@ -169,7 +173,19 @@ pub(super) fn get_sdk_root(sess: &Session) -> Option { Some(path) } Err(err) => { - let mut diag = sess.dcx().create_err(err); + // Failure to find the SDK is not a hard error, since the user might have specified it + // in a manner unknown to us (moreso if cross-compiling): + // - A compiler driver like `zig cc` which links using an internally bundled SDK. + // - Extra linker arguments (`-Clink-arg=-syslibroot`). + // - A custom linker or custom compiler driver. + // + // Though we still warn, since such cases are uncommon, and it is very hard to debug if + // you do not know the details. + // + // FIXME(madsmtm): Make this a lint, to allow deny warnings to work. + // (Or fix ). + let mut diag = sess.dcx().create_warn(err); + diag.note(fluent::codegen_ssa_xcrun_about); // Recognize common error cases, and give more Rust-specific error messages for those. if let Some(developer_dir) = xcode_select_developer_dir() { @@ -209,6 +225,8 @@ fn xcrun_show_sdk_path( sdk_name: &'static str, verbose: bool, ) -> Result<(PathBuf, String), XcrunError> { + // Intentionally invoke the `xcrun` in PATH, since e.g. nixpkgs provide an `xcrun` shim, so we + // don't want to require `/usr/bin/xcrun`. let mut cmd = Command::new("xcrun"); if verbose { cmd.arg("--verbose"); @@ -280,7 +298,7 @@ fn stdout_to_path(mut stdout: Vec) -> PathBuf { } #[cfg(unix)] let path = ::from_vec(stdout); - #[cfg(not(unix))] // Unimportant, this is only used on macOS - let path = OsString::from(String::from_utf8(stdout).unwrap()); + #[cfg(not(unix))] // Not so important, this is mostly used on macOS + let path = OsString::from(String::from_utf8(stdout).expect("stdout must be UTF-8")); PathBuf::from(path) } diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs index 84a56f6b0b55..cfd8ceac3a60 100644 --- a/compiler/rustc_codegen_ssa/src/back/archive.rs +++ b/compiler/rustc_codegen_ssa/src/back/archive.rs @@ -40,16 +40,18 @@ pub struct ImportLibraryItem { pub is_data: bool, } -impl From for COFFShortExport { - fn from(item: ImportLibraryItem) -> Self { +impl ImportLibraryItem { + fn into_coff_short_export(self, sess: &Session) -> COFFShortExport { + let import_name = (sess.target.arch == "arm64ec").then(|| self.name.clone()); COFFShortExport { - name: item.name, + name: self.name, ext_name: None, - symbol_name: item.symbol_name, - alias_target: None, - ordinal: item.ordinal.unwrap_or(0), - noname: item.ordinal.is_some(), - data: item.is_data, + symbol_name: self.symbol_name, + import_name, + export_as: None, + ordinal: self.ordinal.unwrap_or(0), + noname: self.ordinal.is_some(), + data: self.is_data, private: false, constant: false, } @@ -113,7 +115,8 @@ pub trait ArchiveBuilderBuilder { .emit_fatal(ErrorCreatingImportLibrary { lib_name, error: error.to_string() }), }; - let exports = items.into_iter().map(Into::into).collect::>(); + let exports = + items.into_iter().map(|item| item.into_coff_short_export(sess)).collect::>(); let machine = match &*sess.target.arch { "x86_64" => MachineTypes::AMD64, "x86" => MachineTypes::I386, @@ -134,6 +137,7 @@ pub trait ArchiveBuilderBuilder { // when linking a rust staticlib using `/WHOLEARCHIVE`. // See #129020 true, + &[], ) { sess.dcx() .emit_fatal(ErrorCreatingImportLibrary { lib_name, error: error.to_string() }); @@ -527,7 +531,7 @@ impl<'a> ArArchiveBuilder<'a> { &entries, archive_kind, false, - /* is_ec = */ self.sess.target.arch == "arm64ec", + /* is_ec = */ Some(self.sess.target.arch == "arm64ec"), )?; archive_tmpfile.flush()?; drop(archive_tmpfile); diff --git a/compiler/rustc_codegen_ssa/src/back/command.rs b/compiler/rustc_codegen_ssa/src/back/command.rs index 05351bd6ca37..7420f18aacb3 100644 --- a/compiler/rustc_codegen_ssa/src/back/command.rs +++ b/compiler/rustc_codegen_ssa/src/back/command.rs @@ -109,7 +109,7 @@ impl Command { } Program::Lld(ref p, flavor) => { let mut c = process::Command::new(p); - c.arg("-flavor").arg(flavor.as_str()); + c.arg("-flavor").arg(flavor.desc()); c } }; diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 162fbf3d6e24..ea538d3d4698 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -9,16 +9,18 @@ use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; use std::{env, fmt, fs, io, mem, str}; -use cc::windows_registry; +use find_msvc_tools; use itertools::Itertools; use regex::Regex; use rustc_arena::TypedArena; use rustc_ast::CRATE_NODE_ID; +use rustc_attr_parsing::{ShouldEmit, eval_config_entry}; use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{DiagCtxtHandle, LintDiagnostic}; use rustc_fs_util::{TempDirBuilder, fix_windows_verbatim_for_gcc, try_canonicalize}; +use rustc_hir::attrs::NativeLibKind; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_macros::LintDiagnostic; use rustc_metadata::fs::{METADATA_FILENAME, copy_to_stdout, emit_wrapper_file}; @@ -38,7 +40,6 @@ use rustc_session::config::{ use rustc_session::lint::builtin::LINKER_MESSAGES; use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename}; use rustc_session::search_paths::PathKind; -use rustc_session::utils::NativeLibKind; /// For all the linkers we support, and information they might /// need out of the shared crate context before we get rid of it. use rustc_session::{Session, filesearch}; @@ -46,8 +47,8 @@ use rustc_span::Symbol; use rustc_target::spec::crt_objects::CrtObjects; use rustc_target::spec::{ BinaryFormat, Cc, LinkOutputKind, LinkSelfContainedComponents, LinkSelfContainedDefault, - LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy, RelocModel, RelroLevel, - SanitizerSet, SplitDebuginfo, + LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, RelocModel, RelroLevel, SanitizerSet, + SplitDebuginfo, }; use tracing::{debug, info, warn}; @@ -57,6 +58,7 @@ use super::linker::{self, Linker}; use super::metadata::{MetadataPosition, create_wrapper_file}; use super::rpath::{self, RPathConfig}; use super::{apple, versioned_llvm_target}; +use crate::base::needs_allocator_shim_for_linking; use crate::{ CodegenResults, CompiledModule, CrateInfo, NativeLib, errors, looks_like_rust_object_file, }; @@ -77,6 +79,7 @@ pub fn link_binary( codegen_results: CodegenResults, metadata: EncodedMetadata, outputs: &OutputFilenames, + codegen_backend: &'static str, ) { let _timer = sess.timer("link_binary"); let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); @@ -152,6 +155,7 @@ pub fn link_binary( &codegen_results, &metadata, path.as_ref(), + codegen_backend, ); } } @@ -678,6 +682,7 @@ fn link_natively( codegen_results: &CodegenResults, metadata: &EncodedMetadata, tmpdir: &Path, + codegen_backend: &'static str, ) { info!("preparing {:?} to {:?}", crate_type, out_filename); let (linker_path, flavor) = linker_and_flavor(sess); @@ -703,6 +708,7 @@ fn link_natively( codegen_results, metadata, self_contained_components, + codegen_backend, ); linker::disable_localization(&mut cmd); @@ -875,11 +881,19 @@ fn link_natively( // All Microsoft `link.exe` linking ror codes are // four digit numbers in the range 1000 to 9999 inclusive if is_msvc_link_exe && (code < 1000 || code > 9999) { - let is_vs_installed = windows_registry::find_vs_version().is_ok(); + let is_vs_installed = find_msvc_tools::find_vs_version().is_ok(); let has_linker = - windows_registry::find_tool(&sess.target.arch, "link.exe").is_some(); + find_msvc_tools::find_tool(&sess.target.arch, "link.exe").is_some(); sess.dcx().emit_note(errors::LinkExeUnexpectedError); + + // STATUS_STACK_BUFFER_OVERRUN is also used for fast abnormal program termination, e.g. abort(). + // Emit a special diagnostic to let people know that this most likely doesn't indicate a stack buffer overrun. + const STATUS_STACK_BUFFER_OVERRUN: i32 = 0xc0000409u32 as _; + if code == STATUS_STACK_BUFFER_OVERRUN { + sess.dcx().emit_note(errors::LinkExeStatusStackBufferOverrun); + } + if is_vs_installed && has_linker { // the linker is broken sess.dcx().emit_note(errors::RepairVSBuildTools); @@ -1011,11 +1025,12 @@ fn link_natively( (Strip::Debuginfo, _) => { strip_with_external_utility(sess, stripcmd, out_filename, &["--strip-debug"]) } - // Per the manpage, `-x` is the maximum safe strip level for dynamic libraries. (#93988) + + // Per the manpage, --discard-all is the maximum safe strip level for dynamic libraries. (#93988) ( Strip::Symbols, CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro | CrateType::Sdylib, - ) => strip_with_external_utility(sess, stripcmd, out_filename, &["-x"]), + ) => strip_with_external_utility(sess, stripcmd, out_filename, &["--discard-all"]), (Strip::Symbols, _) => { strip_with_external_utility(sess, stripcmd, out_filename, &["--strip-all"]) } @@ -1475,7 +1490,7 @@ fn print_native_static_libs( NativeLibKind::Static { bundle: None | Some(true), .. } | NativeLibKind::LinkArg | NativeLibKind::WasmImportModule - | NativeLibKind::RawDylib => None, + | NativeLibKind::RawDylib { .. } => None, } }) // deduplication of consecutive repeated libraries, see rust-lang/rust#113209 @@ -2070,9 +2085,17 @@ fn add_local_crate_regular_objects(cmd: &mut dyn Linker, codegen_results: &Codeg } /// Add object files for allocator code linked once for the whole crate tree. -fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &CodegenResults) { - if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) { - cmd.add_object(obj); +fn add_local_crate_allocator_objects( + cmd: &mut dyn Linker, + codegen_results: &CodegenResults, + crate_type: CrateType, +) { + if needs_allocator_shim_for_linking(&codegen_results.crate_info.dependency_formats, crate_type) + { + if let Some(obj) = codegen_results.allocator_module.as_ref().and_then(|m| m.object.as_ref()) + { + cmd.add_object(obj); + } } } @@ -2189,6 +2212,7 @@ fn linker_with_args( codegen_results: &CodegenResults, metadata: &EncodedMetadata, self_contained_components: LinkSelfContainedComponents, + codegen_backend: &'static str, ) -> Command { let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled(); let cmd = &mut *super::linker::get_linker( @@ -2197,6 +2221,7 @@ fn linker_with_args( flavor, self_contained_components.are_any_components_enabled(), &codegen_results.crate_info.target_cpu, + codegen_backend, ); let link_output_kind = link_output_kind(sess, crate_type); @@ -2271,7 +2296,7 @@ fn linker_with_args( codegen_results, metadata, ); - add_local_crate_allocator_objects(cmd, codegen_results); + add_local_crate_allocator_objects(cmd, codegen_results, crate_type); // Avoid linking to dynamic libraries unless they satisfy some undefined symbols // at the point at which they are specified on the command line. @@ -2339,13 +2364,13 @@ fn linker_with_args( cmd.add_object(&output_path); } } else { - for link_path in raw_dylib::create_raw_dylib_elf_stub_shared_objects( + for (link_path, as_needed) in raw_dylib::create_raw_dylib_elf_stub_shared_objects( sess, codegen_results.crate_info.used_libraries.iter(), &raw_dylib_dir, ) { // Always use verbatim linkage, see comments in create_raw_dylib_elf_stub_shared_objects. - cmd.link_dylib_by_name(&link_path, true, false); + cmd.link_dylib_by_name(&link_path, true, as_needed); } } // As with add_upstream_native_libraries, we need to add the upstream raw-dylib symbols in case @@ -2386,13 +2411,13 @@ fn linker_with_args( cmd.add_object(&output_path); } } else { - for link_path in raw_dylib::create_raw_dylib_elf_stub_shared_objects( + for (link_path, as_needed) in raw_dylib::create_raw_dylib_elf_stub_shared_objects( sess, native_libraries_from_nonstatics, &raw_dylib_dir, ) { // Always use verbatim linkage, see comments in create_raw_dylib_elf_stub_shared_objects. - cmd.link_dylib_by_name(&link_path, true, false); + cmd.link_dylib_by_name(&link_path, true, as_needed); } } @@ -2426,6 +2451,13 @@ fn linker_with_args( // Passed after compiler-generated options to support manual overriding when necessary. add_user_defined_link_args(cmd, sess); + // ------------ Builtin configurable linker scripts ------------ + // The user's link args should be able to overwrite symbols in the compiler's + // linker script that were weakly defined (i.e. defined with `PROVIDE()`). For this + // to work correctly, the user needs to be able to specify linker arguments like + // `--defsym` and `--script` *before* any builtin linker scripts are evaluated. + add_link_script(cmd, sess, tmpdir, crate_type); + // ------------ Object code and libraries, order-dependent ------------ // Post-link CRT objects. @@ -2460,8 +2492,6 @@ fn add_order_independent_options( let apple_sdk_root = add_apple_sdk(cmd, sess, flavor); - add_link_script(cmd, sess, tmpdir, crate_type); - if sess.target.os == "fuchsia" && crate_type == CrateType::Executable && !matches!(flavor, LinkerFlavor::Gnu(Cc::Yes, _)) @@ -2488,10 +2518,10 @@ fn add_order_independent_options( if sess.target.os == "emscripten" { cmd.cc_arg(if sess.opts.unstable_opts.emscripten_wasm_eh { "-fwasm-exceptions" - } else if sess.panic_strategy() == PanicStrategy::Abort { - "-sDISABLE_EXCEPTION_CATCHING=1" - } else { + } else if sess.panic_strategy().unwinds() { "-sDISABLE_EXCEPTION_CATCHING=0" + } else { + "-sDISABLE_EXCEPTION_CATCHING=1" }); } @@ -2696,7 +2726,7 @@ fn add_native_libs_from_crate( cmd.link_framework_by_name(name, verbatim, as_needed.unwrap_or(true)) } } - NativeLibKind::RawDylib => { + NativeLibKind::RawDylib { as_needed: _ } => { // Handled separately in `linker_with_args`. } NativeLibKind::WasmImportModule => {} @@ -3005,7 +3035,9 @@ fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) { fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool { match lib.cfg { - Some(ref cfg) => rustc_attr_parsing::cfg_matches(cfg, sess, CRATE_NODE_ID, None), + Some(ref cfg) => { + eval_config_entry(sess, cfg, CRATE_NODE_ID, None, ShouldEmit::ErrorsAndLints).as_bool() + } None => true, } } @@ -3025,7 +3057,7 @@ pub(crate) fn are_upstream_rust_objects_already_included(sess: &Session) -> bool /// We need to communicate five things to the linker on Apple/Darwin targets: /// - The architecture. /// - The operating system (and that it's an Apple platform). -/// - The environment / ABI. +/// - The environment. /// - The deployment target. /// - The SDK version. fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) { @@ -3039,7 +3071,7 @@ fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavo // `sess.target.arch` (`target_arch`) is not detailed enough. let llvm_arch = sess.target.llvm_target.split_once('-').expect("LLVM target must have arch").0; let target_os = &*sess.target.os; - let target_abi = &*sess.target.abi; + let target_env = &*sess.target.env; // The architecture name to forward to the linker. // @@ -3090,14 +3122,14 @@ fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavo // > - visionos-simulator // > - xros-simulator // > - driverkit - let platform_name = match (target_os, target_abi) { + let platform_name = match (target_os, target_env) { (os, "") => os, ("ios", "macabi") => "mac-catalyst", ("ios", "sim") => "ios-simulator", ("tvos", "sim") => "tvos-simulator", ("watchos", "sim") => "watchos-simulator", ("visionos", "sim") => "visionos-simulator", - _ => bug!("invalid OS/ABI combination for Apple target: {target_os}, {target_abi}"), + _ => bug!("invalid OS/env combination for Apple target: {target_os}, {target_env}"), }; let min_version = sess.apple_deployment_target().fmt_full().to_string(); @@ -3185,39 +3217,60 @@ fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavo } fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) -> Option { - let os = &sess.target.os; - if sess.target.vendor != "apple" - || !matches!(os.as_ref(), "ios" | "tvos" | "watchos" | "visionos" | "macos") - || !matches!(flavor, LinkerFlavor::Darwin(..)) - { + if !sess.target.is_like_darwin { return None; } - - if os == "macos" && !matches!(flavor, LinkerFlavor::Darwin(Cc::No, _)) { + let LinkerFlavor::Darwin(cc, _) = flavor else { return None; + }; + + // The default compiler driver on macOS is at `/usr/bin/cc`. This is a trampoline binary that + // effectively invokes `xcrun cc` internally to look up both the compiler binary and the SDK + // root from the current Xcode installation. When cross-compiling, when `rustc` is invoked + // inside Xcode, or when invoking the linker directly, this default logic is unsuitable, so + // instead we invoke `xcrun` manually. + // + // (Note that this doesn't mean we get a duplicate lookup here - passing `SDKROOT` below will + // cause the trampoline binary to skip looking up the SDK itself). + let sdkroot = sess.time("get_apple_sdk_root", || get_apple_sdk_root(sess))?; + + if cc == Cc::Yes { + // There are a few options to pass the SDK root when linking with a C/C++ compiler: + // - The `--sysroot` flag. + // - The `-isysroot` flag. + // - The `SDKROOT` environment variable. + // + // `--sysroot` isn't actually enough to get Clang to treat it as a platform SDK, you need + // to specify `-isysroot`. This is admittedly a bit strange, as on most targets `-isysroot` + // only applies to include header files, but on Apple targets it also applies to libraries + // and frameworks. + // + // This leaves the choice between `-isysroot` and `SDKROOT`. Both are supported by Clang and + // GCC, though they may not be supported by all compiler drivers. We choose `SDKROOT`, + // primarily because that is the same interface that is used when invoking the tool under + // `xcrun -sdk macosx $tool`. + // + // In that sense, if a given compiler driver does not support `SDKROOT`, the blame is fairly + // clearly in the tool in question, since they also don't support being run under `xcrun`. + // + // Additionally, `SDKROOT` is an environment variable and thus optional. It also has lower + // precedence than `-isysroot`, so a custom compiler driver that does not support it and + // instead figures out the SDK on their own can easily do so by using `-isysroot`. + // + // (This in particular affects Clang built with the `DEFAULT_SYSROOT` CMake flag, such as + // the one provided by some versions of Homebrew's `llvm` package. Those will end up + // ignoring the value we set here, and instead use their built-in sysroot). + cmd.cmd().env("SDKROOT", &sdkroot); + } else { + // When invoking the linker directly, we use the `-syslibroot` parameter. `SDKROOT` is not + // read by the linker, so it's really the only option. + // + // This is also what Clang does. + cmd.link_arg("-syslibroot"); + cmd.link_arg(&sdkroot); } - let sdk_root = sess.time("get_apple_sdk_root", || get_apple_sdk_root(sess))?; - - match flavor { - LinkerFlavor::Darwin(Cc::Yes, _) => { - // Use `-isysroot` instead of `--sysroot`, as only the former - // makes Clang treat it as a platform SDK. - // - // This is admittedly a bit strange, as on most targets - // `-isysroot` only applies to include header files, but on Apple - // targets this also applies to libraries and frameworks. - cmd.cc_arg("-isysroot"); - cmd.cc_arg(&sdk_root); - } - LinkerFlavor::Darwin(Cc::No, _) => { - cmd.link_arg("-syslibroot"); - cmd.link_arg(&sdk_root); - } - _ => unreachable!(), - } - - Some(sdk_root) + Some(sdkroot) } fn get_apple_sdk_root(sess: &Session) -> Option { @@ -3246,7 +3299,13 @@ fn get_apple_sdk_root(sess: &Session) -> Option { } "macosx" if sdkroot.contains("iPhoneOS.platform") - || sdkroot.contains("iPhoneSimulator.platform") => {} + || sdkroot.contains("iPhoneSimulator.platform") + || sdkroot.contains("AppleTVOS.platform") + || sdkroot.contains("AppleTVSimulator.platform") + || sdkroot.contains("WatchOS.platform") + || sdkroot.contains("WatchSimulator.platform") + || sdkroot.contains("XROS.platform") + || sdkroot.contains("XRSimulator.platform") => {} "watchos" if sdkroot.contains("WatchSimulator.platform") || sdkroot.contains("MacOSX.platform") => {} diff --git a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs index b9e0c9573633..7321bc1da391 100644 --- a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs +++ b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs @@ -7,9 +7,9 @@ use rustc_data_structures::base_n::{CASE_INSENSITIVE, ToBaseN}; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_data_structures::stable_hasher::StableHasher; use rustc_hashes::Hash128; +use rustc_hir::attrs::NativeLibKind; use rustc_session::Session; use rustc_session::cstore::DllImport; -use rustc_session::utils::NativeLibKind; use rustc_span::Symbol; use crate::back::archive::ImportLibraryItem; @@ -31,7 +31,7 @@ fn collate_raw_dylibs_windows<'a>( let mut dylib_table = FxIndexMap::>::default(); for lib in used_libraries { - if lib.kind == NativeLibKind::RawDylib { + if let NativeLibKind::RawDylib { .. } = lib.kind { let ext = if lib.verbatim { "" } else { ".dll" }; let name = format!("{}{}", lib.name, ext); let imports = dylib_table.entry(name.clone()).or_default(); @@ -128,12 +128,12 @@ pub(super) fn create_raw_dylib_dll_import_libs<'a>( fn collate_raw_dylibs_elf<'a>( sess: &Session, used_libraries: impl IntoIterator, -) -> Vec<(String, Vec)> { +) -> Vec<(String, Vec, bool)> { // Use index maps to preserve original order of imports and libraries. - let mut dylib_table = FxIndexMap::>::default(); + let mut dylib_table = FxIndexMap::, bool)>::default(); for lib in used_libraries { - if lib.kind == NativeLibKind::RawDylib { + if let NativeLibKind::RawDylib { as_needed } = lib.kind { let filename = if lib.verbatim { lib.name.as_str().to_owned() } else { @@ -142,17 +142,19 @@ fn collate_raw_dylibs_elf<'a>( format!("{prefix}{}{ext}", lib.name) }; - let imports = dylib_table.entry(filename.clone()).or_default(); + let (stub_imports, stub_as_needed) = + dylib_table.entry(filename.clone()).or_insert((Default::default(), true)); for import in &lib.dll_imports { - imports.insert(import.name, import); + stub_imports.insert(import.name, import); } + *stub_as_needed = *stub_as_needed && as_needed.unwrap_or(true); } } sess.dcx().abort_if_errors(); dylib_table .into_iter() - .map(|(name, imports)| { - (name, imports.into_iter().map(|(_, import)| import.clone()).collect()) + .map(|(name, (imports, as_needed))| { + (name, imports.into_iter().map(|(_, import)| import.clone()).collect(), as_needed) }) .collect() } @@ -161,10 +163,10 @@ pub(super) fn create_raw_dylib_elf_stub_shared_objects<'a>( sess: &Session, used_libraries: impl IntoIterator, raw_dylib_so_dir: &Path, -) -> Vec { +) -> Vec<(String, bool)> { collate_raw_dylibs_elf(sess, used_libraries) .into_iter() - .map(|(load_filename, raw_dylib_imports)| { + .map(|(load_filename, raw_dylib_imports, as_needed)| { use std::hash::Hash; // `load_filename` is the *target/loader* filename that will end up in NEEDED. @@ -205,7 +207,7 @@ pub(super) fn create_raw_dylib_elf_stub_shared_objects<'a>( }); }; - temporary_lib_name + (temporary_lib_name, as_needed) }) .collect() } @@ -307,11 +309,14 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport] stub.reserve_section_headers(); stub.reserve_dynsym(); stub.reserve_dynstr(); + let verdef_count = 1 + vers.len(); + let mut dynamic_entries = 2; // DT_SONAME, DT_NULL if !vers.is_empty() { stub.reserve_gnu_versym(); - stub.reserve_gnu_verdef(1 + vers.len(), 1 + vers.len()); + stub.reserve_gnu_verdef(verdef_count, verdef_count); + dynamic_entries += 1; // DT_VERDEFNUM } - stub.reserve_dynamic(2); // DT_SONAME, DT_NULL + stub.reserve_dynamic(dynamic_entries); // First write the ELF header with the arch information. let e_machine = match (arch, sub_arch) { @@ -443,9 +448,13 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport] // .dynamic // the DT_SONAME will be used by the linker to populate DT_NEEDED // which the loader uses to find the library. - // DT_NULL terminates the .dynamic table. stub.write_align_dynamic(); stub.write_dynamic_string(elf::DT_SONAME, soname); + // LSB section "2.7. Symbol Versioning" requires `DT_VERDEFNUM` to be reliable. + if verdef_count > 1 { + stub.write_dynamic(elf::DT_VERDEFNUM, verdef_count as u64); + } + // DT_NULL terminates the .dynamic table. stub.write_dynamic(elf::DT_NULL, 0); stub_buf diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs index 050797354b4a..ac1231437382 100644 --- a/compiler/rustc_codegen_ssa/src/back/linker.rs +++ b/compiler/rustc_codegen_ssa/src/back/linker.rs @@ -4,24 +4,26 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::{env, io, iter, mem, str}; -use cc::windows_registry; +use find_msvc_tools; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_metadata::{ find_native_static_library, try_find_native_dynamic_library, try_find_native_static_library, }; use rustc_middle::bug; use rustc_middle::middle::dependency_format::Linkage; -use rustc_middle::middle::exported_symbols; -use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind}; +use rustc_middle::middle::exported_symbols::{ + self, ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, +}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; use rustc_session::config::{self, CrateType, DebugInfo, LinkerPluginLto, Lto, OptLevel, Strip}; -use rustc_span::sym; use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, Lld}; use tracing::{debug, warn}; use super::command::Command; use super::symbol_export; +use crate::back::symbol_export::allocator_shim_symbols; +use crate::base::needs_allocator_shim_for_linking; use crate::errors; #[cfg(test)] @@ -49,8 +51,9 @@ pub(crate) fn get_linker<'a>( flavor: LinkerFlavor, self_contained: bool, target_cpu: &'a str, + codegen_backend: &'static str, ) -> Box { - let msvc_tool = windows_registry::find_tool(&sess.target.arch, "link.exe"); + let msvc_tool = find_msvc_tools::find_tool(&sess.target.arch, "link.exe"); // If our linker looks like a batch script on Windows then to execute this // we'll need to spawn `cmd` explicitly. This is primarily done to handle @@ -114,7 +117,6 @@ pub(crate) fn get_linker<'a>( if sess.target.is_like_msvc && let Some(ref tool) = msvc_tool { - cmd.args(tool.args()); for (k, v) in tool.env() { if k == "PATH" { new_path.extend(env::split_paths(v)); @@ -152,6 +154,7 @@ pub(crate) fn get_linker<'a>( is_ld: cc == Cc::No, is_gnu: flavor.is_gnu(), uses_lld: flavor.uses_lld(), + codegen_backend, }) as Box, LinkerFlavor::Msvc(..) => Box::new(MsvcLinker { cmd, sess }) as Box, LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box, @@ -365,6 +368,7 @@ struct GccLinker<'a> { is_ld: bool, is_gnu: bool, uses_lld: bool, + codegen_backend: &'static str, } impl<'a> GccLinker<'a> { @@ -421,9 +425,15 @@ impl<'a> GccLinker<'a> { if let Some(path) = &self.sess.opts.unstable_opts.profile_sample_use { self.link_arg(&format!("-plugin-opt=sample-profile={}", path.display())); }; + let prefix = if self.codegen_backend == "gcc" { + // The GCC linker plugin requires a leading dash. + "-" + } else { + "" + }; self.link_args(&[ - &format!("-plugin-opt={opt_level}"), - &format!("-plugin-opt=mcpu={}", self.target_cpu), + &format!("-plugin-opt={prefix}{opt_level}"), + &format!("-plugin-opt={prefix}mcpu={}", self.target_cpu), ]); } @@ -842,6 +852,11 @@ impl<'a> Linker for GccLinker<'a> { self.sess.dcx().emit_fatal(errors::VersionScriptWriteFailure { error }); } self.link_arg("--dynamic-list").link_arg(path); + } else if self.sess.target.is_like_wasm { + self.link_arg("--no-export-dynamic"); + for (sym, _) in symbols { + self.link_arg("--export").link_arg(sym); + } } else { // Write an LD version script let res: io::Result<()> = try { @@ -1308,37 +1323,7 @@ struct WasmLd<'a> { impl<'a> WasmLd<'a> { fn new(cmd: Command, sess: &'a Session) -> WasmLd<'a> { - // If the atomics feature is enabled for wasm then we need a whole bunch - // of flags: - // - // * `--shared-memory` - the link won't even succeed without this, flags - // the one linear memory as `shared` - // - // * `--max-memory=1G` - when specifying a shared memory this must also - // be specified. We conservatively choose 1GB but users should be able - // to override this with `-C link-arg`. - // - // * `--import-memory` - it doesn't make much sense for memory to be - // exported in a threaded module because typically you're - // sharing memory and instantiating the module multiple times. As a - // result if it were exported then we'd just have no sharing. - // - // On wasm32-unknown-unknown, we also export symbols for glue code to use: - // * `--export=*tls*` - when `#[thread_local]` symbols are used these - // symbols are how the TLS segments are initialized and configured. - let mut wasm_ld = WasmLd { cmd, sess }; - if sess.target_features.contains(&sym::atomics) { - wasm_ld.link_args(&["--shared-memory", "--max-memory=1073741824", "--import-memory"]); - if sess.target.os == "unknown" || sess.target.os == "none" { - wasm_ld.link_args(&[ - "--export=__wasm_init_tls", - "--export=__tls_size", - "--export=__tls_align", - "--export=__tls_base", - ]); - } - } - wasm_ld + WasmLd { cmd, sess } } } @@ -1805,11 +1790,18 @@ pub(crate) fn exported_symbols( .collect(); } - if let CrateType::ProcMacro = crate_type { + let mut symbols = if let CrateType::ProcMacro = crate_type { exported_symbols_for_proc_macro_crate(tcx) } else { exported_symbols_for_non_proc_macro(tcx, crate_type) + }; + + if crate_type == CrateType::Dylib || crate_type == CrateType::ProcMacro { + let metadata_symbol_name = exported_symbols::metadata_symbol_name(tcx); + symbols.push((metadata_symbol_name, SymbolExportKind::Data)); } + + symbols } fn exported_symbols_for_non_proc_macro( @@ -1820,7 +1812,7 @@ fn exported_symbols_for_non_proc_macro( let export_threshold = symbol_export::crates_export_threshold(&[crate_type]); for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| { // Do not export mangled symbols from cdylibs and don't attempt to export compiler-builtins - // from any cdylib. The latter doesn't work anyway as we use hidden visibility for + // from any dylib. The latter doesn't work anyway as we use hidden visibility for // compiler-builtins. Most linkers silently ignore it, but ld64 gives a warning. if info.level.is_below_threshold(export_threshold) && !tcx.is_compiler_builtins(cnum) { symbols.push(( @@ -1831,6 +1823,14 @@ fn exported_symbols_for_non_proc_macro( } }); + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust + && needs_allocator_shim_for_linking(tcx.dependency_formats(()), crate_type) + && tcx.allocator_kind(()).is_some() + { + symbols.extend(allocator_shim_symbols(tcx)); + } + symbols } @@ -1842,12 +1842,8 @@ fn exported_symbols_for_proc_macro_crate(tcx: TyCtxt<'_>) -> Vec<(String, Symbol let stable_crate_id = tcx.stable_crate_id(LOCAL_CRATE); let proc_macro_decls_name = tcx.sess.generate_proc_macro_decls_symbol(stable_crate_id); - let metadata_symbol_name = exported_symbols::metadata_symbol_name(tcx); - vec![ - (proc_macro_decls_name, SymbolExportKind::Data), - (metadata_symbol_name, SymbolExportKind::Data), - ] + vec![(proc_macro_decls_name, SymbolExportKind::Data)] } pub(crate) fn linked_symbols( diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs index c95038375a1b..e6df6a2469f3 100644 --- a/compiler/rustc_codegen_ssa/src/back/lto.rs +++ b/compiler/rustc_codegen_ssa/src/back/lto.rs @@ -8,8 +8,9 @@ use rustc_middle::ty::TyCtxt; use rustc_session::config::{CrateType, Lto}; use tracing::info; -use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate}; +use crate::back::symbol_export::{self, allocator_shim_symbols, symbol_name_for_instance_in_crate}; use crate::back::write::CodegenContext; +use crate::base::allocator_kind_for_codegen; use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro}; use crate::traits::*; @@ -115,6 +116,11 @@ pub(super) fn exported_symbols_for_lto( } } + // Mark allocator shim symbols as exported only if they were generated. + if export_threshold == SymbolExportLevel::Rust && allocator_kind_for_codegen(tcx).is_some() { + symbols_below_threshold.extend(allocator_shim_symbols(tcx).map(|(name, _kind)| name)); + } + symbols_below_threshold } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index bf38c02e908e..10aaadd5688a 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -329,12 +329,18 @@ pub(super) fn elf_e_flags(architecture: Architecture, sess: &Session) -> u32 { // Source: https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/079772828bd10933d34121117a222b4cc0ee2200/riscv-elf.adoc let mut e_flags: u32 = 0x0; - // Check if compressed is enabled - // `unstable_target_features` is used here because "c" is gated behind riscv_target_feature. - if sess.unstable_target_features.contains(&sym::c) { + // Check if compression is enabled + // `unstable_target_features` is used here because "zca" is gated behind riscv_target_feature. + if sess.unstable_target_features.contains(&sym::zca) { e_flags |= elf::EF_RISCV_RVC; } + // Check if RVTSO is enabled + // `unstable_target_features` is used here because "ztso" is gated behind riscv_target_feature. + if sess.unstable_target_features.contains(&sym::ztso) { + e_flags |= elf::EF_RISCV_TSO; + } + // Set the appropriate flag based on ABI // This needs to match LLVM `RISCVELFStreamer.cpp` match &*sess.target.llvm_abiname { diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs index 297bdec2bc21..5bc18e2d7f8b 100644 --- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs +++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs @@ -1,24 +1,26 @@ use std::collections::hash_map::Entry::*; use rustc_abi::{CanonAbi, X86Call}; -use rustc_ast::expand::allocator::{ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name}; +use rustc_ast::expand::allocator::{ + ALLOC_ERROR_HANDLER, ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name, +}; use rustc_data_structures::unord::UnordMap; use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE, LocalDefId}; use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::middle::exported_symbols::{ - ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, metadata_symbol_name, + ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel, }; use rustc_middle::query::LocalCrate; use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, SymbolName, Ty, TyCtxt}; use rustc_middle::util::Providers; use rustc_session::config::{CrateType, OomStrategy}; use rustc_symbol_mangling::mangle_internal_symbol; -use rustc_target::spec::{SanitizerSet, TlsModel}; +use rustc_target::spec::TlsModel; use tracing::debug; -use crate::base::allocator_kind_for_codegen; +use crate::back::symbol_export; fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel { crates_export_threshold(tcx.crate_types()) @@ -217,95 +219,6 @@ fn exported_non_generic_symbols_provider_local<'tcx>( )); } - // Mark allocator shim symbols as exported only if they were generated. - if allocator_kind_for_codegen(tcx).is_some() { - for symbol_name in ALLOCATOR_METHODS - .iter() - .map(|method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) - .chain([ - mangle_internal_symbol(tcx, "__rust_alloc_error_handler"), - mangle_internal_symbol(tcx, OomStrategy::SYMBOL), - mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), - ]) - { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::Rust, - kind: SymbolExportKind::Text, - used: false, - rustc_std_internal_symbol: true, - }, - )); - } - } - - if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() { - // These are weak symbols that point to the profile version and the - // profile name, which need to be treated as exported so LTO doesn't nix - // them. - const PROFILER_WEAK_SYMBOLS: [&str; 2] = - ["__llvm_profile_raw_version", "__llvm_profile_filename"]; - - symbols.extend(PROFILER_WEAK_SYMBOLS.iter().map(|sym| { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - ( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - rustc_std_internal_symbol: false, - }, - ) - })); - } - - if tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) { - let mut msan_weak_symbols = Vec::new(); - - // Similar to profiling, preserve weak msan symbol during LTO. - if tcx.sess.opts.unstable_opts.sanitizer_recover.contains(SanitizerSet::MEMORY) { - msan_weak_symbols.push("__msan_keep_going"); - } - - if tcx.sess.opts.unstable_opts.sanitizer_memory_track_origins != 0 { - msan_weak_symbols.push("__msan_track_origins"); - } - - symbols.extend(msan_weak_symbols.into_iter().map(|sym| { - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym)); - ( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: false, - rustc_std_internal_symbol: false, - }, - ) - })); - } - - if tcx.crate_types().contains(&CrateType::Dylib) - || tcx.crate_types().contains(&CrateType::ProcMacro) - { - let symbol_name = metadata_symbol_name(tcx); - let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); - - symbols.push(( - exported_symbol, - SymbolExportInfo { - level: SymbolExportLevel::C, - kind: SymbolExportKind::Data, - used: true, - rustc_std_internal_symbol: false, - }, - )); - } - // Sort so we get a stable incr. comp. hash. symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx)); @@ -323,7 +236,8 @@ fn exported_generic_symbols_provider_local<'tcx>( let mut symbols: Vec<_> = vec![]; if tcx.local_crate_exports_generics() { - use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility}; + use rustc_hir::attrs::Linkage; + use rustc_middle::mir::mono::{MonoItem, Visibility}; use rustc_middle::ty::InstanceKind; // Normally, we require that shared monomorphizations are not hidden, @@ -384,7 +298,7 @@ fn exported_generic_symbols_provider_local<'tcx>( if !tcx.sess.opts.share_generics() { if tcx.codegen_fn_attrs(mono_item.def_id()).inline - == rustc_attr_data_structures::InlineAttr::Never + == rustc_hir::attrs::InlineAttr::Never { // this is OK, we explicitly allow sharing inline(never) across crates even // without share-generics. @@ -579,6 +493,31 @@ pub(crate) fn provide(providers: &mut Providers) { upstream_monomorphizations_for_provider; } +pub(crate) fn allocator_shim_symbols( + tcx: TyCtxt<'_>, +) -> impl Iterator { + ALLOCATOR_METHODS + .iter() + .map(move |method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str())) + .chain([ + mangle_internal_symbol(tcx, global_fn_name(ALLOC_ERROR_HANDLER).as_str()), + mangle_internal_symbol(tcx, OomStrategy::SYMBOL), + mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE), + ]) + .map(move |symbol_name| { + let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name)); + + ( + symbol_export::exporting_symbol_name_for_instance_in_crate( + tcx, + exported_symbol, + LOCAL_CRATE, + ), + SymbolExportKind::Text, + ) + }) +} + fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel { // We export anything that's not mangled at the "C" layer as it probably has // to do with ABI concerns. We do not, however, apply such treatment to diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index 6773d3e24e94..368a2e307bb2 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -1,5 +1,6 @@ use std::assert_matches::assert_matches; use std::marker::PhantomData; +use std::panic::AssertUnwindSafe; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::mpsc::{Receiver, Sender, channel}; @@ -7,7 +8,6 @@ use std::{fs, io, mem, str, thread}; use rustc_abi::Size; use rustc_ast::attr; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::jobserver::{self, Acquired}; use rustc_data_structures::memmap::Mmap; @@ -15,8 +15,8 @@ use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard}; use rustc_errors::emitter::Emitter; use rustc_errors::translation::Translator; use rustc_errors::{ - Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, Level, MultiSpan, Style, - Suggestions, + Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FatalErrorMarker, Level, + MultiSpan, Style, Suggestions, }; use rustc_fs_util::link_or_copy; use rustc_incremental::{ @@ -38,7 +38,7 @@ use tracing::debug; use super::link::{self, ensure_removed}; use super::lto::{self, SerializedModule}; use crate::back::lto::check_lto_allowed; -use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir}; +use crate::errors::ErrorCreatingRemarkDir; use crate::traits::*; use crate::{ CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind, @@ -76,12 +76,9 @@ pub struct ModuleConfig { /// Names of additional optimization passes to run. pub passes: Vec, /// Some(level) to optimize at a certain level, or None to run - /// absolutely no optimizations (used for the metadata module). + /// absolutely no optimizations (used for the allocator module). pub opt_level: Option, - /// Some(level) to optimize binary size, or None to not affect program size. - pub opt_size: Option, - pub pgo_gen: SwitchWithOptPath, pub pgo_use: Option, pub pgo_sample_use: Option, @@ -102,7 +99,6 @@ pub struct ModuleConfig { pub emit_obj: EmitObj, pub emit_thin_lto: bool, pub emit_thin_lto_summary: bool, - pub bc_cmdline: String, // Miscellaneous flags. These are mostly copied from command-line // options. @@ -110,7 +106,6 @@ pub struct ModuleConfig { pub lint_llvm_ir: bool, pub no_prepopulate_passes: bool, pub no_builtins: bool, - pub time_module: bool, pub vectorize_loop: bool, pub vectorize_slp: bool, pub merge_functions: bool, @@ -171,7 +166,6 @@ impl ModuleConfig { passes: if_regular!(sess.opts.cg.passes.clone(), vec![]), opt_level: opt_level_and_size, - opt_size: opt_level_and_size, pgo_gen: if_regular!( sess.opts.cg.profile_generate.clone(), @@ -214,22 +208,19 @@ impl ModuleConfig { false ), emit_obj, - emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto, + // thin lto summaries prevent fat lto, so do not emit them if fat + // lto is requested. See PR #136840 for background information. + emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto && sess.lto() != Lto::Fat, emit_thin_lto_summary: if_regular!( sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode), false ), - bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(), verify_llvm_ir: sess.verify_llvm_ir(), lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir, no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes, no_builtins: no_builtins || sess.target.no_builtins, - // Exclude metadata and allocator modules from time_passes output, - // since they throw off the "LLVM passes" measurement. - time_module: if_regular!(true, false), - // Copy what clang does by turning on loop vectorization at O2 and // slp vectorization at O3. vectorize_loop: !sess.opts.cg.no_vectorize_loops @@ -342,8 +333,8 @@ pub struct CodegenContext { pub crate_types: Vec, pub output_filenames: Arc, pub invocation_temp: Option, - pub regular_module_config: Arc, - pub allocator_module_config: Arc, + pub module_config: Arc, + pub allocator_config: Arc, pub tm_factory: TargetMachineFactoryFn, pub msvc_imps_needed: bool, pub is_pe_coff: bool, @@ -355,12 +346,6 @@ pub struct CodegenContext { pub split_dwarf_kind: rustc_session::config::SplitDwarfKind, pub pointer_size: Size, - /// All commandline args used to invoke the compiler, with @file args fully expanded. - /// This will only be used within debug info, e.g. in the pdb file on windows - /// This is mainly useful for other tools that reads that debuginfo to figure out - /// how to call the compiler with the same arguments. - pub expanded_args: Vec, - /// Emitter to use for diagnostics produced during codegen. pub diag_emitter: SharedEmitter, /// LLVM optimizations for which we want to print remarks. @@ -381,13 +366,6 @@ impl CodegenContext { pub fn create_dcx(&self) -> DiagCtxt { DiagCtxt::new(Box::new(self.diag_emitter.clone())) } - - pub fn config(&self, kind: ModuleKind) -> &ModuleConfig { - match kind { - ModuleKind::Regular => &self.regular_module_config, - ModuleKind::Allocator => &self.allocator_module_config, - } - } } fn generate_thin_lto_work( @@ -396,7 +374,7 @@ fn generate_thin_lto_work( each_linked_rlib_for_lto: &[PathBuf], needs_thin_lto: Vec<(String, B::ThinBuffer)>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, -) -> Vec<(WorkItem, u64)> { +) -> Vec<(ThinLtoWorkItem, u64)> { let _prof_timer = cgcx.prof.generic_activity("codegen_thin_generate_lto_work"); let (lto_modules, copy_jobs) = B::run_thin_lto( @@ -405,17 +383,16 @@ fn generate_thin_lto_work( each_linked_rlib_for_lto, needs_thin_lto, import_only_modules, - ) - .unwrap_or_else(|e| e.raise()); + ); lto_modules .into_iter() .map(|module| { let cost = module.cost(); - (WorkItem::ThinLto(module), cost) + (ThinLtoWorkItem::ThinLto(module), cost) }) .chain(copy_jobs.into_iter().map(|wp| { ( - WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { + ThinLtoWorkItem::CopyPostLtoArtifacts(CachedModuleCodegen { name: wp.cgu_name.clone(), source: wp, }), @@ -452,7 +429,7 @@ pub(crate) fn start_async_codegen( backend: B, tcx: TyCtxt<'_>, target_cpu: String, - autodiff_items: &[AutoDiffItem], + allocator_module: Option>, ) -> OngoingCodegen { let (coordinator_send, coordinator_receive) = channel(); @@ -471,12 +448,12 @@ pub(crate) fn start_async_codegen( backend.clone(), tcx, &crate_info, - autodiff_items, shared_emitter, codegen_worker_send, coordinator_receive, Arc::new(regular_config), Arc::new(allocator_config), + allocator_module, coordinator_send.clone(), ); @@ -720,74 +697,73 @@ pub(crate) enum WorkItem { /// Copy the post-LTO artifacts from the incremental cache to the output /// directory. CopyPostLtoArtifacts(CachedModuleCodegen), - /// Performs fat LTO on the given module. - FatLto { - exported_symbols_for_lto: Arc>, - each_linked_rlib_for_lto: Vec, - needs_fat_lto: Vec>, - import_only_modules: Vec<(SerializedModule, WorkProduct)>, - autodiff: Vec, - }, +} + +enum ThinLtoWorkItem { + /// Copy the post-LTO artifacts from the incremental cache to the output + /// directory. + CopyPostLtoArtifacts(CachedModuleCodegen), /// Performs thin-LTO on the given module. ThinLto(lto::ThinModule), } -impl WorkItem { - fn module_kind(&self) -> ModuleKind { - match *self { - WorkItem::Optimize(ref m) => m.kind, - WorkItem::CopyPostLtoArtifacts(_) | WorkItem::FatLto { .. } | WorkItem::ThinLto(_) => { - ModuleKind::Regular - } - } - } +// `pthread_setname()` on *nix ignores anything beyond the first 15 +// bytes. Use short descriptions to maximize the space available for +// the module name. +#[cfg(not(windows))] +fn desc(short: &str, _long: &str, name: &str) -> String { + // The short label is three bytes, and is followed by a space. That + // leaves 11 bytes for the CGU name. How we obtain those 11 bytes + // depends on the CGU name form. + // + // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part + // before the `-cgu.0` is the same for every CGU, so use the + // `cgu.0` part. The number suffix will be different for each + // CGU. + // + // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole + // name because each CGU will have a unique ASCII hash, and the + // first 11 bytes will be enough to identify it. + // + // - Incremental (with `-Zhuman-readable-cgu-names`), e.g. + // `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole + // name. The first 11 bytes won't be enough to uniquely identify + // it, but no obvious substring will, and this is a rarely used + // option so it doesn't matter much. + // + assert_eq!(short.len(), 3); + let name = if let Some(index) = name.find("-cgu.") { + &name[index + 1..] // +1 skips the leading '-'. + } else { + name + }; + format!("{short} {name}") +} +// Windows has no thread name length limit, so use more descriptive names. +#[cfg(windows)] +fn desc(_short: &str, long: &str, name: &str) -> String { + format!("{long} {name}") +} + +impl WorkItem { /// Generate a short description of this work item suitable for use as a thread name. fn short_description(&self) -> String { - // `pthread_setname()` on *nix ignores anything beyond the first 15 - // bytes. Use short descriptions to maximize the space available for - // the module name. - #[cfg(not(windows))] - fn desc(short: &str, _long: &str, name: &str) -> String { - // The short label is three bytes, and is followed by a space. That - // leaves 11 bytes for the CGU name. How we obtain those 11 bytes - // depends on the CGU name form. - // - // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part - // before the `-cgu.0` is the same for every CGU, so use the - // `cgu.0` part. The number suffix will be different for each - // CGU. - // - // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole - // name because each CGU will have a unique ASCII hash, and the - // first 11 bytes will be enough to identify it. - // - // - Incremental (with `-Zhuman-readable-cgu-names`), e.g. - // `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole - // name. The first 11 bytes won't be enough to uniquely identify - // it, but no obvious substring will, and this is a rarely used - // option so it doesn't matter much. - // - assert_eq!(short.len(), 3); - let name = if let Some(index) = name.find("-cgu.") { - &name[index + 1..] // +1 skips the leading '-'. - } else { - name - }; - format!("{short} {name}") - } - - // Windows has no thread name length limit, so use more descriptive names. - #[cfg(windows)] - fn desc(_short: &str, long: &str, name: &str) -> String { - format!("{long} {name}") - } - match self { WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name), WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name), - WorkItem::FatLto { .. } => desc("lto", "fat LTO module", "everything"), - WorkItem::ThinLto(m) => desc("lto", "thin-LTO module", m.name()), + } + } +} + +impl ThinLtoWorkItem { + /// Generate a short description of this work item suitable for use as a thread name. + fn short_description(&self) -> String { + match self { + ThinLtoWorkItem::CopyPostLtoArtifacts(m) => { + desc("cpy", "copy LTO artifacts for", &m.name) + } + ThinLtoWorkItem::ThinLto(m) => desc("lto", "thin-LTO module", m.name()), } } } @@ -830,7 +806,7 @@ pub(crate) fn compute_per_cgu_lto_type( let linker_does_lto = opts.cg.linker_plugin_lto.enabled(); // When we're automatically doing ThinLTO for multi-codegen-unit - // builds we don't actually want to LTO the allocator modules if + // builds we don't actually want to LTO the allocator module if // it shows up. This is due to various linker shenanigans that // we'll encounter later. let is_allocator = module_kind == ModuleKind::Allocator; @@ -856,12 +832,18 @@ pub(crate) fn compute_per_cgu_lto_type( fn execute_optimize_work_item( cgcx: &CodegenContext, mut module: ModuleCodegen, - module_config: &ModuleConfig, -) -> Result, FatalError> { +) -> WorkItemResult { + let _timer = cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*module.name); + let dcx = cgcx.create_dcx(); let dcx = dcx.handle(); - B::optimize(cgcx, dcx, &mut module, module_config)?; + let module_config = match module.kind { + ModuleKind::Regular => &cgcx.module_config, + ModuleKind::Allocator => &cgcx.allocator_config, + }; + + B::optimize(cgcx, dcx, &mut module, module_config); // After we've done the initial round of optimizations we need to // decide whether to synchronously codegen this module or ship it @@ -872,7 +854,7 @@ fn execute_optimize_work_item( // If we're doing some form of incremental LTO then we need to be sure to // save our module to disk first. - let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc { + let bitcode = if module_config.emit_pre_lto_bc { let filename = pre_lto_bitcode_filename(&module.name); cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename)) } else { @@ -881,17 +863,17 @@ fn execute_optimize_work_item( match lto_type { ComputedLtoType::No => { - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) + let module = B::codegen(cgcx, module, module_config); + WorkItemResult::Finished(module) } ComputedLtoType::Thin => { - let (name, thin_buffer) = B::prepare_thin(module, false); + let (name, thin_buffer) = B::prepare_thin(module); if let Some(path) = bitcode { fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); } - Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) + WorkItemResult::NeedsThinLto(name, thin_buffer) } ComputedLtoType::Fat => match bitcode { Some(path) => { @@ -899,12 +881,12 @@ fn execute_optimize_work_item( fs::write(&path, buffer.data()).unwrap_or_else(|e| { panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e); }); - Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { + WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer: SerializedModule::Local(buffer), - })) + }) } - None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))), + None => WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module)), }, } } @@ -912,8 +894,11 @@ fn execute_optimize_work_item( fn execute_copy_from_cache_work_item( cgcx: &CodegenContext, module: CachedModuleCodegen, - module_config: &ModuleConfig, -) -> WorkItemResult { +) -> CompiledModule { + let _timer = cgcx + .prof + .generic_activity_with_arg("codegen_copy_artifacts_from_incr_cache", &*module.name); + let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap(); let mut links_from_incr_cache = Vec::new(); @@ -972,6 +957,7 @@ fn execute_copy_from_cache_work_item( } }; + let module_config = &cgcx.module_config; let should_emit_obj = module_config.emit_obj != EmitObj::None; let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly); let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly); @@ -981,27 +967,29 @@ fn execute_copy_from_cache_work_item( cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name }) } - WorkItemResult::Finished(CompiledModule { + CompiledModule { links_from_incr_cache, - name: module.name, kind: ModuleKind::Regular, + name: module.name, object, dwarf_object, bytecode, assembly, llvm_ir, - }) + } } -fn execute_fat_lto_work_item( +fn do_fat_lto( cgcx: &CodegenContext, exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], mut needs_fat_lto: Vec>, import_only_modules: Vec<(SerializedModule, WorkProduct)>, - autodiff: Vec, - module_config: &ModuleConfig, -) -> Result, FatalError> { +) -> CompiledModule { + let _timer = cgcx.prof.verbose_generic_activity("LLVM_fatlto"); + + check_lto_allowed(&cgcx); + for (module, wp) in import_only_modules { needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module }) } @@ -1011,20 +999,156 @@ fn execute_fat_lto_work_item( exported_symbols_for_lto, each_linked_rlib_for_lto, needs_fat_lto, - autodiff, - )?; - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) + ); + B::codegen(cgcx, module, &cgcx.module_config) +} + +fn do_thin_lto<'a, B: ExtraBackendMethods>( + cgcx: &'a CodegenContext, + exported_symbols_for_lto: Arc>, + each_linked_rlib_for_lto: Vec, + needs_thin_lto: Vec<(String, ::ThinBuffer)>, + lto_import_only_modules: Vec<( + SerializedModule<::ModuleBuffer>, + WorkProduct, + )>, +) -> Vec { + let _timer = cgcx.prof.verbose_generic_activity("LLVM_thinlto"); + + check_lto_allowed(&cgcx); + + let (coordinator_send, coordinator_receive) = channel(); + + // First up, convert our jobserver into a helper thread so we can use normal + // mpsc channels to manage our messages and such. + // After we've requested tokens then we'll, when we can, + // get tokens on `coordinator_receive` which will + // get managed in the main loop below. + let coordinator_send2 = coordinator_send.clone(); + let helper = jobserver::client() + .into_helper_thread(move |token| { + drop(coordinator_send2.send(ThinLtoMessage::Token(token))); + }) + .expect("failed to spawn helper thread"); + + let mut work_items = vec![]; + + // We have LTO work to do. Perform the serial work here of + // figuring out what we're going to LTO and then push a + // bunch of work items onto our queue to do LTO. This all + // happens on the coordinator thread but it's very quick so + // we don't worry about tokens. + for (work, cost) in generate_thin_lto_work( + cgcx, + &exported_symbols_for_lto, + &each_linked_rlib_for_lto, + needs_thin_lto, + lto_import_only_modules, + ) { + let insertion_index = + work_items.binary_search_by_key(&cost, |&(_, cost)| cost).unwrap_or_else(|e| e); + work_items.insert(insertion_index, (work, cost)); + if cgcx.parallel { + helper.request_token(); + } + } + + let mut codegen_aborted = None; + + // These are the Jobserver Tokens we currently hold. Does not include + // the implicit Token the compiler process owns no matter what. + let mut tokens = vec![]; + + // Amount of tokens that are used (including the implicit token). + let mut used_token_count = 0; + + let mut compiled_modules = vec![]; + + // Run the message loop while there's still anything that needs message + // processing. Note that as soon as codegen is aborted we simply want to + // wait for all existing work to finish, so many of the conditions here + // only apply if codegen hasn't been aborted as they represent pending + // work to be done. + loop { + if codegen_aborted.is_none() { + if used_token_count == 0 && work_items.is_empty() { + // All codegen work is done. + break; + } + + // Spin up what work we can, only doing this while we've got available + // parallelism slots and work left to spawn. + while used_token_count < tokens.len() + 1 + && let Some((item, _)) = work_items.pop() + { + spawn_thin_lto_work(&cgcx, coordinator_send.clone(), item); + used_token_count += 1; + } + } else { + // Don't queue up any more work if codegen was aborted, we're + // just waiting for our existing children to finish. + if used_token_count == 0 { + break; + } + } + + // Relinquish accidentally acquired extra tokens. Subtract 1 for the implicit token. + tokens.truncate(used_token_count.saturating_sub(1)); + + match coordinator_receive.recv().unwrap() { + // Save the token locally and the next turn of the loop will use + // this to spawn a new unit of work, or it may get dropped + // immediately if we have no more work to spawn. + ThinLtoMessage::Token(token) => match token { + Ok(token) => { + tokens.push(token); + } + Err(e) => { + let msg = &format!("failed to acquire jobserver token: {e}"); + cgcx.diag_emitter.fatal(msg); + codegen_aborted = Some(FatalError); + } + }, + + ThinLtoMessage::WorkItem { result } => { + // If a thread exits successfully then we drop a token associated + // with that worker and update our `used_token_count` count. + // We may later re-acquire a token to continue running more work. + // We may also not actually drop a token here if the worker was + // running with an "ephemeral token". + used_token_count -= 1; + + match result { + Ok(compiled_module) => compiled_modules.push(compiled_module), + Err(Some(WorkerFatalError)) => { + // Like `CodegenAborted`, wait for remaining work to finish. + codegen_aborted = Some(FatalError); + } + Err(None) => { + // If the thread failed that means it panicked, so + // we abort immediately. + bug!("worker thread panicked"); + } + } + } + } + } + + if let Some(codegen_aborted) = codegen_aborted { + codegen_aborted.raise(); + } + + compiled_modules } fn execute_thin_lto_work_item( cgcx: &CodegenContext, module: lto::ThinModule, - module_config: &ModuleConfig, -) -> Result, FatalError> { - let module = B::optimize_thin(cgcx, module)?; - let module = B::codegen(cgcx, module, module_config)?; - Ok(WorkItemResult::Finished(module)) +) -> CompiledModule { + let _timer = cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", module.name()); + + let module = B::optimize_thin(cgcx, module); + B::codegen(cgcx, module, &cgcx.module_config) } /// Messages sent to the coordinator. @@ -1058,6 +1182,17 @@ pub(crate) enum Message { CodegenAborted, } +/// Messages sent to the coordinator. +pub(crate) enum ThinLtoMessage { + /// A jobserver token has become available. Sent from the jobserver helper + /// thread. + Token(io::Result), + + /// The backend has finished processing a work item for a codegen unit. + /// Sent from a backend worker thread. + WorkItem { result: Result> }, +} + /// A message sent from the coordinator thread to the main thread telling it to /// process another codegen unit. pub struct CguMessage; @@ -1103,17 +1238,15 @@ fn start_executing_work( backend: B, tcx: TyCtxt<'_>, crate_info: &CrateInfo, - autodiff_items: &[AutoDiffItem], shared_emitter: SharedEmitter, codegen_worker_send: Sender, coordinator_receive: Receiver>, regular_config: Arc, allocator_config: Arc, - tx_to_llvm_workers: Sender>, + allocator_module: Option>, + coordinator_send: Sender>, ) -> thread::JoinHandle> { - let coordinator_send = tx_to_llvm_workers; let sess = tcx.sess; - let autodiff_items = autodiff_items.to_vec(); let mut each_linked_rlib_for_lto = Vec::new(); let mut each_linked_rlib_file_for_lto = Vec::new(); @@ -1141,13 +1274,7 @@ fn start_executing_work( }) .expect("failed to spawn helper thread"); - let ol = - if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - // If we know that we won’t be doing codegen, create target machines without optimisation. - config::OptLevel::No - } else { - tcx.backend_optimization_level(()) - }; + let ol = tcx.backend_optimization_level(()); let backend_features = tcx.global_backend_features(()); let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir { @@ -1171,11 +1298,10 @@ fn start_executing_work( remark: sess.opts.cg.remark.clone(), remark_dir, incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()), - expanded_args: tcx.sess.expanded_args.clone(), diag_emitter: shared_emitter.clone(), output_filenames: Arc::clone(tcx.output_filenames(())), - regular_module_config: regular_config, - allocator_module_config: allocator_config, + module_config: regular_config, + allocator_config, tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features), msvc_imps_needed: msvc_imps_needed(tcx), is_pe_coff: tcx.sess.target.is_like_windows, @@ -1329,11 +1455,9 @@ fn start_executing_work( // This is where we collect codegen units that have gone all the way // through codegen and LLVM. let mut compiled_modules = vec![]; - let mut compiled_allocator_module = None; let mut needs_fat_lto = Vec::new(); let mut needs_thin_lto = Vec::new(); let mut lto_import_only_modules = Vec::new(); - let mut started_lto = false; /// Possible state transitions: /// - Ongoing -> Completed @@ -1370,6 +1494,17 @@ fn start_executing_work( let mut llvm_start_time: Option> = None; + let compiled_allocator_module = allocator_module.and_then(|allocator_module| { + match execute_optimize_work_item(&cgcx, allocator_module) { + WorkItemResult::Finished(compiled_module) => return Some(compiled_module), + WorkItemResult::NeedsFatLto(fat_lto_input) => needs_fat_lto.push(fat_lto_input), + WorkItemResult::NeedsThinLto(name, thin_buffer) => { + needs_thin_lto.push((name, thin_buffer)) + } + } + None + }); + // Run the message loop while there's still anything that needs message // processing. Note that as soon as codegen is aborted we simply want to // wait for all existing work to finish, so many of the conditions here @@ -1412,69 +1547,8 @@ fn start_executing_work( if running_with_any_token(main_thread_state, running_with_own_token) == 0 && work_items.is_empty() { - // All codegen work is done. Do we have LTO work to do? - if needs_fat_lto.is_empty() - && needs_thin_lto.is_empty() - && lto_import_only_modules.is_empty() - { - // Nothing more to do! - break; - } - - // We have LTO work to do. Perform the serial work here of - // figuring out what we're going to LTO and then push a - // bunch of work items onto our queue to do LTO. This all - // happens on the coordinator thread but it's very quick so - // we don't worry about tokens. - assert!(!started_lto); - started_lto = true; - - let needs_fat_lto = mem::take(&mut needs_fat_lto); - let needs_thin_lto = mem::take(&mut needs_thin_lto); - let import_only_modules = mem::take(&mut lto_import_only_modules); - let each_linked_rlib_file_for_lto = - mem::take(&mut each_linked_rlib_file_for_lto); - - check_lto_allowed(&cgcx); - - if !needs_fat_lto.is_empty() { - assert!(needs_thin_lto.is_empty()); - - work_items.push(( - WorkItem::FatLto { - exported_symbols_for_lto: Arc::clone(&exported_symbols_for_lto), - each_linked_rlib_for_lto: each_linked_rlib_file_for_lto, - needs_fat_lto, - import_only_modules, - autodiff: autodiff_items.clone(), - }, - 0, - )); - if cgcx.parallel { - helper.request_token(); - } - } else { - if !autodiff_items.is_empty() { - let dcx = cgcx.create_dcx(); - dcx.handle().emit_fatal(AutodiffWithoutLto {}); - } - - for (work, cost) in generate_thin_lto_work( - &cgcx, - &exported_symbols_for_lto, - &each_linked_rlib_file_for_lto, - needs_thin_lto, - import_only_modules, - ) { - let insertion_index = work_items - .binary_search_by_key(&cost, |&(_, cost)| cost) - .unwrap_or_else(|e| e); - work_items.insert(insertion_index, (work, cost)); - if cgcx.parallel { - helper.request_token(); - } - } - } + // All codegen work is done. + break; } // In this branch, we know that everything has been codegened, @@ -1609,23 +1683,13 @@ fn start_executing_work( match result { Ok(WorkItemResult::Finished(compiled_module)) => { - match compiled_module.kind { - ModuleKind::Regular => { - compiled_modules.push(compiled_module); - } - ModuleKind::Allocator => { - assert!(compiled_allocator_module.is_none()); - compiled_allocator_module = Some(compiled_module); - } - } + compiled_modules.push(compiled_module); } Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => { - assert!(!started_lto); assert!(needs_thin_lto.is_empty()); needs_fat_lto.push(fat_lto_input); } Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => { - assert!(!started_lto); assert!(needs_fat_lto.is_empty()); needs_thin_lto.push((name, thin_buffer)); } @@ -1642,7 +1706,6 @@ fn start_executing_work( } Message::AddImportOnlyModule { module_data, work_product } => { - assert!(!started_lto); assert_eq!(codegen_state, Ongoing); assert_eq!(main_thread_state, MainThreadState::Codegenning); lto_import_only_modules.push((module_data, work_product)); @@ -1651,12 +1714,43 @@ fn start_executing_work( } } + // Drop to print timings + drop(llvm_start_time); + if codegen_state == Aborted { return Err(()); } - // Drop to print timings - drop(llvm_start_time); + drop(codegen_state); + drop(tokens); + drop(helper); + assert!(work_items.is_empty()); + + if !needs_fat_lto.is_empty() { + assert!(compiled_modules.is_empty()); + assert!(needs_thin_lto.is_empty()); + + // This uses the implicit token + let module = do_fat_lto( + &cgcx, + &exported_symbols_for_lto, + &each_linked_rlib_file_for_lto, + needs_fat_lto, + lto_import_only_modules, + ); + compiled_modules.push(module); + } else if !needs_thin_lto.is_empty() || !lto_import_only_modules.is_empty() { + assert!(compiled_modules.is_empty()); + assert!(needs_fat_lto.is_empty()); + + compiled_modules.extend(do_thin_lto( + &cgcx, + exported_symbols_for_lto, + each_linked_rlib_file_for_lto, + needs_thin_lto, + lto_import_only_modules, + )); + } // Regardless of what order these modules completed in, report them to // the backend in the same order every time to ensure that we're handing @@ -1738,83 +1832,63 @@ fn spawn_work<'a, B: ExtraBackendMethods>( llvm_start_time: &mut Option>, work: WorkItem, ) { - if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() { + if llvm_start_time.is_none() { *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes")); } let cgcx = cgcx.clone(); B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { - // Set up a destructor which will fire off a message that we're done as - // we exit. - struct Bomb { - coordinator_send: Sender>, - result: Option, FatalError>>, - } - impl Drop for Bomb { - fn drop(&mut self) { - let msg = match self.result.take() { - Some(Ok(result)) => Message::WorkItem:: { result: Ok(result) }, - Some(Err(FatalError)) => { - Message::WorkItem:: { result: Err(Some(WorkerFatalError)) } - } - None => Message::WorkItem:: { result: Err(None) }, - }; - drop(self.coordinator_send.send(msg)); + let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { + WorkItem::Optimize(m) => execute_optimize_work_item(&cgcx, m), + WorkItem::CopyPostLtoArtifacts(m) => { + WorkItemResult::Finished(execute_copy_from_cache_work_item(&cgcx, m)) } - } + })); - let mut bomb = Bomb:: { coordinator_send, result: None }; + let msg = match result { + Ok(result) => Message::WorkItem:: { result: Ok(result) }, - // Execute the work itself, and if it finishes successfully then flag - // ourselves as a success as well. - // - // Note that we ignore any `FatalError` coming out of `execute_work_item`, - // as a diagnostic was already sent off to the main thread - just - // surface that there was an error in this worker. - bomb.result = { - let module_config = cgcx.config(work.module_kind()); + // We ignore any `FatalError` coming out of `execute_work_item`, as a + // diagnostic was already sent off to the main thread - just surface + // that there was an error in this worker. + Err(err) if err.is::() => { + Message::WorkItem:: { result: Err(Some(WorkerFatalError)) } + } - Some(match work { - WorkItem::Optimize(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name); - execute_optimize_work_item(&cgcx, m, module_config) - } - WorkItem::CopyPostLtoArtifacts(m) => { - let _timer = cgcx.prof.generic_activity_with_arg( - "codegen_copy_artifacts_from_incr_cache", - &*m.name, - ); - Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config)) - } - WorkItem::FatLto { - exported_symbols_for_lto, - each_linked_rlib_for_lto, - needs_fat_lto, - import_only_modules, - autodiff, - } => { - let _timer = cgcx - .prof - .generic_activity_with_arg("codegen_module_perform_lto", "everything"); - execute_fat_lto_work_item( - &cgcx, - &exported_symbols_for_lto, - &each_linked_rlib_for_lto, - needs_fat_lto, - import_only_modules, - autodiff, - module_config, - ) - } - WorkItem::ThinLto(m) => { - let _timer = - cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name()); - execute_thin_lto_work_item(&cgcx, m, module_config) - } - }) + Err(_) => Message::WorkItem:: { result: Err(None) }, }; + drop(coordinator_send.send(msg)); + }) + .expect("failed to spawn work thread"); +} + +fn spawn_thin_lto_work<'a, B: ExtraBackendMethods>( + cgcx: &'a CodegenContext, + coordinator_send: Sender, + work: ThinLtoWorkItem, +) { + let cgcx = cgcx.clone(); + + B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || { + let result = std::panic::catch_unwind(AssertUnwindSafe(|| match work { + ThinLtoWorkItem::CopyPostLtoArtifacts(m) => execute_copy_from_cache_work_item(&cgcx, m), + ThinLtoWorkItem::ThinLto(m) => execute_thin_lto_work_item(&cgcx, m), + })); + + let msg = match result { + Ok(result) => ThinLtoMessage::WorkItem { result: Ok(result) }, + + // We ignore any `FatalError` coming out of `execute_work_item`, as a + // diagnostic was already sent off to the main thread - just surface + // that there was an error in this worker. + Err(err) if err.is::() => { + ThinLtoMessage::WorkItem { result: Err(Some(WorkerFatalError)) } + } + + Err(_) => ThinLtoMessage::WorkItem { result: Err(None) }, + }; + drop(coordinator_send.send(msg)); }) .expect("failed to spawn work thread"); } @@ -1995,10 +2069,13 @@ impl Drop for Coordinator { pub struct OngoingCodegen { pub backend: B, pub crate_info: CrateInfo, + pub output_filenames: Arc, + // Field order below is intended to terminate the coordinator thread before two fields below + // drop and prematurely close channels used by coordinator thread. See `Coordinator`'s + // `Drop` implementation for more info. + pub coordinator: Coordinator, pub codegen_worker_receive: Receiver, pub shared_emitter_main: SharedEmitterMain, - pub output_filenames: Arc, - pub coordinator: Coordinator, } impl OngoingCodegen { diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index a5807c56e317..f58be0a3b9a3 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -6,17 +6,20 @@ use std::time::{Duration, Instant}; use itertools::Itertools; use rustc_abi::FIRST_VARIANT; use rustc_ast as ast; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_attr_data_structures::OptimizeAttr; +use rustc_ast::expand::allocator::{ + ALLOC_ERROR_HANDLER, ALLOCATOR_METHODS, AllocatorKind, AllocatorMethod, AllocatorTy, +}; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry}; use rustc_data_structures::sync::{IntoDynSyncSend, par_map}; use rustc_data_structures::unord::UnordMap; +use rustc_hir::attrs::{DebuggerVisualizerType, OptimizeAttr}; use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_hir::lang_items::LangItem; use rustc_hir::{ItemId, Target}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType}; +use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile; +use rustc_middle::middle::dependency_format::Dependencies; use rustc_middle::middle::exported_symbols::{self, SymbolExportKind}; use rustc_middle::middle::lang_items; use rustc_middle::mir::BinOp; @@ -138,7 +141,7 @@ pub fn validate_trivial_unsize<'tcx>( ) else { return false; }; - if !ocx.select_all_or_error().is_empty() { + if !ocx.evaluate_obligations_error_on_ambiguity().is_empty() { return false; } infcx.leak_check(universe, None).is_ok() @@ -167,9 +170,7 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( (&ty::Array(_, len), &ty::Slice(_)) => cx.const_usize( len.try_to_target_usize(cx.tcx()).expect("expected monomorphic const in codegen"), ), - (&ty::Dynamic(data_a, _, src_dyn_kind), &ty::Dynamic(data_b, _, target_dyn_kind)) - if src_dyn_kind == target_dyn_kind => - { + (&ty::Dynamic(data_a, _), &ty::Dynamic(data_b, _)) => { let old_info = old_info.expect("unsized_info: missing old info for trait upcasting coercion"); let b_principal_def_id = data_b.principal_def_id(); @@ -207,7 +208,7 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( old_info } } - (_, ty::Dynamic(data, _, _)) => meth::get_vtable( + (_, ty::Dynamic(data, _)) => meth::get_vtable( cx, source, data.principal() @@ -227,6 +228,7 @@ pub(crate) fn unsize_ptr<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ) -> (Bx::Value, Bx::Value) { debug!("unsize_ptr: {:?} => {:?}", src_ty, dst_ty); match (src_ty.kind(), dst_ty.kind()) { + (&ty::Pat(a, _), &ty::Pat(b, _)) => unsize_ptr(bx, src, a, b, old_info), (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(b, _)) | (&ty::RawPtr(a, _), &ty::RawPtr(b, _)) => { assert_eq!(bx.cx().type_is_sized(a), old_info.is_none()); @@ -630,14 +632,51 @@ pub fn allocator_kind_for_codegen(tcx: TyCtxt<'_>) -> Option { // If the crate doesn't have an `allocator_kind` set then there's definitely // no shim to generate. Otherwise we also check our dependency graph for all // our output crate types. If anything there looks like its a `Dynamic` - // linkage, then it's already got an allocator shim and we'll be using that - // one instead. If nothing exists then it's our job to generate the - // allocator! - let any_dynamic_crate = tcx.dependency_formats(()).iter().any(|(_, list)| { + // linkage for all crate types we may link as, then it's already got an + // allocator shim and we'll be using that one instead. If nothing exists + // then it's our job to generate the allocator! If crate types disagree + // about whether an allocator shim is necessary or not, we generate one + // and let needs_allocator_shim_for_linking decide at link time whether or + // not to use it for any particular linker invocation. + let all_crate_types_any_dynamic_crate = tcx.dependency_formats(()).iter().all(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; list.iter().any(|&linkage| linkage == Linkage::Dynamic) }); - if any_dynamic_crate { None } else { tcx.allocator_kind(()) } + if all_crate_types_any_dynamic_crate { None } else { tcx.allocator_kind(()) } +} + +/// Decide if this particular crate type needs an allocator shim linked in. +/// This may return true even when allocator_kind_for_codegen returns false. In +/// this case no allocator shim shall be linked. +pub(crate) fn needs_allocator_shim_for_linking( + dependency_formats: &Dependencies, + crate_type: CrateType, +) -> bool { + use rustc_middle::middle::dependency_format::Linkage; + let any_dynamic_crate = + dependency_formats[&crate_type].iter().any(|&linkage| linkage == Linkage::Dynamic); + !any_dynamic_crate +} + +pub fn allocator_shim_contents(tcx: TyCtxt<'_>, kind: AllocatorKind) -> Vec { + let mut methods = Vec::new(); + + if kind == AllocatorKind::Default { + methods.extend(ALLOCATOR_METHODS.into_iter().copied()); + } + + // If the return value of allocator_kind_for_codegen is Some then + // alloc_error_handler_kind must also be Some. + if tcx.alloc_error_handler_kind(()).unwrap() == AllocatorKind::Default { + methods.push(AllocatorMethod { + name: ALLOC_ERROR_HANDLER, + special: None, + inputs: &[], + output: AllocatorTy::Never, + }); + } + + methods } pub fn codegen_crate( @@ -645,17 +684,6 @@ pub fn codegen_crate( tcx: TyCtxt<'_>, target_cpu: String, ) -> OngoingCodegen { - // Skip crate items and just output metadata in -Z no-codegen mode. - if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() { - let ongoing_codegen = start_async_codegen(backend, tcx, target_cpu, &[]); - - ongoing_codegen.codegen_finished(tcx); - - ongoing_codegen.check_for_errors(tcx.sess); - - return ongoing_codegen; - } - if tcx.sess.target.need_explicit_cpu && tcx.sess.opts.cg.target_cpu.is_none() { // The target has no default cpu, but none is set explicitly tcx.dcx().emit_fatal(errors::CpuRequired); @@ -665,8 +693,7 @@ pub fn codegen_crate( // Run the monomorphization collector and partition the collected items into // codegen units. - let MonoItemPartitions { codegen_units, autodiff_items, .. } = - tcx.collect_and_partition_mono_items(()); + let MonoItemPartitions { codegen_units, .. } = tcx.collect_and_partition_mono_items(()); // Force all codegen_unit queries so they are already either red or green // when compile_codegen_unit accesses them. We are not able to re-execute @@ -679,34 +706,21 @@ pub fn codegen_crate( } } - let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu, autodiff_items); - // Codegen an allocator shim, if necessary. - if let Some(kind) = allocator_kind_for_codegen(tcx) { + let allocator_module = if let Some(kind) = allocator_kind_for_codegen(tcx) { let llmod_id = cgu_name_builder.build_cgu_name(LOCAL_CRATE, &["crate"], Some("allocator")).to_string(); - let module_llvm = tcx.sess.time("write_allocator_module", || { - backend.codegen_allocator( - tcx, - &llmod_id, - kind, - // If allocator_kind is Some then alloc_error_handler_kind must - // also be Some. - tcx.alloc_error_handler_kind(()).unwrap(), - ) - }); - ongoing_codegen.wait_for_signal_to_codegen_item(); - ongoing_codegen.check_for_errors(tcx.sess); + tcx.sess.time("write_allocator_module", || { + let module = + backend.codegen_allocator(tcx, &llmod_id, &allocator_shim_contents(tcx, kind)); + Some(ModuleCodegen::new_allocator(llmod_id, module)) + }) + } else { + None + }; - // These modules are generally cheap and won't throw off scheduling. - let cost = 0; - submit_codegened_module_to_llvm( - &ongoing_codegen.coordinator, - ModuleCodegen::new_allocator(llmod_id, module_llvm), - cost, - ); - } + let ongoing_codegen = start_async_codegen(backend.clone(), tcx, target_cpu, allocator_module); // For better throughput during parallel processing by LLVM, we used to sort // CGUs largest to smallest. This would lead to better thread utilization @@ -857,7 +871,7 @@ pub fn is_call_from_compiler_builtins_to_upstream_monomorphization<'tcx>( instance: Instance<'tcx>, ) -> bool { fn is_llvm_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - if let Some(name) = tcx.codegen_fn_attrs(def_id).link_name { + if let Some(name) = tcx.codegen_fn_attrs(def_id).symbol_name { name.as_str().starts_with("llvm.") } else { false diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 5c54bce6e035..44de48a3ada5 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -3,17 +3,13 @@ use std::str::FromStr; use rustc_abi::{Align, ExternAbi}; use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, DiffActivity, DiffMode}; use rustc_ast::{LitKind, MetaItem, MetaItemInner, attr}; -use rustc_attr_data_structures::{ - AttributeKind, InlineAttr, InstructionSetAttr, UsedBy, find_attr, -}; +use rustc_hir::attrs::{AttributeKind, InlineAttr, InstructionSetAttr, UsedBy}; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; -use rustc_hir::weak_lang_items::WEAK_LANG_ITEMS; -use rustc_hir::{self as hir, Attribute, LangItem, lang_items}; +use rustc_hir::{self as hir, Attribute, LangItem, find_attr, lang_items}; use rustc_middle::middle::codegen_fn_attrs::{ CodegenFnAttrFlags, CodegenFnAttrs, PatchableFunctionEntry, }; -use rustc_middle::mir::mono::Linkage; use rustc_middle::query::Providers; use rustc_middle::span_bug; use rustc_middle::ty::{self as ty, TyCtxt}; @@ -23,36 +19,10 @@ use rustc_span::{Ident, Span, sym}; use rustc_target::spec::SanitizerSet; use crate::errors; -use crate::errors::NoMangleNameless; use crate::target_features::{ check_target_feature_trait_unsafe, check_tied_features, from_target_feature_attr, }; -fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage { - use rustc_middle::mir::mono::Linkage::*; - - // Use the names from src/llvm/docs/LangRef.rst here. Most types are only - // applicable to variable declarations and may not really make sense for - // Rust code in the first place but allow them anyway and trust that the - // user knows what they're doing. Who knows, unanticipated use cases may pop - // up in the future. - // - // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported - // and don't have to be, LLVM treats them as no-ops. - match name { - "available_externally" => AvailableExternally, - "common" => Common, - "extern_weak" => ExternalWeak, - "external" => External, - "internal" => Internal, - "linkonce" => LinkOnceAny, - "linkonce_odr" => LinkOnceODR, - "weak" => WeakAny, - "weak_odr" => WeakODR, - _ => tcx.dcx().span_fatal(tcx.def_span(def_id), "invalid linkage specified"), - } -} - /// In some cases, attributes are only valid on functions, but it's the `check_attr` /// pass that checks that they aren't used anywhere else, rather than this module. /// In these cases, we bail from performing further checks that are only meaningful for @@ -105,39 +75,6 @@ fn parse_instruction_set_attr(tcx: TyCtxt<'_>, attr: &Attribute) -> Option, did: LocalDefId, attr: &Attribute) -> Option { - let val = attr.value_str()?; - let linkage = linkage_by_name(tcx, did, val.as_str()); - Some(linkage) -} - -// FIXME(jdonszelmann): remove when no_sanitize becomes a parsed attr -fn parse_no_sanitize_attr(tcx: TyCtxt<'_>, attr: &Attribute) -> Option { - let list = attr.meta_item_list()?; - let mut sanitizer_set = SanitizerSet::empty(); - - for item in list.iter() { - match item.name() { - Some(sym::address) => { - sanitizer_set |= SanitizerSet::ADDRESS | SanitizerSet::KERNELADDRESS - } - Some(sym::cfi) => sanitizer_set |= SanitizerSet::CFI, - Some(sym::kcfi) => sanitizer_set |= SanitizerSet::KCFI, - Some(sym::memory) => sanitizer_set |= SanitizerSet::MEMORY, - Some(sym::memtag) => sanitizer_set |= SanitizerSet::MEMTAG, - Some(sym::shadow_call_stack) => sanitizer_set |= SanitizerSet::SHADOWCALLSTACK, - Some(sym::thread) => sanitizer_set |= SanitizerSet::THREAD, - Some(sym::hwaddress) => sanitizer_set |= SanitizerSet::HWADDRESS, - _ => { - tcx.dcx().emit_err(errors::InvalidNoSanitize { span: item.span() }); - } - } - } - - Some(sanitizer_set) -} - // FIXME(jdonszelmann): remove when patchable_function_entry becomes a parsed attr fn parse_patchable_function_entry( tcx: TyCtxt<'_>, @@ -196,7 +133,7 @@ fn parse_patchable_function_entry( #[derive(Default)] struct InterestingAttributeDiagnosticSpans { link_ordinal: Option, - no_sanitize: Option, + sanitize: Option, inline: Option, no_mangle: Option, } @@ -212,20 +149,12 @@ fn process_builtin_attrs( let mut interesting_spans = InterestingAttributeDiagnosticSpans::default(); let rust_target_features = tcx.rust_target_features(LOCAL_CRATE); - // If our rustc version supports autodiff/enzyme, then we call our handler - // to check for any `#[rustc_autodiff(...)]` attributes. - // FIXME(jdonszelmann): merge with loop below - if cfg!(llvm_enzyme) { - let ad = autodiff_attrs(tcx, did.into()); - codegen_fn_attrs.autodiff_item = ad; - } - for attr in attrs.iter() { if let hir::Attribute::Parsed(p) = attr { match p { AttributeKind::Cold(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::COLD, AttributeKind::ExportName { name, .. } => { - codegen_fn_attrs.export_name = Some(*name) + codegen_fn_attrs.symbol_name = Some(*name) } AttributeKind::Inline(inline, span) => { codegen_fn_attrs.inline = *inline; @@ -233,7 +162,13 @@ fn process_builtin_attrs( } AttributeKind::Naked(_) => codegen_fn_attrs.flags |= CodegenFnAttrFlags::NAKED, AttributeKind::Align { align, .. } => codegen_fn_attrs.alignment = Some(*align), - AttributeKind::LinkName { name, .. } => codegen_fn_attrs.link_name = Some(*name), + AttributeKind::LinkName { name, .. } => { + // FIXME Remove check for foreign functions once #[link_name] on non-foreign + // functions is a hard error + if tcx.is_foreign_item(did) { + codegen_fn_attrs.symbol_name = Some(*name); + } + } AttributeKind::LinkOrdinal { ordinal, span } => { codegen_fn_attrs.link_ordinal = Some(*ordinal); interesting_spans.link_ordinal = Some(*span); @@ -246,18 +181,14 @@ fn process_builtin_attrs( if tcx.opt_item_name(did.to_def_id()).is_some() { codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; } else { - tcx.dcx().emit_err(NoMangleNameless { - span: *attr_span, - definition: format!( - "{} {}", - tcx.def_descr_article(did.to_def_id()), - tcx.def_descr(did.to_def_id()) - ), - }); + tcx.dcx().span_delayed_bug( + *attr_span, + "no_mangle should be on a named function", + ); } } AttributeKind::Optimize(optimize, _) => codegen_fn_attrs.optimize = *optimize, - AttributeKind::TargetFeature(features, attr_span) => { + AttributeKind::TargetFeature { features, attr_span, was_forced } => { let Some(sig) = tcx.hir_node_by_def_id(did).fn_sig() else { tcx.dcx().span_delayed_bug(*attr_span, "target_feature applied to non-fn"); continue; @@ -265,7 +196,7 @@ fn process_builtin_attrs( let safe_target_features = matches!(sig.header.safety, hir::HeaderSafety::SafeTargetFeatures); codegen_fn_attrs.safe_target_features = safe_target_features; - if safe_target_features { + if safe_target_features && !was_forced { if tcx.sess.target.is_like_wasm || tcx.sess.opts.actually_rustdoc { // The `#[target_feature]` attribute is allowed on // WebAssembly targets on all functions. Prior to stabilizing @@ -296,6 +227,7 @@ fn process_builtin_attrs( tcx, did, features, + *was_forced, rust_target_features, &mut codegen_fn_attrs.target_features, ); @@ -326,6 +258,19 @@ fn process_builtin_attrs( AttributeKind::Used { used_by, .. } => match used_by { UsedBy::Compiler => codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_COMPILER, UsedBy::Linker => codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_LINKER, + UsedBy::Default => { + let used_form = if tcx.sess.target.os == "illumos" { + // illumos' `ld` doesn't support a section header that would represent + // `#[used(linker)]`, see + // https://github.com/rust-lang/rust/issues/146169. For that target, + // downgrade as if `#[used(compiler)]` was requested and hope for the + // best. + CodegenFnAttrFlags::USED_COMPILER + } else { + CodegenFnAttrFlags::USED_LINKER + }; + codegen_fn_attrs.flags |= used_form; + } }, AttributeKind::FfiConst(_) => { codegen_fn_attrs.flags |= CodegenFnAttrFlags::FFI_CONST @@ -334,6 +279,37 @@ fn process_builtin_attrs( AttributeKind::StdInternalSymbol(_) => { codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL } + AttributeKind::Linkage(linkage, span) => { + let linkage = Some(*linkage); + + if tcx.is_foreign_item(did) { + codegen_fn_attrs.import_linkage = linkage; + + if tcx.is_mutable_static(did.into()) { + let mut diag = tcx.dcx().struct_span_err( + *span, + "extern mutable statics are not allowed with `#[linkage]`", + ); + diag.note( + "marking the extern static mutable would allow changing which \ + symbol the static references rather than make the target of the \ + symbol mutable", + ); + diag.emit(); + } + } else { + codegen_fn_attrs.linkage = linkage; + } + } + AttributeKind::Sanitize { span, .. } => { + interesting_spans.sanitize = Some(*span); + } + AttributeKind::ObjcClass { classname, .. } => { + codegen_fn_attrs.objc_class = Some(*classname); + } + AttributeKind::ObjcSelector { methname, .. } => { + codegen_fn_attrs.objc_selector = Some(*methname); + } _ => {} } } @@ -351,33 +327,6 @@ fn process_builtin_attrs( codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED } sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL, - sym::linkage => { - let linkage = parse_linkage_attr(tcx, did, attr); - - if tcx.is_foreign_item(did) { - codegen_fn_attrs.import_linkage = linkage; - - if tcx.is_mutable_static(did.into()) { - let mut diag = tcx.dcx().struct_span_err( - attr.span(), - "extern mutable statics are not allowed with `#[linkage]`", - ); - diag.note( - "marking the extern static mutable would allow changing which \ - symbol the static references rather than make the target of the \ - symbol mutable", - ); - diag.emit(); - } - } else { - codegen_fn_attrs.linkage = linkage; - } - } - sym::no_sanitize => { - interesting_spans.no_sanitize = Some(attr.span()); - codegen_fn_attrs.no_sanitize |= - parse_no_sanitize_attr(tcx, attr).unwrap_or_default(); - } sym::instruction_set => { codegen_fn_attrs.instruction_set = parse_instruction_set_attr(tcx, attr) } @@ -401,6 +350,8 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code codegen_fn_attrs.alignment = Ord::max(codegen_fn_attrs.alignment, tcx.sess.opts.unstable_opts.min_function_alignment); + // Compute the disabled sanitizers. + codegen_fn_attrs.no_sanitize |= tcx.disabled_sanitizers_for(did); // On trait methods, inherit the `#[align]` of the trait's method prototype. codegen_fn_attrs.alignment = Ord::max(codegen_fn_attrs.alignment, tcx.inherited_align(did)); @@ -445,6 +396,29 @@ fn apply_overrides(tcx: TyCtxt<'_>, did: LocalDefId, codegen_fn_attrs: &mut Code if tcx.should_inherit_track_caller(did) { codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER; } + + // Foreign items by default use no mangling for their symbol name. + if tcx.is_foreign_item(did) { + codegen_fn_attrs.flags |= CodegenFnAttrFlags::FOREIGN_ITEM; + + // There's a few exceptions to this rule though: + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) { + // * `#[rustc_std_internal_symbol]` mangles the symbol name in a special way + // both for exports and imports through foreign items. This is handled further, + // during symbol mangling logic. + } else if codegen_fn_attrs.symbol_name.is_some() { + // * This can be overridden with the `#[link_name]` attribute + } else { + // NOTE: there's one more exception that we cannot apply here. On wasm, + // some items cannot be `no_mangle`. + // However, we don't have enough information here to determine that. + // As such, no_mangle foreign items on wasm that have the same defid as some + // import will *still* be mangled despite this. + // + // if none of the exceptions apply; apply no_mangle + codegen_fn_attrs.flags |= CodegenFnAttrFlags::NO_MANGLE; + } + } } fn check_result( @@ -468,26 +442,33 @@ fn check_result( // llvm/llvm-project#70563). if !codegen_fn_attrs.target_features.is_empty() && matches!(codegen_fn_attrs.inline, InlineAttr::Always) + && !tcx.features().target_feature_inline_always() && let Some(span) = interesting_spans.inline { - tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`"); + feature_err( + tcx.sess, + sym::target_feature_inline_always, + span, + "cannot use `#[inline(always)]` with `#[target_feature]`", + ) + .emit(); } // warn that inline has no effect when no_sanitize is present if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() && let (Some(no_sanitize_span), Some(inline_span)) = - (interesting_spans.no_sanitize, interesting_spans.inline) + (interesting_spans.sanitize, interesting_spans.inline) { let hir_id = tcx.local_def_id_to_hir_id(did); tcx.node_span_lint(lint::builtin::INLINE_NO_SANITIZE, hir_id, no_sanitize_span, |lint| { - lint.primary_message("`no_sanitize` will have no effect after inlining"); + lint.primary_message("setting `sanitize` off will have no effect after inlining"); lint.span_note(inline_span, "inlining requested here"); }) } // error when specifying link_name together with link_ordinal - if let Some(_) = codegen_fn_attrs.link_name + if let Some(_) = codegen_fn_attrs.symbol_name && let Some(_) = codegen_fn_attrs.link_ordinal { let msg = "cannot use `#[link_name]` with `#[link_ordinal]`"; @@ -507,7 +488,7 @@ fn check_result( .collect(), ) { let span = - find_attr!(tcx.get_all_attrs(did), AttributeKind::TargetFeature(_, span) => *span) + find_attr!(tcx.get_all_attrs(did), AttributeKind::TargetFeature{attr_span: span, ..} => *span) .unwrap_or_else(|| tcx.def_span(did)); tcx.dcx() @@ -527,29 +508,24 @@ fn handle_lang_items( attrs: &[Attribute], codegen_fn_attrs: &mut CodegenFnAttrs, ) { + let lang_item = lang_items::extract(attrs).and_then(|(name, _)| LangItem::from_name(name)); + // Weak lang items have the same semantics as "std internal" symbols in the // sense that they're preserved through all our LTO passes and only // strippable by the linker. // // Additionally weak lang items have predetermined symbol names. - if let Some((name, _)) = lang_items::extract(attrs) - && let Some(lang_item) = LangItem::from_name(name) + if let Some(lang_item) = lang_item + && let Some(link_name) = lang_item.link_name() { - if WEAK_LANG_ITEMS.contains(&lang_item) { - codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL; - } - if let Some(link_name) = lang_item.link_name() { - codegen_fn_attrs.export_name = Some(link_name); - codegen_fn_attrs.link_name = Some(link_name); - } + codegen_fn_attrs.flags |= CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL; + codegen_fn_attrs.symbol_name = Some(link_name); } // error when using no_mangle on a lang item item if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL) && codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_MANGLE) { - let lang_item = - lang_items::extract(attrs).map_or(None, |(name, _span)| LangItem::from_name(name)); let mut err = tcx .dcx() .struct_span_err( @@ -600,26 +576,44 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs { codegen_fn_attrs } -/// If the provided DefId is a method in a trait impl, return the DefId of the method prototype. -fn opt_trait_item(tcx: TyCtxt<'_>, def_id: DefId) -> Option { - let impl_item = tcx.opt_associated_item(def_id)?; - match impl_item.container { - ty::AssocItemContainer::Impl => impl_item.trait_item_def_id, - _ => None, +fn disabled_sanitizers_for(tcx: TyCtxt<'_>, did: LocalDefId) -> SanitizerSet { + // Backtrack to the crate root. + let mut disabled = match tcx.opt_local_parent(did) { + // Check the parent (recursively). + Some(parent) => tcx.disabled_sanitizers_for(parent), + // We reached the crate root without seeing an attribute, so + // there is no sanitizers to exclude. + None => SanitizerSet::empty(), + }; + + // Check for a sanitize annotation directly on this def. + if let Some((on_set, off_set)) = find_attr!(tcx.get_all_attrs(did), AttributeKind::Sanitize {on_set, off_set, ..} => (on_set, off_set)) + { + // the on set is the set of sanitizers explicitly enabled. + // we mask those out since we want the set of disabled sanitizers here + disabled &= !*on_set; + // the off set is the set of sanitizers explicitly disabled. + // we or those in here. + disabled |= *off_set; + // the on set and off set are distjoint since there's a third option: unset. + // a node may not set the sanitizer setting in which case it inherits from parents. + // the code above in this function does this backtracking } + disabled } /// Checks if the provided DefId is a method in a trait impl for a trait which has track_caller /// applied to the method prototype. fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool { - let Some(trait_item) = opt_trait_item(tcx, def_id) else { return false }; - tcx.codegen_fn_attrs(trait_item).flags.intersects(CodegenFnAttrFlags::TRACK_CALLER) + tcx.trait_item_of(def_id).is_some_and(|id| { + tcx.codegen_fn_attrs(id).flags.intersects(CodegenFnAttrFlags::TRACK_CALLER) + }) } /// If the provided DefId is a method in a trait impl, return the value of the `#[align]` /// attribute on the method prototype (if any). fn inherited_align<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option { - tcx.codegen_fn_attrs(opt_trait_item(tcx, def_id)?).alignment + tcx.codegen_fn_attrs(tcx.trait_item_of(def_id)?).alignment } /// We now check the #\[rustc_autodiff\] attributes which we generated from the #[autodiff(...)] @@ -628,7 +622,7 @@ fn inherited_align<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option { /// placeholder functions. We wrote the rustc_autodiff attributes ourself, so this should never /// panic, unless we introduced a bug when parsing the autodiff macro. //FIXME(jdonszelmann): put in the main loop. No need to have two..... :/ Let's do that when we make autodiff parsed. -fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { +pub fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { let attrs = tcx.get_attrs(id, sym::rustc_autodiff); let attrs = attrs.filter(|attr| attr.has_name(sym::rustc_autodiff)).collect::>(); @@ -733,6 +727,11 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option { } pub(crate) fn provide(providers: &mut Providers) { - *providers = - Providers { codegen_fn_attrs, should_inherit_track_caller, inherited_align, ..*providers }; + *providers = Providers { + codegen_fn_attrs, + should_inherit_track_caller, + inherited_align, + disabled_sanitizers_for, + ..*providers + }; } diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs index a6fd6c763edd..08e2f3559533 100644 --- a/compiler/rustc_codegen_ssa/src/common.rs +++ b/compiler/rustc_codegen_ssa/src/common.rs @@ -1,10 +1,11 @@ #![allow(non_camel_case_types)] use rustc_hir::LangItem; +use rustc_hir::attrs::PeImportNameType; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Instance, TyCtxt}; use rustc_middle::{bug, mir, span_bug}; -use rustc_session::cstore::{DllCallingConvention, DllImport, PeImportNameType}; +use rustc_session::cstore::{DllCallingConvention, DllImport}; use rustc_span::Span; use rustc_target::spec::Target; diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index 3d787d8bdbde..2dd7c6fa7c08 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -37,10 +37,6 @@ pub(crate) struct CguNotRecorded<'a> { pub cgu_name: &'a str, } -#[derive(Diagnostic)] -#[diag(codegen_ssa_autodiff_without_lto)] -pub struct AutodiffWithoutLto; - #[derive(Diagnostic)] #[diag(codegen_ssa_unknown_reuse_kind)] pub(crate) struct UnknownReuseKind { @@ -550,6 +546,18 @@ impl Diagnostic<'_, G> for LinkingFailed<'_> { #[diag(codegen_ssa_link_exe_unexpected_error)] pub(crate) struct LinkExeUnexpectedError; +pub(crate) struct LinkExeStatusStackBufferOverrun; + +impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for LinkExeStatusStackBufferOverrun { + fn into_diag(self, dcx: rustc_errors::DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> { + let mut diag = + Diag::new(dcx, level, fluent::codegen_ssa_link_exe_status_stack_buffer_overrun); + diag.note(fluent::codegen_ssa_abort_note); + diag.note(fluent::codegen_ssa_event_log_note); + diag + } +} + #[derive(Diagnostic)] #[diag(codegen_ssa_repair_vs_build_tools)] pub(crate) struct RepairVSBuildTools; @@ -764,6 +772,14 @@ pub enum InvalidMonomorphization<'tcx> { ty: Ty<'tcx>, }, + #[diag(codegen_ssa_invalid_monomorphization_basic_integer_or_ptr_type, code = E0511)] + BasicIntegerOrPtrType { + #[primary_span] + span: Span, + name: Symbol, + ty: Ty<'tcx>, + }, + #[diag(codegen_ssa_invalid_monomorphization_basic_float_type, code = E0511)] BasicFloatType { #[primary_span] @@ -1100,14 +1116,6 @@ impl IntoDiagArg for ExpectedPointerMutability { } } -#[derive(Diagnostic)] -#[diag(codegen_ssa_invalid_no_sanitize)] -#[note] -pub(crate) struct InvalidNoSanitize { - #[primary_span] - pub span: Span, -} - #[derive(Diagnostic)] #[diag(codegen_ssa_target_feature_safe_trait)] pub(crate) struct TargetFeatureSafeTrait { @@ -1276,14 +1284,6 @@ impl Diagnostic<'_, G> for TargetFeatureDisableOrEnable<'_ } } -#[derive(Diagnostic)] -#[diag(codegen_ssa_no_mangle_nameless)] -pub(crate) struct NoMangleNameless { - #[primary_span] - pub span: Span, - pub definition: String, -} - #[derive(Diagnostic)] #[diag(codegen_ssa_feature_not_valid)] pub(crate) struct FeatureNotValid<'a> { diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 23ed387a3ff9..baba8f9ca3e8 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -25,10 +25,10 @@ use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; -use rustc_ast as ast; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_data_structures::unord::UnordMap; use rustc_hir::CRATE_HIR_ID; +use rustc_hir::attrs::{CfgEntry, NativeLibKind}; use rustc_hir::def_id::CrateNum; use rustc_macros::{Decodable, Encodable, HashStable}; use rustc_metadata::EncodedMetadata; @@ -45,7 +45,6 @@ use rustc_session::Session; use rustc_session::config::{CrateType, OutputFilenames, OutputType, RUST_CGU_EXT}; use rustc_session::cstore::{self, CrateSource}; use rustc_session::lint::builtin::LINKER_MESSAGES; -use rustc_session::utils::NativeLibKind; use rustc_span::Symbol; pub mod assert_module_sources; @@ -120,7 +119,7 @@ impl ModuleCodegen { }); CompiledModule { - name: self.name.clone(), + name: self.name, kind: self.kind, object, dwarf_object, @@ -187,7 +186,7 @@ pub struct NativeLib { pub kind: NativeLibKind, pub name: Symbol, pub filename: Option, - pub cfg: Option, + pub cfg: Option, pub verbatim: bool, pub dll_imports: Vec, } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 34ad35a729b9..2fa466b50017 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -78,7 +78,7 @@ fn dyn_trait_in_self<'tcx>( ) -> Option> { for arg in ty.peel_refs().walk() { if let GenericArgKind::Type(ty) = arg.kind() - && let ty::Dynamic(data, _, _) = ty.kind() + && let ty::Dynamic(data, _) = ty.kind() { // FIXME(arbitrary_self_types): This is likely broken for receivers which // have a "non-self" trait objects as a generic argument. diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index c2c023af090a..de755d561780 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -150,10 +150,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx> { layout.for_variant(self.fx.cx, vidx) } - mir::PlaceElem::Subtype(subtype_ty) => { - let subtype_ty = self.fx.monomorphize(subtype_ty); - self.fx.cx.layout_of(subtype_ty) - } _ => { self.locals[place_ref.local] = LocalKind::Memory; return; @@ -233,7 +229,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer PlaceContext::MutatingUse( MutatingUseContext::Store - | MutatingUseContext::Deinit | MutatingUseContext::SetDiscriminant | MutatingUseContext::AsmOutput | MutatingUseContext::Borrow @@ -264,6 +259,10 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer PlaceContext::MutatingUse(MutatingUseContext::Yield) => bug!(), } } + + fn visit_statement_debuginfo(&mut self, _: &mir::StmtDebugInfo<'tcx>, _: Location) { + // Debuginfo does not generate actual code. + } } #[derive(Copy, Clone, Debug, PartialEq, Eq)] diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index bde63fd501aa..e2241a77d186 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -5,6 +5,7 @@ use rustc_ast as ast; use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_data_structures::packed::Pu128; use rustc_hir::lang_items::LangItem; +use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER; use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; @@ -35,6 +36,14 @@ enum MergingSucc { True, } +/// Indicates to the call terminator codegen whether a call +/// is a normal call or an explicit tail call. +#[derive(Debug, PartialEq)] +enum CallKind { + Normal, + Tail, +} + /// Used by `FunctionCx::codegen_terminator` for emitting common patterns /// e.g., creating a basic block, calling a function, etc. struct TerminatorCodegenHelper<'tcx> { @@ -160,6 +169,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { mut unwind: mir::UnwindAction, lifetime_ends_after_call: &[(Bx::Value, Size)], instance: Option>, + kind: CallKind, mergeable_succ: bool, ) -> MergingSucc { let tcx = bx.tcx(); @@ -190,10 +200,11 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { let fn_ty = bx.fn_decl_backend_type(fn_abi); let fn_attrs = if bx.tcx().def_kind(fx.instance.def_id()).has_codegen_attrs() { - Some(bx.tcx().codegen_fn_attrs(fx.instance.def_id())) + Some(bx.tcx().codegen_instance_attrs(fx.instance.def)) } else { None }; + let fn_attrs = fn_attrs.as_deref(); if !fn_abi.can_unwind { unwind = mir::UnwindAction::Unreachable; @@ -221,6 +232,11 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } }; + if kind == CallKind::Tail { + bx.tail_call(fn_ty, fn_attrs, fn_abi, fn_ptr, llargs, self.funclet(fx), instance); + return MergingSucc::False; + } + if let Some(unwind_block) = unwind_block { let ret_llbb = if let Some((_, target)) = destination { fx.llbb(target) @@ -504,6 +520,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match self.locals[mir::Local::from_usize(1 + va_list_arg_idx)] { LocalRef::Place(va_list) => { bx.va_end(va_list.val.llval); + + // Explicitly end the lifetime of the `va_list`, improves LLVM codegen. + bx.lifetime_end(va_list.val.llval, va_list.layout.size); } _ => bug!("C-variadic function must have a `VaList` place"), } @@ -596,7 +615,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() { // FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`? - ty::Dynamic(_, _, ty::Dyn) => { + ty::Dynamic(_, _) => { // IN THIS ARM, WE HAVE: // ty = *mut (dyn Trait) // which is: exists ( *mut T, Vtable ) @@ -659,6 +678,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { unwind, &[], Some(drop_instance), + CallKind::Normal, !maybe_null && mergeable_succ, ) } @@ -747,8 +767,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let (fn_abi, llfn, instance) = common::build_langcall(bx, span, lang_item); // Codegen the actual panic invoke/call. - let merging_succ = - helper.do_call(self, bx, fn_abi, llfn, &args, None, unwind, &[], Some(instance), false); + let merging_succ = helper.do_call( + self, + bx, + fn_abi, + llfn, + &args, + None, + unwind, + &[], + Some(instance), + CallKind::Normal, + false, + ); assert_eq!(merging_succ, MergingSucc::False); MergingSucc::False } @@ -777,6 +808,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::UnwindAction::Unreachable, &[], Some(instance), + CallKind::Normal, false, ); assert_eq!(merging_succ, MergingSucc::False); @@ -845,6 +877,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { unwind, &[], Some(instance), + CallKind::Normal, mergeable_succ, )) } @@ -860,6 +893,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { target: Option, unwind: mir::UnwindAction, fn_span: Span, + kind: CallKind, mergeable_succ: bool, ) -> MergingSucc { let source_info = mir::SourceInfo { span: fn_span, ..terminator.source_info }; @@ -877,7 +911,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_span, ); - let instance = match instance.def { + match instance.def { // We don't need AsyncDropGlueCtorShim here because it is not `noop func`, // it is `func returning noop future` ty::InstanceKind::DropGlue(_, None) => { @@ -966,14 +1000,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { intrinsic.name, ); } - instance + (Some(instance), None) } } } - _ => instance, - }; - (Some(instance), None) + _ if kind == CallKind::Tail + && instance.def.requires_caller_location(bx.tcx()) => + { + if let Some(hir_id) = + terminator.source_info.scope.lint_root(&self.mir.source_scopes) + { + let msg = "tail calling a function marked with `#[track_caller]` has no special effect"; + bx.tcx().node_lint(TAIL_CALL_TRACK_CALLER, hir_id, |d| { + _ = d.primary_message(msg).span(fn_span) + }); + } + + let instance = ty::Instance::resolve_for_fn_ptr( + bx.tcx(), + bx.typing_env(), + def_id, + generic_args, + ) + .unwrap(); + + (None, Some(bx.get_fn_addr(instance))) + } + _ => (Some(instance), None), + } } ty::FnPtr(..) => (None, Some(callee.immediate())), _ => bug!("{} is not callable", callee.layout.ty), @@ -1003,8 +1058,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); - let destination = target.map(|target| (return_dest, target)); + let destination = match kind { + CallKind::Normal => { + let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); + target.map(|target| (return_dest, target)) + } + CallKind::Tail => None, + }; // Split the rust-call tupled arguments off. let (first_args, untuple) = if sig.abi() == ExternAbi::RustCall @@ -1020,6 +1080,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // to generate `lifetime_end` when the call returns. let mut lifetime_ends_after_call: Vec<(Bx::Value, Size)> = Vec::new(); 'make_args: for (i, arg) in first_args.iter().enumerate() { + if kind == CallKind::Tail && matches!(fn_abi.args[i].mode, PassMode::Indirect { .. }) { + // FIXME: https://github.com/rust-lang/rust/pull/144232#discussion_r2218543841 + span_bug!( + fn_span, + "arguments using PassMode::Indirect are currently not supported for tail calls" + ); + } + let mut op = self.codegen_operand(bx, &arg.node); if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, instance.map(|i| i.def)) { @@ -1147,6 +1215,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { unwind, &lifetime_ends_after_call, instance, + kind, mergeable_succ, ) } @@ -1252,6 +1321,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { for statement in &data.statements { self.codegen_statement(bx, statement); } + self.codegen_stmt_debuginfos(bx, &data.after_last_stmt_debuginfos); let merging_succ = self.codegen_terminator(bx, bb, data.terminator()); if let MergingSucc::False = merging_succ { @@ -1388,15 +1458,23 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { target, unwind, fn_span, + CallKind::Normal, mergeable_succ(), ), - mir::TerminatorKind::TailCall { .. } => { - // FIXME(explicit_tail_calls): implement tail calls in ssa backend - span_bug!( - terminator.source_info.span, - "`TailCall` terminator is not yet supported by `rustc_codegen_ssa`" - ) - } + mir::TerminatorKind::TailCall { ref func, ref args, fn_span } => self + .codegen_call_terminator( + helper, + bx, + terminator, + func, + args, + mir::Place::from(mir::RETURN_PLACE), + None, + mir::UnwindAction::Unreachable, + fn_span, + CallKind::Tail, + mergeable_succ(), + ), mir::TerminatorKind::CoroutineDrop | mir::TerminatorKind::Yield { .. } => { bug!("coroutine ops in codegen") } @@ -1550,6 +1628,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { align, bx.const_usize(copy_bytes), MemFlags::empty(), + None, ); // ...and then load it with the ABI type. llval = load_cast(bx, cast, llscratch, scratch_align); diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index b8f635ab7816..0c9acf087f9f 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -253,6 +253,53 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { spill_slot } + // Indicates that local is set to a new value. The `layout` and `projection` are used to + // calculate the offset. + pub(crate) fn debug_new_val_to_local( + &self, + bx: &mut Bx, + local: mir::Local, + base: PlaceRef<'tcx, Bx::Value>, + projection: &[mir::PlaceElem<'tcx>], + ) { + let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full; + if !full_debug_info { + return; + } + + let vars = match &self.per_local_var_debug_info { + Some(per_local) => &per_local[local], + None => return, + }; + + let DebugInfoOffset { direct_offset, indirect_offsets, result: _ } = + calculate_debuginfo_offset(bx, projection, base.layout); + for var in vars.iter() { + let Some(dbg_var) = var.dbg_var else { + continue; + }; + let Some(dbg_loc) = self.dbg_loc(var.source_info) else { + continue; + }; + bx.dbg_var_value( + dbg_var, + dbg_loc, + base.val.llval, + direct_offset, + &indirect_offsets, + &var.fragment, + ); + } + } + + pub(crate) fn debug_poison_to_local(&self, bx: &mut Bx, local: mir::Local) { + let ty = self.monomorphize(self.mir.local_decls[local].ty); + let layout = bx.cx().layout_of(ty); + let to_backend_ty = bx.cx().immediate_backend_type(layout); + let place_ref = PlaceRef::new_sized(bx.cx().const_poison(to_backend_ty), layout); + self.debug_new_val_to_local(bx, local, place_ref, &[]); + } + /// Apply debuginfo and/or name, after creating the `alloca` for a local, /// or initializing the local with an operand (whichever applies). pub(crate) fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) { @@ -424,7 +471,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { alloca.val.llval, Size::ZERO, &[Size::ZERO], - var.fragment, + &var.fragment, ); } else { bx.dbg_var_addr( @@ -433,7 +480,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { base.val.llval, direct_offset, &indirect_offsets, - var.fragment, + &var.fragment, ); } } @@ -455,7 +502,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let base = FunctionCx::spill_operand_to_stack(operand, Some(name), bx); bx.clear_dbg_loc(); - bx.dbg_var_addr(dbg_var, dbg_loc, base.val.llval, Size::ZERO, &[], fragment); + bx.dbg_var_addr(dbg_var, dbg_loc, base.val.llval, Size::ZERO, &[], &fragment); } } } diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index fc95f62b4a43..cc3316c7f8cc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -30,7 +30,7 @@ fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if allow_overlap { bx.memmove(dst, align, src, align, size, flags); } else { - bx.memcpy(dst, align, src, align, size, flags); + bx.memcpy(dst, align, src, align, size, flags, None); } } @@ -92,6 +92,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let invalid_monomorphization_int_type = |ty| { bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); }; + let invalid_monomorphization_int_or_ptr_type = |ty| { + bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerOrPtrType { + span, + name, + ty, + }); + }; let parse_atomic_ordering = |ord: ty::Value<'tcx>| { let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf(); @@ -113,8 +120,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | sym::atomic_singlethreadfence | sym::caller_location => {} _ => { - span_bug!(span, "nullary intrinsic {name} must either be in a const block or explicitly opted out because it is inherently a runtime intrinsic -"); + span_bug!( + span, + "Nullary intrinsic {name} must be called in a const block. \ + If you are seeing this message from code outside the standard library, the \ + unstable implementation details of the relevant intrinsic may have changed. \ + Consider using stable APIs instead. \ + If you are adding a new nullary intrinsic that is inherently a runtime \ + intrinsic, update this check." + ); } } } @@ -351,7 +365,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::atomic_load => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let ordering = fn_args.const_at(1).to_value(); @@ -367,7 +381,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { sym::atomic_store => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let ordering = fn_args.const_at(1).to_value(); @@ -377,10 +391,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size); return Ok(()); } + // These are all AtomicRMW ops sym::atomic_cxchg | sym::atomic_cxchgweak => { let ty = fn_args.type_at(0); if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty); return Ok(()); } let succ_ordering = fn_args.const_at(1).to_value(); @@ -407,7 +422,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { return Ok(()); } - // These are all AtomicRMW ops sym::atomic_max | sym::atomic_min => { let atom_op = if name == sym::atomic_max { AtomicRmwBinOp::AtomicMax @@ -420,7 +434,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let ordering = fn_args.const_at(1).to_value(); let ptr = args[0].immediate(); let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ false, + ) } else { invalid_monomorphization_int_type(ty); return Ok(()); @@ -438,21 +458,44 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let ordering = fn_args.const_at(1).to_value(); let ptr = args[0].immediate(); let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ false, + ) } else { invalid_monomorphization_int_type(ty); return Ok(()); } } - sym::atomic_xchg - | sym::atomic_xadd + sym::atomic_xchg => { + let ty = fn_args.type_at(0); + let ordering = fn_args.const_at(1).to_value(); + if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { + let ptr = args[0].immediate(); + let val = args[1].immediate(); + let atomic_op = AtomicRmwBinOp::AtomicXchg; + bx.atomic_rmw( + atomic_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ ty.is_raw_ptr(), + ) + } else { + invalid_monomorphization_int_or_ptr_type(ty); + return Ok(()); + } + } + sym::atomic_xadd | sym::atomic_xsub | sym::atomic_and | sym::atomic_nand | sym::atomic_or | sym::atomic_xor => { let atom_op = match name { - sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg, sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd, sym::atomic_xsub => AtomicRmwBinOp::AtomicSub, sym::atomic_and => AtomicRmwBinOp::AtomicAnd, @@ -462,14 +505,28 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => unreachable!(), }; - let ty = fn_args.type_at(0); - if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() { - let ordering = fn_args.const_at(1).to_value(); - let ptr = args[0].immediate(); - let val = args[1].immediate(); - bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering)) + // The type of the in-memory data. + let ty_mem = fn_args.type_at(0); + // The type of the 2nd operand, given by-value. + let ty_op = fn_args.type_at(1); + + let ordering = fn_args.const_at(2).to_value(); + // We require either both arguments to have the same integer type, or the first to + // be a pointer and the second to be `usize`. + if (int_type_width_signed(ty_mem, bx.tcx()).is_some() && ty_op == ty_mem) + || (ty_mem.is_raw_ptr() && ty_op == bx.tcx().types.usize) + { + let ptr = args[0].immediate(); // of type "pointer to `ty_mem`" + let val = args[1].immediate(); // of type `ty_op` + bx.atomic_rmw( + atom_op, + ptr, + val, + parse_atomic_ordering(ordering), + /* ret_ptr */ ty_mem.is_raw_ptr(), + ) } else { - invalid_monomorphization_int_type(ty); + invalid_monomorphization_int_or_ptr_type(ty_mem); return Ok(()); } } diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index 93f0ab36f2a2..5d586d97641d 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -40,12 +40,12 @@ impl<'tcx, V> Locals<'tcx, V> { impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { pub(super) fn initialize_locals(&mut self, values: Vec>) { assert!(self.locals.values.is_empty()); + self.locals.values = IndexVec::from_raw(values); // FIXME(#115215): After #115025 get's merged this might not be necessary - for (local, value) in values.into_iter().enumerate() { + for (local, value) in self.locals.values.iter_enumerated() { match value { LocalRef::Place(_) | LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => (), LocalRef::Operand(op) => { - let local = mir::Local::from_usize(local); let expected_ty = self.monomorphize(self.mir.local_decls[local].ty); if expected_ty != op.layout.ty { warn!( @@ -56,7 +56,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } } - self.locals.values.push(value); } } diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index 50d0f9107445..1670e2da71fb 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -180,6 +180,9 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let llfn = cx.get_fn(instance); let mut mir = tcx.instance_mir(instance.def); + // Note that the ABI logic has deduced facts about the functions' parameters based on the MIR we + // got here (`deduce_param_attrs`). That means we can *not* apply arbitrary further MIR + // transforms as that may invalidate those deduced facts! let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); @@ -296,10 +299,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // Apply debuginfo to the newly allocated locals. fx.debug_introduce_locals(&mut start_bx, consts_debug_info.unwrap_or_default()); - // If the backend supports coverage, and coverage is enabled for this function, - // do any necessary start-of-function codegen (e.g. locals for MC/DC bitmaps). - start_bx.init_coverage(instance); - // The builders will be created separately for each basic block at `codegen_block`. // So drop the builder of `start_llbb` to avoid having two at the same time. drop(start_bx); @@ -321,6 +320,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } +/// Replace `clone` calls that come from `use` statements with direct copies if possible. // FIXME: Move this function to mir::transform when post-mono MIR passes land. fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( cx: &'a Bx::CodegenCx, @@ -442,6 +442,10 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() { let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty)); + + // Explicitly start the lifetime of the `va_list`, improves LLVM codegen. + bx.lifetime_start(va_list.val.llval, va_list.layout.size); + bx.va_start(va_list.val.llval); return LocalRef::Place(va_list); diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs index 42e435cf0a32..e7239ebfecf6 100644 --- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs +++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs @@ -1,6 +1,6 @@ use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind}; -use rustc_attr_data_structures::InstructionSetAttr; -use rustc_middle::mir::mono::{Linkage, MonoItemData, Visibility}; +use rustc_hir::attrs::{InstructionSetAttr, Linkage}; +use rustc_middle::mir::mono::{MonoItemData, Visibility}; use rustc_middle::mir::{InlineAsmOperand, START_BLOCK}; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::{Instance, Ty, TyCtxt, TypeVisitableExt}; @@ -228,6 +228,9 @@ fn prefix_and_suffix<'tcx>( writeln!(begin, "{asm_name}:").unwrap(); writeln!(end).unwrap(); + // emit a label starting with `func_end` for `cargo asm` and other tooling that might + // pattern match on assembly generated by LLVM. + writeln!(end, ".Lfunc_end_{asm_name}:").unwrap(); writeln!(end, ".size {asm_name}, . - {asm_name}").unwrap(); writeln!(end, ".popsection").unwrap(); if !arch_suffix.is_empty() { @@ -246,6 +249,7 @@ fn prefix_and_suffix<'tcx>( writeln!(begin, "{asm_name}:").unwrap(); writeln!(end).unwrap(); + writeln!(end, ".Lfunc_end_{asm_name}:").unwrap(); writeln!(end, ".popsection").unwrap(); if !arch_suffix.is_empty() { writeln!(end, "{}", arch_suffix).unwrap(); @@ -263,6 +267,7 @@ fn prefix_and_suffix<'tcx>( writeln!(begin, "{asm_name}:").unwrap(); writeln!(end).unwrap(); + writeln!(end, ".Lfunc_end_{asm_name}:").unwrap(); writeln!(end, ".popsection").unwrap(); if !arch_suffix.is_empty() { writeln!(end, "{}", arch_suffix).unwrap(); @@ -287,6 +292,7 @@ fn prefix_and_suffix<'tcx>( writeln!(end).unwrap(); // .size is ignored for function symbols, so we can skip it writeln!(end, "end_function").unwrap(); + writeln!(end, ".Lfunc_end_{asm_name}:").unwrap(); } BinaryFormat::Xcoff => { // the LLVM XCOFFAsmParser is extremely incomplete and does not implement many of the diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 5459f95c1860..88a8e2a844cb 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -71,16 +71,23 @@ pub enum OperandValue { } impl OperandValue { + /// Return the data pointer and optional metadata as backend values + /// if this value can be treat as a pointer. + pub(crate) fn try_pointer_parts(self) -> Option<(V, Option)> { + match self { + OperandValue::Immediate(llptr) => Some((llptr, None)), + OperandValue::Pair(llptr, llextra) => Some((llptr, Some(llextra))), + OperandValue::Ref(_) | OperandValue::ZeroSized => None, + } + } + /// Treat this value as a pointer and return the data pointer and /// optional metadata as backend values. /// /// If you're making a place, use [`Self::deref`] instead. pub(crate) fn pointer_parts(self) -> (V, Option) { - match self { - OperandValue::Immediate(llptr) => (llptr, None), - OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)), - _ => bug!("OperandValue cannot be a pointer: {self:?}"), - } + self.try_pointer_parts() + .unwrap_or_else(|| bug!("OperandValue cannot be a pointer: {self:?}")) } /// Treat this value as a pointer and return the place to which it points. @@ -498,6 +505,35 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64); (is_niche, tagged_discr, 0) } else { + // Thanks to parameter attributes and load metadata, LLVM already knows + // the general valid range of the tag. It's possible, though, for there + // to be an impossible value *in the middle*, which those ranges don't + // communicate, so it's worth an `assume` to let the optimizer know. + // Most importantly, this means when optimizing a variant test like + // `SELECT(is_niche, complex, CONST) == CONST` it's ok to simplify that + // to `!is_niche` because the `complex` part can't possibly match. + // + // This was previously asserted on `tagged_discr` below, where the + // impossible value is more obvious, but that caused an intermediate + // value to become multi-use and thus not optimize, so instead this + // assumes on the original input which is always multi-use. See + // + // + // FIXME: If we ever get range assume operand bundles in LLVM (so we + // don't need the `icmp`s in the instruction stream any more), it + // might be worth moving this back to being on the switch argument + // where it's more obviously applicable. + if niche_variants.contains(&untagged_variant) + && bx.cx().sess().opts.optimize != OptLevel::No + { + let impossible = niche_start + .wrapping_add(u128::from(untagged_variant.as_u32())) + .wrapping_sub(u128::from(niche_variants.start().as_u32())); + let impossible = bx.cx().const_uint_big(tag_llty, impossible); + let ne = bx.icmp(IntPredicate::IntNE, tag, impossible); + bx.assume(ne); + } + // With multiple niched variants we'll have to actually compute // the variant index from the stored tag. // @@ -588,20 +624,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let untagged_variant_const = bx.cx().const_uint(cast_to, u64::from(untagged_variant.as_u32())); - // Thanks to parameter attributes and load metadata, LLVM already knows - // the general valid range of the tag. It's possible, though, for there - // to be an impossible value *in the middle*, which those ranges don't - // communicate, so it's worth an `assume` to let the optimizer know. - // Most importantly, this means when optimizing a variant test like - // `SELECT(is_niche, complex, CONST) == CONST` it's ok to simplify that - // to `!is_niche` because the `complex` part can't possibly match. - if niche_variants.contains(&untagged_variant) - && bx.cx().sess().opts.optimize != OptLevel::No - { - let ne = bx.icmp(IntPredicate::IntNE, tagged_discr, untagged_variant_const); - bx.assume(ne); - } - let discr = bx.select(is_niche, tagged_discr, untagged_variant_const); // In principle we could insert assumes on the possible range of `discr`, but @@ -941,11 +963,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let layout = o.layout.for_variant(bx.cx(), vidx); o = OperandRef { val: o.val, layout } } - mir::PlaceElem::Subtype(subtype_ty) => { - let subtype_ty = self.monomorphize(subtype_ty); - let layout = self.cx.layout_of(subtype_ty); - o = OperandRef { val: o.val, layout } - } _ => return None, } } diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 0090be9fdef0..50f56f913a51 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -347,7 +347,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::ProjectionElem::OpaqueCast(ty) => { bug!("encountered OpaqueCast({ty}) in codegen") } - mir::ProjectionElem::Subtype(ty) => cg_base.project_type(bx, self.monomorphize(ty)), mir::ProjectionElem::UnwrapUnsafeBinder(ty) => { cg_base.project_type(bx, self.monomorphize(ty)) } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 8a67b8d6e5f1..640f7211dc99 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -1,4 +1,5 @@ -use rustc_abi::{self as abi, FIRST_VARIANT}; +use itertools::Itertools as _; +use rustc_abi::{self as abi, BackendRepr, FIRST_VARIANT}; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; @@ -6,9 +7,9 @@ use rustc_middle::{bug, mir, span_bug}; use rustc_session::config::OptLevel; use tracing::{debug, instrument}; +use super::FunctionCx; use super::operand::{OperandRef, OperandRefBuilder, OperandValue}; use super::place::{PlaceRef, PlaceValue, codegen_tag_value}; -use super::{FunctionCx, LocalRef}; use crate::common::{IntPredicate, TypeKind}; use crate::traits::*; use crate::{MemFlags, base}; @@ -24,6 +25,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { match *rvalue { mir::Rvalue::Use(ref operand) => { let cg_operand = self.codegen_operand(bx, operand); + // Crucially, we do *not* use `OperandValue::Ref` for types with + // `BackendRepr::Scalar | BackendRepr::ScalarPair`. This ensures we match the MIR + // semantics regarding when assignment operators allow overlap of LHS and RHS. + if matches!( + cg_operand.layout.backend_repr, + BackendRepr::Scalar(..) | BackendRepr::ScalarPair(..), + ) { + debug_assert!(!matches!(cg_operand.val, OperandValue::Ref(..))); + } // FIXME: consider not copying constants through stack. (Fixable by codegen'ing // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?) cg_operand.val.store(bx, dest); @@ -76,7 +86,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, _ty) => { + mir::Rvalue::Cast( + mir::CastKind::Transmute | mir::CastKind::Subtype, + ref operand, + _ty, + ) => { let src = self.codegen_operand(bx, operand); self.codegen_transmute(bx, src, dest); } @@ -111,14 +125,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let size = bx.const_usize(dest.layout.size.bytes()); // Use llvm.memset.p0i8.* to initialize all same byte arrays - if let Some(int) = bx.cx().const_to_opt_u128(v, false) { - let bytes = &int.to_le_bytes()[..cg_elem.layout.size.bytes_usize()]; - let first = bytes[0]; - if bytes[1..].iter().all(|&b| b == first) { - let fill = bx.cx().const_u8(first); - bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); - return true; - } + if let Some(int) = bx.cx().const_to_opt_u128(v, false) + && let bytes = &int.to_le_bytes()[..cg_elem.layout.size.bytes_usize()] + && let Ok(&byte) = bytes.iter().all_equal_value() + { + let fill = bx.cx().const_u8(byte); + bx.memset(start, fill, size, dest.val.align, MemFlags::empty()); + return true; } // Use llvm.memset.p0i8.* to initialize byte arrays @@ -130,13 +143,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { false }; - match cg_elem.val { - OperandValue::Immediate(v) => { - if try_init_all_same(bx, v) { - return; - } - } - _ => (), + if let OperandValue::Immediate(v) = cg_elem.val + && try_init_all_same(bx, v) + { + return; } let count = self @@ -480,7 +490,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bug!("Unsupported cast of {operand:?} to {cast:?}"); }) } - mir::CastKind::Transmute => { + mir::CastKind::Transmute | mir::CastKind::Subtype => { self.codegen_transmute_operand(bx, operand, cast) } }; @@ -494,9 +504,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_place_to_pointer(bx, place, mk_ref) } - mir::Rvalue::CopyForDeref(place) => { - self.codegen_operand(bx, &mir::Operand::Copy(place)) - } mir::Rvalue::RawPtr(kind, place) => { let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| { Ty::new_ptr(tcx, ty, kind.to_mutbl_lossy()) @@ -504,14 +511,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.codegen_place_to_pointer(bx, place, mk_ptr) } - mir::Rvalue::Len(place) => { - let size = self.evaluate_array_len(bx, place); - OperandRef { - val: OperandValue::Immediate(size), - layout: bx.cx().layout_of(bx.tcx().types.usize), - } - } - mir::Rvalue::BinaryOp(op_with_overflow, box (ref lhs, ref rhs)) if let Some(op) = op_with_overflow.overflowing_to_wrapping() => { @@ -612,16 +611,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let ty = self.monomorphize(ty); let layout = bx.cx().layout_of(ty); let val = match null_op { - mir::NullOp::SizeOf => { - assert!(bx.cx().type_is_sized(ty)); - let val = layout.size.bytes(); - bx.cx().const_usize(val) - } - mir::NullOp::AlignOf => { - assert!(bx.cx().type_is_sized(ty)); - let val = layout.align.abi.bytes(); - bx.cx().const_usize(val) - } mir::NullOp::OffsetOf(fields) => { let val = bx .tcx() @@ -725,39 +714,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } } - mir::Rvalue::ShallowInitBox(ref operand, content_ty) => { - let operand = self.codegen_operand(bx, operand); - let val = operand.immediate(); - - let content_ty = self.monomorphize(content_ty); - let box_layout = bx.cx().layout_of(Ty::new_box(bx.tcx(), content_ty)); - - OperandRef { val: OperandValue::Immediate(val), layout: box_layout } - } mir::Rvalue::WrapUnsafeBinder(ref operand, binder_ty) => { let operand = self.codegen_operand(bx, operand); let binder_ty = self.monomorphize(binder_ty); let layout = bx.cx().layout_of(binder_ty); OperandRef { val: operand.val, layout } } + mir::Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in codegen"), + mir::Rvalue::ShallowInitBox(..) => bug!("`ShallowInitBox` in codegen"), } } - fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value { - // ZST are passed as operands and require special handling - // because codegen_place() panics if Local is operand. - if let Some(index) = place.as_local() - && let LocalRef::Operand(op) = self.locals[index] - && let ty::Array(_, n) = op.layout.ty.kind() - { - let n = n.try_to_target_usize(bx.tcx()).expect("expected monomorphic const in codegen"); - return bx.cx().const_usize(n); - } - // use common size calculation for non zero-sized types - let cg_value = self.codegen_place(bx, place.as_ref()); - cg_value.len(bx.cx()) - } - /// Codegen an `Rvalue::RawPtr` or `Rvalue::Ref` fn codegen_place_to_pointer( &mut self, @@ -895,36 +862,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } mir::BinOp::Cmp => { - use std::cmp::Ordering; assert!(!is_float); - if let Some(value) = bx.three_way_compare(lhs_ty, lhs, rhs) { - return value; - } - let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed); - if bx.cx().tcx().sess.opts.optimize == OptLevel::No { - // FIXME: This actually generates tighter assembly, and is a classic trick - // - // However, as of 2023-11 it optimizes worse in things like derived - // `PartialOrd`, so only use it in debug for now. Once LLVM can handle it - // better (see ), it'll - // be worth trying it in optimized builds as well. - let is_gt = bx.icmp(pred(mir::BinOp::Gt), lhs, rhs); - let gtext = bx.zext(is_gt, bx.type_i8()); - let is_lt = bx.icmp(pred(mir::BinOp::Lt), lhs, rhs); - let ltext = bx.zext(is_lt, bx.type_i8()); - bx.unchecked_ssub(gtext, ltext) - } else { - // These operations are those expected by `tests/codegen-llvm/integer-cmp.rs`, - // from . - let is_lt = bx.icmp(pred(mir::BinOp::Lt), lhs, rhs); - let is_ne = bx.icmp(pred(mir::BinOp::Ne), lhs, rhs); - let ge = bx.select( - is_ne, - bx.cx().const_i8(Ordering::Greater as i8), - bx.cx().const_i8(Ordering::Equal as i8), - ); - bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge) - } + bx.three_way_compare(lhs_ty, lhs, rhs) } mir::BinOp::AddWithOverflow | mir::BinOp::SubWithOverflow @@ -1051,14 +990,14 @@ pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( imm = match (from_scalar.primitive(), to_scalar.primitive()) { (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty), (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty), - (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm), + (Int(..), Pointer(..)) => bx.inttoptr(imm, to_backend_ty), (Pointer(..), Int(..)) => { // FIXME: this exposes the provenance, which shouldn't be necessary. bx.ptrtoint(imm, to_backend_ty) } (Float(_), Pointer(..)) => { let int_imm = bx.bitcast(imm, bx.cx().type_isize()); - bx.ptradd(bx.const_null(bx.type_ptr()), int_imm) + bx.inttoptr(int_imm, to_backend_ty) } (Pointer(..), Float(_)) => { // FIXME: this exposes the provenance, which shouldn't be necessary. diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index f164e0f91237..d0121f764380 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -1,4 +1,4 @@ -use rustc_middle::mir::{self, NonDivergingIntrinsic}; +use rustc_middle::mir::{self, NonDivergingIntrinsic, StmtDebugInfo}; use rustc_middle::span_bug; use tracing::instrument; @@ -8,6 +8,7 @@ use crate::traits::*; impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[instrument(level = "debug", skip(self, bx))] pub(crate) fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) { + self.codegen_stmt_debuginfos(bx, &statement.debuginfos); self.set_debug_loc(bx, statement.source_info); match statement.kind { mir::StatementKind::Assign(box (ref place, ref rvalue)) => { @@ -49,11 +50,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::StatementKind::SetDiscriminant { box ref place, variant_index } => { self.codegen_place(bx, place.as_ref()).codegen_set_discr(bx, variant_index); } - mir::StatementKind::Deinit(..) => { - // For now, don't codegen this to anything. In the future it may be worth - // experimenting with what kind of information we can emit to LLVM without hurting - // perf here - } mir::StatementKind::StorageLive(local) => { if let LocalRef::Place(cg_place) = self.locals[local] { cg_place.storage_live(bx); @@ -90,7 +86,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let align = pointee_layout.align; let dst = dst_val.immediate(); let src = src_val.immediate(); - bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty()); + bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty(), None); } mir::StatementKind::FakeRead(..) | mir::StatementKind::Retag { .. } @@ -101,4 +97,49 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | mir::StatementKind::Nop => {} } } + + pub(crate) fn codegen_stmt_debuginfo(&mut self, bx: &mut Bx, debuginfo: &StmtDebugInfo<'tcx>) { + match debuginfo { + StmtDebugInfo::AssignRef(dest, place) => { + let local_ref = match self.locals[place.local] { + // For an rvalue like `&(_1.1)`, when `BackendRepr` is `BackendRepr::Memory`, we allocate a block of memory to this place. + // The place is an indirect pointer, we can refer to it directly. + LocalRef::Place(place_ref) => Some((place_ref, place.projection.as_slice())), + // For an rvalue like `&((*_1).1)`, we are calculating the address of `_1.1`. + // The deref projection is no-op here. + LocalRef::Operand(operand_ref) if place.is_indirect_first_projection() => { + Some((operand_ref.deref(bx.cx()), &place.projection[1..])) + } + // For an rvalue like `&1`, when `BackendRepr` is `BackendRepr::Scalar`, + // we cannot get the address. + // N.B. `non_ssa_locals` returns that this is an SSA local. + LocalRef::Operand(_) => None, + LocalRef::UnsizedPlace(_) | LocalRef::PendingOperand => None, + } + .filter(|(_, projection)| { + // Drop unsupported projections. + projection.iter().all(|p| p.can_use_in_debuginfo()) + }); + if let Some((base, projection)) = local_ref { + self.debug_new_val_to_local(bx, *dest, base, projection); + } else { + // If the address cannot be calculated, use poison to indicate that the value has been optimized out. + self.debug_poison_to_local(bx, *dest); + } + } + StmtDebugInfo::InvalidAssign(local) => { + self.debug_poison_to_local(bx, *local); + } + } + } + + pub(crate) fn codegen_stmt_debuginfos( + &mut self, + bx: &mut Bx, + debuginfos: &[StmtDebugInfo<'tcx>], + ) { + for debuginfo in debuginfos { + self.codegen_stmt_debuginfo(bx, debuginfo); + } + } } diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs index b9040c330fb1..8f03dc1e6b5e 100644 --- a/compiler/rustc_codegen_ssa/src/mono_item.rs +++ b/compiler/rustc_codegen_ssa/src/mono_item.rs @@ -1,5 +1,6 @@ +use rustc_hir::attrs::Linkage; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility}; +use rustc_middle::mir::mono::{MonoItem, MonoItemData, Visibility}; use rustc_middle::ty::layout::HasTyCtxt; use tracing::debug; diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 577012151e49..e1bd8014d7a2 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -21,7 +21,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( trace!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout); if layout.is_sized() { let size = bx.const_usize(layout.size.bytes()); - let align = bx.const_usize(layout.align.abi.bytes()); + let align = bx.const_usize(layout.align.bytes()); return (size, align); } match t.kind() { @@ -49,7 +49,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // All slice sizes must fit into `isize`, so this multiplication cannot // wrap -- neither signed nor unsigned. bx.unchecked_sumul(info.unwrap(), bx.const_usize(unit.size.bytes())), - bx.const_usize(unit.align.abi.bytes()), + bx.const_usize(unit.align.bytes()), ) } ty::Foreign(_) => { @@ -82,7 +82,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // This function does not return so we can now return whatever we want. let size = bx.const_usize(layout.size.bytes()); - let align = bx.const_usize(layout.align.abi.bytes()); + let align = bx.const_usize(layout.align.bytes()); (size, align) } ty::Adt(..) | ty::Tuple(..) => { @@ -94,7 +94,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let i = layout.fields.count() - 1; let unsized_offset_unadjusted = layout.fields.offset(i).bytes(); - let sized_align = layout.align.abi.bytes(); + let sized_align = layout.align.bytes(); debug!( "DST {} offset of dyn field: {}, statically sized align: {}", t, unsized_offset_unadjusted, sized_align diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs index def4ec13e87b..54584999d61b 100644 --- a/compiler/rustc_codegen_ssa/src/target_features.rs +++ b/compiler/rustc_codegen_ssa/src/target_features.rs @@ -1,9 +1,9 @@ -use rustc_attr_data_structures::InstructionSetAttr; use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; use rustc_data_structures::unord::{UnordMap, UnordSet}; +use rustc_hir::attrs::InstructionSetAttr; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId}; -use rustc_middle::middle::codegen_fn_attrs::TargetFeature; +use rustc_middle::middle::codegen_fn_attrs::{TargetFeature, TargetFeatureKind}; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; use rustc_session::Session; @@ -22,6 +22,7 @@ pub(crate) fn from_target_feature_attr( tcx: TyCtxt<'_>, did: LocalDefId, features: &[(Symbol, Span)], + was_forced: bool, rust_target_features: &UnordMap, target_features: &mut Vec, ) { @@ -88,7 +89,14 @@ pub(crate) fn from_target_feature_attr( } } } - target_features.push(TargetFeature { name, implied: name != feature }) + let kind = if name != feature { + TargetFeatureKind::Implied + } else if was_forced { + TargetFeatureKind::Forced + } else { + TargetFeatureKind::Enabled + }; + target_features.push(TargetFeature { name, kind }) } } } @@ -180,6 +188,7 @@ fn parse_rust_feature_flag<'a>( while let Some(new_feature) = new_features.pop() { if features.insert(new_feature) { if let Some(implied_features) = inverse_implied_features.get(&new_feature) { + #[allow(rustc::potential_query_instability)] new_features.extend(implied_features) } } @@ -197,7 +206,10 @@ fn parse_rust_feature_flag<'a>( /// 2nd component of the return value, respectively). /// /// `target_base_has_feature` should check whether the given feature (a Rust feature name!) is -/// enabled in the "base" target machine, i.e., without applying `-Ctarget-feature`. +/// enabled in the "base" target machine, i.e., without applying `-Ctarget-feature`. Note that LLVM +/// may consider features to be implied that we do not and vice-versa. We want `cfg` to be entirely +/// consistent with Rust feature implications, and thus only consult LLVM to expand the target CPU +/// to target features. /// /// We do not have to worry about RUSTC_SPECIFIC_FEATURES here, those are handled elsewhere. pub fn cfg_target_feature( @@ -211,7 +223,15 @@ pub fn cfg_target_feature( .rust_target_features() .iter() .filter(|(feature, _, _)| target_base_has_feature(feature)) - .map(|(feature, _, _)| Symbol::intern(feature)) + .flat_map(|(base_feature, _, _)| { + // Expand the direct base feature into all transitively-implied features. Note that we + // cannot simply use the `implied` field of the tuple since that only contains + // directly-implied features. + // + // Iteration order is irrelevant because we're collecting into an `UnordSet`. + #[allow(rustc::potential_query_instability)] + sess.target.implied_target_features(base_feature).into_iter().map(|f| Symbol::intern(f)) + }) .collect(); // Add enabled and remove disabled features. diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs index 29ec7eb1da3b..85bff4540814 100644 --- a/compiler/rustc_codegen_ssa/src/traits/backend.rs +++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs @@ -1,7 +1,7 @@ use std::any::Any; use std::hash::Hash; -use rustc_ast::expand::allocator::AllocatorKind; +use rustc_ast::expand::allocator::AllocatorMethod; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::sync::{DynSend, DynSync}; use rustc_metadata::EncodedMetadata; @@ -10,7 +10,7 @@ use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::ty::TyCtxt; use rustc_middle::util::Providers; use rustc_session::Session; -use rustc_session::config::{self, OutputFilenames, PrintRequest}; +use rustc_session::config::{self, CrateType, OutputFilenames, PrintRequest}; use rustc_span::Symbol; use super::CodegenObject; @@ -41,6 +41,8 @@ pub trait CodegenBackend { /// Called before `init` so that all other functions are able to emit translatable diagnostics. fn locale_resource(&self) -> &'static str; + fn name(&self) -> &'static str; + fn init(&self, _sess: &Session) {} fn print(&self, _req: &PrintRequest, _out: &mut String, _sess: &Session) {} @@ -60,6 +62,18 @@ pub trait CodegenBackend { } } + fn supported_crate_types(&self, _sess: &Session) -> Vec { + vec![ + CrateType::Executable, + CrateType::Dylib, + CrateType::Rlib, + CrateType::Staticlib, + CrateType::Cdylib, + CrateType::ProcMacro, + CrateType::Sdylib, + ] + } + fn print_passes(&self) {} fn print_version(&self) {} @@ -96,7 +110,14 @@ pub trait CodegenBackend { metadata: EncodedMetadata, outputs: &OutputFilenames, ) { - link_binary(sess, &ArArchiveBuilderBuilder, codegen_results, metadata, outputs); + link_binary( + sess, + &ArArchiveBuilderBuilder, + codegen_results, + metadata, + outputs, + self.name(), + ); } } @@ -107,8 +128,7 @@ pub trait ExtraBackendMethods: &self, tcx: TyCtxt<'tcx>, module_name: &str, - kind: AllocatorKind, - alloc_error_handler_kind: AllocatorKind, + methods: &[AllocatorMethod], ) -> Self::Module; /// This generates the codegen unit and returns it along with diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 979456a6ba70..60296e36e0c1 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -3,6 +3,7 @@ use std::ops::Deref; use rustc_abi::{Align, Scalar, Size, WrappingRange}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::mir; use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; use rustc_middle::ty::{AtomicOrdering, Instance, Ty}; use rustc_session::config::OptLevel; @@ -405,15 +406,41 @@ pub trait BuilderMethods<'a, 'tcx>: fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value; /// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`. - // FIXME: Move the default implementation from `codegen_scalar_binop` into this method and - // remove the `Option` return once LLVM 20 is the minimum version. fn three_way_compare( &mut self, - _ty: Ty<'tcx>, - _lhs: Self::Value, - _rhs: Self::Value, - ) -> Option { - None + ty: Ty<'tcx>, + lhs: Self::Value, + rhs: Self::Value, + ) -> Self::Value { + // FIXME: This implementation was designed around LLVM's ability to optimize, but `cg_llvm` + // overrides this to just use `@llvm.scmp`/`ucmp` since LLVM 20. This default impl should be + // reevaluated with respect to the remaining backends like cg_gcc, whether they might use + // specialized implementations as well, or continue to use a generic implementation here. + use std::cmp::Ordering; + let pred = |op| crate::base::bin_op_to_icmp_predicate(op, ty.is_signed()); + if self.cx().sess().opts.optimize == OptLevel::No { + // This actually generates tighter assembly, and is a classic trick: + // . + // However, as of 2023-11 it optimized worse in LLVM in things like derived + // `PartialOrd`, so we were only using it in debug. Since LLVM now uses its own + // intrinsics, it may be be worth trying it in optimized builds for other backends. + let is_gt = self.icmp(pred(mir::BinOp::Gt), lhs, rhs); + let gtext = self.zext(is_gt, self.type_i8()); + let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs); + let ltext = self.zext(is_lt, self.type_i8()); + self.unchecked_ssub(gtext, ltext) + } else { + // These operations were better optimized by LLVM, before `@llvm.scmp`/`ucmp` in 20. + // See . + let is_lt = self.icmp(pred(mir::BinOp::Lt), lhs, rhs); + let is_ne = self.icmp(pred(mir::BinOp::Ne), lhs, rhs); + let ge = self.select( + is_ne, + self.cx().const_i8(Ordering::Greater as i8), + self.cx().const_i8(Ordering::Equal as i8), + ); + self.select(is_lt, self.cx().const_i8(Ordering::Less as i8), ge) + } } fn memcpy( @@ -424,6 +451,7 @@ pub trait BuilderMethods<'a, 'tcx>: src_align: Align, size: Self::Value, flags: MemFlags, + tt: Option, ); fn memmove( &mut self, @@ -480,7 +508,7 @@ pub trait BuilderMethods<'a, 'tcx>: temp.val.store_with_flags(self, dst.with_type(layout), flags); } else if !layout.is_zst() { let bytes = self.const_usize(layout.size.bytes()); - self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags); + self.memcpy(dst.llval, dst.align, src.llval, src.align, bytes, flags, None); } } @@ -548,12 +576,15 @@ pub trait BuilderMethods<'a, 'tcx>: failure_order: AtomicOrdering, weak: bool, ) -> (Self::Value, Self::Value); + /// `ret_ptr` indicates whether the return type (which is also the type `dst` points to) + /// is a pointer or the same type as `src`. fn atomic_rmw( &mut self, op: AtomicRmwBinOp, dst: Self::Value, src: Self::Value, order: AtomicOrdering, + ret_ptr: bool, ) -> Self::Value; fn atomic_fence(&mut self, order: AtomicOrdering, scope: SynchronizationScope); fn set_invariant_load(&mut self, load: Self::Value); @@ -595,6 +626,18 @@ pub trait BuilderMethods<'a, 'tcx>: funclet: Option<&Self::Funclet>, instance: Option>, ) -> Self::Value; + + fn tail_call( + &mut self, + llty: Self::Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Value, + args: &[Self::Value], + funclet: Option<&Self::Funclet>, + instance: Option>, + ); + fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value); diff --git a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs index 0b513dac5037..31482a53b6d4 100644 --- a/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/coverageinfo.rs @@ -2,11 +2,6 @@ use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::ty::Instance; pub trait CoverageInfoBuilderMethods<'tcx> { - /// Performs any start-of-function codegen needed for coverage instrumentation. - /// - /// Can be a no-op in backends that don't support coverage instrumentation. - fn init_coverage(&mut self, _instance: Instance<'tcx>) {} - /// Handle the MIR coverage info in a backend-specific way. /// /// This can potentially be a no-op in backends that don't support diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs index b9d4950e0ad3..a4da6c915dee 100644 --- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs @@ -77,7 +77,19 @@ pub trait DebugInfoBuilderMethods: BackendTypes { indirect_offsets: &[Size], // Byte range in the `dbg_var` covered by this fragment, // if this is a fragment of a composite `DIVariable`. - fragment: Option>, + fragment: &Option>, + ); + fn dbg_var_value( + &mut self, + dbg_var: Self::DIVariable, + dbg_loc: Self::DILocation, + value: Self::Value, + direct_offset: Size, + // NB: each offset implies a deref (i.e. they're steps in a pointer chain). + indirect_offsets: &[Size], + // Byte range in the `dbg_var` covered by this fragment, + // if this is a fragment of a composite `DIVariable`. + fragment: &Option>, ); fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation); fn clear_dbg_loc(&mut self); diff --git a/compiler/rustc_codegen_ssa/src/traits/declare.rs b/compiler/rustc_codegen_ssa/src/traits/declare.rs index 9f735546558b..8d5f0a5b939a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/declare.rs +++ b/compiler/rustc_codegen_ssa/src/traits/declare.rs @@ -1,5 +1,6 @@ +use rustc_hir::attrs::Linkage; use rustc_hir::def_id::DefId; -use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::Instance; pub trait PreDefineCodegenMethods<'tcx> { diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs index f391c198e1a1..1ac1d7ef2e2e 100644 --- a/compiler/rustc_codegen_ssa/src/traits/write.rs +++ b/compiler/rustc_codegen_ssa/src/traits/write.rs @@ -1,7 +1,6 @@ use std::path::PathBuf; -use rustc_ast::expand::autodiff_attrs::AutoDiffItem; -use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_errors::DiagCtxtHandle; use rustc_middle::dep_graph::WorkProduct; use crate::back::lto::{SerializedModule, ThinModule}; @@ -23,8 +22,7 @@ pub trait WriteBackendMethods: Clone + 'static { exported_symbols_for_lto: &[String], each_linked_rlib_for_lto: &[PathBuf], modules: Vec>, - diff_fncs: Vec, - ) -> Result, FatalError>; + ) -> ModuleCodegen; /// Performs thin LTO by performing necessary global analysis and returning two /// lists, one of the modules that need optimization and another for modules that /// can simply be copied over from the incr. comp. cache. @@ -34,7 +32,7 @@ pub trait WriteBackendMethods: Clone + 'static { each_linked_rlib_for_lto: &[PathBuf], modules: Vec<(String, Self::ThinBuffer)>, cached_modules: Vec<(SerializedModule, WorkProduct)>, - ) -> Result<(Vec>, Vec), FatalError>; + ) -> (Vec>, Vec); fn print_pass_timings(&self); fn print_statistics(&self); fn optimize( @@ -42,26 +40,22 @@ pub trait WriteBackendMethods: Clone + 'static { dcx: DiagCtxtHandle<'_>, module: &mut ModuleCodegen, config: &ModuleConfig, - ) -> Result<(), FatalError>; + ); fn optimize_thin( cgcx: &CodegenContext, thin: ThinModule, - ) -> Result, FatalError>; + ) -> ModuleCodegen; fn codegen( cgcx: &CodegenContext, module: ModuleCodegen, config: &ModuleConfig, - ) -> Result; - fn prepare_thin( - module: ModuleCodegen, - want_summary: bool, - ) -> (String, Self::ThinBuffer); + ) -> CompiledModule; + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer); fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer); } pub trait ThinBufferMethods: Send + Sync { fn data(&self) -> &[u8]; - fn thin_link_data(&self) -> &[u8]; } pub trait ModuleBufferMethods: Send + Sync { diff --git a/compiler/rustc_const_eval/Cargo.toml b/compiler/rustc_const_eval/Cargo.toml index 93d0d5b9a71e..51dcee8d8822 100644 --- a/compiler/rustc_const_eval/Cargo.toml +++ b/compiler/rustc_const_eval/Cargo.toml @@ -9,7 +9,6 @@ either = "1" rustc_abi = { path = "../rustc_abi" } rustc_apfloat = "0.2.0" rustc_ast = { path = "../rustc_ast" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl index 2985eafb63af..3a2b3f884319 100644 --- a/compiler/rustc_const_eval/messages.ftl +++ b/compiler/rustc_const_eval/messages.ftl @@ -57,7 +57,7 @@ const_eval_const_context = {$kind -> } const_eval_const_heap_ptr_in_final = encountered `const_allocate` pointer in final value that was not made global - .note = use `const_make_global` to make allocated pointers immutable before returning + .note = use `const_make_global` to turn allocated pointers into immutable globals before returning const_eval_const_make_global_ptr_already_made_global = attempting to call `const_make_global` twice on the same allocation {$alloc} @@ -299,10 +299,11 @@ const_eval_panic = evaluation panicked: {$msg} const_eval_panic_non_str = argument to `panic!()` in a const context must have type `&str` -const_eval_partial_pointer_copy = - unable to copy parts of a pointer from memory at {$ptr} -const_eval_partial_pointer_overwrite = - unable to overwrite parts of a pointer in memory at {$ptr} +const_eval_partial_pointer_in_final = encountered partial pointer in final value of {const_eval_intern_kind} + .note = while pointers can be broken apart into individual bytes during const-evaluation, only complete pointers (with all their bytes in the right order) are supported in the final value + +const_eval_partial_pointer_read = + unable to read parts of a pointer from memory at {$ptr} const_eval_pointer_arithmetic_overflow = overflowing pointer arithmetic: the total offset in bytes does not fit in an `isize` @@ -456,7 +457,7 @@ const_eval_validation_failure = it is undefined behavior to use this value const_eval_validation_failure_note = - The rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. + the rules on what exactly is undefined behavior aren't clear, so this check might be overzealous. Please open an issue on the rustc repository if you believe it should not be considered undefined behavior. const_eval_validation_front_matter_invalid_value = constructing invalid value const_eval_validation_front_matter_invalid_value_with_path = constructing invalid value at {$path} @@ -475,14 +476,23 @@ const_eval_validation_invalid_vtable_trait = {$front_matter}: wrong trait in wid const_eval_validation_mutable_ref_in_const = {$front_matter}: encountered mutable reference in `const` value const_eval_validation_mutable_ref_to_immutable = {$front_matter}: encountered mutable reference or box pointing to read-only memory const_eval_validation_never_val = {$front_matter}: encountered a value of the never type `!` -const_eval_validation_null_box = {$front_matter}: encountered a null box -const_eval_validation_null_fn_ptr = {$front_matter}: encountered a null function pointer -const_eval_validation_null_ref = {$front_matter}: encountered a null reference -const_eval_validation_nullable_ptr_out_of_range = {$front_matter}: encountered a potentially null pointer, but expected something that cannot possibly fail to be {$in_range} +const_eval_validation_nonnull_ptr_out_of_range = {$front_matter}: encountered a maybe-null pointer, but expected something that is definitely non-zero +const_eval_validation_null_box = {$front_matter}: encountered a {$maybe -> + [true] maybe-null + *[false] null + } box +const_eval_validation_null_fn_ptr = {$front_matter}: encountered a {$maybe -> + [true] maybe-null + *[false] null + } function pointer +const_eval_validation_null_ref = {$front_matter}: encountered a {$maybe -> + [true] maybe-null + *[false] null + } reference const_eval_validation_out_of_range = {$front_matter}: encountered {$value}, but expected something {$in_range} const_eval_validation_partial_pointer = {$front_matter}: encountered a partial pointer or a mix of pointers const_eval_validation_pointer_as_int = {$front_matter}: encountered a pointer, but {$expected} -const_eval_validation_ptr_out_of_range = {$front_matter}: encountered a pointer, but expected something that cannot possibly fail to be {$in_range} +const_eval_validation_ptr_out_of_range = {$front_matter}: encountered a pointer with unknown absolute address, but expected something that is definitely {$in_range} const_eval_validation_ref_to_uninhabited = {$front_matter}: encountered a reference pointing to uninhabited type {$ty} const_eval_validation_unaligned_box = {$front_matter}: encountered an unaligned box (required {$required_bytes} byte alignment but found {$found_bytes}) const_eval_validation_unaligned_ref = {$front_matter}: encountered an unaligned reference (required {$required_bytes} byte alignment but found {$found_bytes}) diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 44e5d1d5ee7e..ca173fe26c2f 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -6,7 +6,6 @@ use std::mem; use std::num::NonZero; use std::ops::Deref; -use rustc_attr_data_structures as attrs; use rustc_errors::{Diag, ErrorGuaranteed}; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; @@ -416,7 +415,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { ) })); - let errors = ocx.select_all_or_error(); + let errors = ocx.evaluate_obligations_error_on_ambiguity(); if errors.is_empty() { Some(ConstConditionsHold::Yes) } else { @@ -466,7 +465,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { /// Check the const stability of the given item (fn or trait). fn check_callee_stability(&mut self, def_id: DefId) { match self.tcx.lookup_const_stability(def_id) { - Some(attrs::ConstStability { level: attrs::StabilityLevel::Stable { .. }, .. }) => { + Some(hir::ConstStability { level: hir::StabilityLevel::Stable { .. }, .. }) => { // All good. } None => { @@ -482,8 +481,8 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { }); } } - Some(attrs::ConstStability { - level: attrs::StabilityLevel::Unstable { implied_by: implied_feature, issue, .. }, + Some(hir::ConstStability { + level: hir::StabilityLevel::Unstable { implied_by: implied_feature, issue, .. }, feature, .. }) => { @@ -574,8 +573,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { Rvalue::Use(_) | Rvalue::CopyForDeref(..) | Rvalue::Repeat(..) - | Rvalue::Discriminant(..) - | Rvalue::Len(_) => {} + | Rvalue::Discriminant(..) => {} Rvalue::Aggregate(kind, ..) => { if let AggregateKind::Coroutine(def_id, ..) = kind.as_ref() @@ -648,11 +646,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { Rvalue::Cast(_, _, _) => {} Rvalue::NullaryOp( - NullOp::SizeOf - | NullOp::AlignOf - | NullOp::OffsetOf(_) - | NullOp::UbChecks - | NullOp::ContractChecks, + NullOp::OffsetOf(_) | NullOp::UbChecks | NullOp::ContractChecks, _, ) => {} Rvalue::ShallowInitBox(_, _) => {} @@ -734,7 +728,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { match statement.kind { StatementKind::Assign(..) | StatementKind::SetDiscriminant { .. } - | StatementKind::Deinit(..) | StatementKind::FakeRead(..) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) @@ -828,7 +821,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { // At this point, we are calling a function, `callee`, whose `DefId` is known... - // `begin_panic` and `#[rustc_const_panic_str]` functions accept generic + // `begin_panic` and `panic_display` functions accept generic // types other than str. Check to enforce that only str can be used in // const-eval. @@ -842,8 +835,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { return; } - // const-eval of `#[rustc_const_panic_str]` functions assumes the argument is `&&str` - if tcx.has_attr(callee, sym::rustc_const_panic_str) { + // const-eval of `panic_display` assumes the argument is `&&str` + if tcx.is_lang_item(callee, LangItem::PanicDisplay) { match args[0].node.ty(&self.ccx.body.local_decls, tcx).kind() { ty::Ref(_, ty, _) if matches!(ty.kind(), ty::Ref(_, ty, _) if ty.is_str()) => {} @@ -891,8 +884,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { }); } } - Some(attrs::ConstStability { - level: attrs::StabilityLevel::Unstable { .. }, + Some(hir::ConstStability { + level: hir::StabilityLevel::Unstable { .. }, feature, .. }) => { @@ -902,8 +895,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { const_stable_indirect: is_const_stable, }); } - Some(attrs::ConstStability { - level: attrs::StabilityLevel::Stable { .. }, + Some(hir::ConstStability { + level: hir::StabilityLevel::Stable { .. }, .. }) => { // All good. Note that a `#[rustc_const_stable]` intrinsic (meaning it diff --git a/compiler/rustc_const_eval/src/check_consts/mod.rs b/compiler/rustc_const_eval/src/check_consts/mod.rs index ebf18c6f2ac8..4a88d039ef3c 100644 --- a/compiler/rustc_const_eval/src/check_consts/mod.rs +++ b/compiler/rustc_const_eval/src/check_consts/mod.rs @@ -5,11 +5,12 @@ //! it finds operations that are invalid in a certain context. use rustc_errors::DiagCtxtHandle; +use rustc_hir::attrs::AttributeKind; use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_hir::{self as hir, find_attr}; use rustc_middle::ty::{self, PolyFnSig, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_span::Symbol; -use {rustc_attr_data_structures as attrs, rustc_hir as hir}; pub use self::qualifs::Qualif; @@ -82,7 +83,7 @@ pub fn rustc_allow_const_fn_unstable( ) -> bool { let attrs = tcx.hir_attrs(tcx.local_def_id_to_hir_id(def_id)); - attrs::find_attr!(attrs, attrs::AttributeKind::AllowConstFnUnstable(syms, _) if syms.contains(&feature_gate)) + find_attr!(attrs, AttributeKind::AllowConstFnUnstable(syms, _) if syms.contains(&feature_gate)) } /// Returns `true` if the given `def_id` (trait or function) is "safe to expose on stable". diff --git a/compiler/rustc_const_eval/src/check_consts/ops.rs b/compiler/rustc_const_eval/src/check_consts/ops.rs index 982e640fa92a..79e32dcf105c 100644 --- a/compiler/rustc_const_eval/src/check_consts/ops.rs +++ b/compiler/rustc_const_eval/src/check_consts/ops.rs @@ -142,7 +142,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> { |err, self_ty, trait_id| { // FIXME(const_trait_impl): Do we need any of this on the non-const codepath? - let trait_ref = TraitRef::from_method(tcx, trait_id, self.args); + let trait_ref = TraitRef::from_assoc(tcx, trait_id, self.args); match self_ty.kind() { Param(param_ty) => { diff --git a/compiler/rustc_const_eval/src/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/check_consts/qualifs.rs index faf41f1658b7..5b65f1726f6c 100644 --- a/compiler/rustc_const_eval/src/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/check_consts/qualifs.rs @@ -119,7 +119,7 @@ impl Qualif for HasMutInterior { ty::TraitRef::new(cx.tcx, freeze_def_id, [ty::GenericArg::from(ty)]), ); ocx.register_obligation(obligation); - let errors = ocx.select_all_or_error(); + let errors = ocx.evaluate_obligations_error_on_ambiguity(); !errors.is_empty() } @@ -197,7 +197,7 @@ impl Qualif for NeedsNonConstDrop { }, ), )); - !ocx.select_all_or_error().is_empty() + !ocx.evaluate_obligations_error_on_ambiguity().is_empty() } fn is_structural_in_adt_value<'tcx>(cx: &ConstCx<'_, 'tcx>, adt: AdtDef<'tcx>) -> bool { @@ -232,9 +232,7 @@ where Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx)) } - Rvalue::Discriminant(place) | Rvalue::Len(place) => { - in_place::(cx, in_local, place.as_ref()) - } + Rvalue::Discriminant(place) => in_place::(cx, in_local, place.as_ref()), Rvalue::CopyForDeref(place) => in_place::(cx, in_local, place.as_ref()), @@ -295,7 +293,6 @@ where ProjectionElem::Index(index) if in_local(index) => return true, ProjectionElem::Deref - | ProjectionElem::Subtype(_) | ProjectionElem::Field(_, _) | ProjectionElem::OpaqueCast(_) | ProjectionElem::ConstantIndex { .. } diff --git a/compiler/rustc_const_eval/src/check_consts/resolver.rs b/compiler/rustc_const_eval/src/check_consts/resolver.rs index d98e5027e4d6..e6e3948305af 100644 --- a/compiler/rustc_const_eval/src/check_consts/resolver.rs +++ b/compiler/rustc_const_eval/src/check_consts/resolver.rs @@ -197,7 +197,6 @@ where | mir::Rvalue::CopyForDeref(..) | mir::Rvalue::ThreadLocalRef(..) | mir::Rvalue::Repeat(..) - | mir::Rvalue::Len(..) | mir::Rvalue::BinaryOp(..) | mir::Rvalue::NullaryOp(..) | mir::Rvalue::UnaryOp(..) @@ -333,7 +332,7 @@ where } fn apply_primary_statement_effect( - &mut self, + &self, state: &mut Self::Domain, statement: &mir::Statement<'tcx>, location: Location, @@ -342,7 +341,7 @@ where } fn apply_primary_terminator_effect<'mir>( - &mut self, + &self, state: &mut Self::Domain, terminator: &'mir mir::Terminator<'tcx>, location: Location, @@ -352,7 +351,7 @@ where } fn apply_call_return_effect( - &mut self, + &self, state: &mut Self::Domain, block: BasicBlock, return_places: CallReturnPlaces<'_, 'tcx>, diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs index e00fb2c1eaf9..69a8f163ca93 100644 --- a/compiler/rustc_const_eval/src/const_eval/error.rs +++ b/compiler/rustc_const_eval/src/const_eval/error.rs @@ -110,7 +110,7 @@ pub fn get_span_and_frames<'tcx>( if frame.times < 3 { let times = frame.times; frame.times = 0; - frames.extend(std::iter::repeat(frame).take(times as usize)); + frames.extend(std::iter::repeat_n(frame, times as usize)); } else { frames.push(frame); } diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs index 5835660e1c38..beea1b4a28c9 100644 --- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs @@ -7,7 +7,7 @@ use rustc_hir::def::DefKind; use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo}; use rustc_middle::mir::{self, ConstAlloc, ConstValue}; use rustc_middle::query::TyCtxtAt; -use rustc_middle::ty::layout::HasTypingEnv; +use rustc_middle::ty::layout::{HasTypingEnv, TyAndLayout}; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, throw_inval}; @@ -24,13 +24,11 @@ use crate::interpret::{ }; use crate::{CTRL_C_RECEIVED, errors}; -// Returns a pointer to where the result lives -#[instrument(level = "trace", skip(ecx, body))] -fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( +fn setup_for_eval<'tcx>( ecx: &mut CompileTimeInterpCx<'tcx>, cid: GlobalId<'tcx>, - body: &'tcx mir::Body<'tcx>, -) -> InterpResult<'tcx, R> { + layout: TyAndLayout<'tcx>, +) -> InterpResult<'tcx, (InternKind, MPlaceTy<'tcx>)> { let tcx = *ecx.tcx; assert!( cid.promoted.is_some() @@ -46,7 +44,6 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( "Unexpected DefKind: {:?}", ecx.tcx.def_kind(cid.instance.def_id()) ); - let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?; assert!(layout.is_sized()); let intern_kind = if cid.promoted.is_some() { @@ -58,12 +55,25 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( } }; - let ret = if let InternKind::Static(_) = intern_kind { - create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)? + let return_place = if let InternKind::Static(_) = intern_kind { + create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout) } else { - ecx.allocate(layout, MemoryKind::Stack)? + ecx.allocate(layout, MemoryKind::Stack) }; + return_place.map(|ret| (intern_kind, ret)) +} + +#[instrument(level = "trace", skip(ecx, body))] +fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( + ecx: &mut CompileTimeInterpCx<'tcx>, + cid: GlobalId<'tcx>, + body: &'tcx mir::Body<'tcx>, +) -> InterpResult<'tcx, R> { + let tcx = *ecx.tcx; + let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?; + let (intern_kind, ret) = setup_for_eval(ecx, cid, layout)?; + trace!( "eval_body_using_ecx: pushing stack frame for global: {}{}", with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())), @@ -87,6 +97,31 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( } } + intern_and_validate(ecx, cid, intern_kind, ret) +} + +#[instrument(level = "trace", skip(ecx))] +fn eval_trivial_const_using_ecx<'tcx, R: InterpretationResult<'tcx>>( + ecx: &mut CompileTimeInterpCx<'tcx>, + cid: GlobalId<'tcx>, + val: ConstValue, + ty: Ty<'tcx>, +) -> InterpResult<'tcx, R> { + let layout = ecx.layout_of(ty)?; + let (intern_kind, return_place) = setup_for_eval(ecx, cid, layout)?; + + let opty = ecx.const_val_to_op(val, ty, Some(layout))?; + ecx.copy_op(&opty, &return_place)?; + + intern_and_validate(ecx, cid, intern_kind, return_place) +} + +fn intern_and_validate<'tcx, R: InterpretationResult<'tcx>>( + ecx: &mut CompileTimeInterpCx<'tcx>, + cid: GlobalId<'tcx>, + intern_kind: InternKind, + ret: MPlaceTy<'tcx>, +) -> InterpResult<'tcx, R> { // Intern the result let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret); @@ -117,6 +152,13 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>( ecx.tcx.dcx().emit_err(errors::ConstHeapPtrInFinal { span: ecx.tcx.span }), ))); } + Err(InternError::PartialPointer) => { + throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error( + ecx.tcx + .dcx() + .emit_err(errors::PartialPtrInFinal { span: ecx.tcx.span, kind: intern_kind }), + ))); + } } interp_ok(R::make_result(ret, ecx)) @@ -285,6 +327,9 @@ pub fn eval_to_const_value_raw_provider<'tcx>( tcx: TyCtxt<'tcx>, key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>, ) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> { + if let Some((value, _ty)) = tcx.trivial_const(key.value.instance.def_id()) { + return Ok(value); + } tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key)) } @@ -361,10 +406,14 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>( // so we have to reject reading mutable global memory. CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error), ); - let res = ecx.load_mir(cid.instance.def, cid.promoted); - res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body)) - .report_err() - .map_err(|error| report_eval_error(&ecx, cid, error)) + + let result = if let Some((value, ty)) = tcx.trivial_const(def) { + eval_trivial_const_using_ecx(&mut ecx, cid, value, ty) + } else { + ecx.load_mir(cid.instance.def, cid.promoted) + .and_then(|body| eval_body_using_ecx(&mut ecx, cid, body)) + }; + result.report_err().map_err(|error| report_eval_error(&ecx, cid, error)) } #[inline(always)] @@ -407,8 +456,6 @@ fn report_eval_error<'tcx>( let (error, backtrace) = error.into_parts(); backtrace.print_backtrace(); - let instance = with_no_trimmed_paths!(cid.instance.to_string()); - super::report( ecx, error, @@ -423,7 +470,7 @@ fn report_eval_error<'tcx>( diag.subdiagnostic(frame); } // Add after the frame rendering above, as it adds its own `instance` args. - diag.arg("instance", instance); + diag.arg("instance", with_no_trimmed_paths!(cid.instance.to_string())); diag.arg("num_frames", num_frames); }, ) diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs index 35c3e3ed3150..530d8d4b1fa6 100644 --- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs +++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs @@ -7,7 +7,7 @@ use rustc_middle::ty::TyCtxt; fn parent_impl_or_trait_constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness { let parent_id = tcx.local_parent(def_id); match tcx.def_kind(parent_id) { - DefKind::Impl { of_trait: true } => tcx.impl_trait_header(parent_id).unwrap().constness, + DefKind::Impl { of_trait: true } => tcx.impl_trait_header(parent_id).constness, DefKind::Trait => { if tcx.is_const_trait(parent_id.into()) { hir::Constness::Const diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index a18ae79f318d..fccb6b171b1c 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -237,7 +237,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> { ) -> InterpResult<'tcx, Option>> { let def_id = instance.def_id(); - if self.tcx.has_attr(def_id, sym::rustc_const_panic_str) + if self.tcx.is_lang_item(def_id, LangItem::PanicDisplay) || self.tcx.is_lang_item(def_id, LangItem::BeginPanic) { let args = self.copy_fn_args(args); @@ -280,22 +280,110 @@ impl<'tcx> CompileTimeInterpCx<'tcx> { interp_ok(match (a, b) { // Comparisons between integers are always known. (Scalar::Int(a), Scalar::Int(b)) => (a == b) as u8, - // Comparisons of null with an arbitrary scalar can be known if `scalar_may_be_null` - // indicates that the scalar can definitely *not* be null. - (Scalar::Int(int), ptr) | (ptr, Scalar::Int(int)) - if int.is_null() && !self.scalar_may_be_null(ptr)? => - { - 0 + // Comparing a pointer `ptr` with an integer `int` is equivalent to comparing + // `ptr-int` with null, so we can reduce this case to a `scalar_may_be_null` test. + (Scalar::Int(int), Scalar::Ptr(ptr, _)) | (Scalar::Ptr(ptr, _), Scalar::Int(int)) => { + let int = int.to_target_usize(*self.tcx); + // The `wrapping_neg` here may produce a value that is not + // a valid target usize any more... but `wrapping_offset` handles that correctly. + let offset_ptr = ptr.wrapping_offset(Size::from_bytes(int.wrapping_neg()), self); + if !self.scalar_may_be_null(Scalar::from_pointer(offset_ptr, self))? { + // `ptr.wrapping_sub(int)` is definitely not equal to `0`, so `ptr != int` + 0 + } else { + // `ptr.wrapping_sub(int)` could be equal to `0`, but might not be, + // so we cannot know for sure if `ptr == int` or not + 2 + } + } + (Scalar::Ptr(a, _), Scalar::Ptr(b, _)) => { + let (a_prov, a_offset) = a.prov_and_relative_offset(); + let (b_prov, b_offset) = b.prov_and_relative_offset(); + let a_allocid = a_prov.alloc_id(); + let b_allocid = b_prov.alloc_id(); + let a_info = self.get_alloc_info(a_allocid); + let b_info = self.get_alloc_info(b_allocid); + + // Check if the pointers cannot be equal due to alignment + if a_info.align > Align::ONE && b_info.align > Align::ONE { + let min_align = Ord::min(a_info.align.bytes(), b_info.align.bytes()); + let a_residue = a_offset.bytes() % min_align; + let b_residue = b_offset.bytes() % min_align; + if a_residue != b_residue { + // If the two pointers have a different residue modulo their + // common alignment, they cannot be equal. + return interp_ok(0); + } + // The pointers have the same residue modulo their common alignment, + // so they could be equal. Try the other checks. + } + + if let (Some(GlobalAlloc::Static(a_did)), Some(GlobalAlloc::Static(b_did))) = ( + self.tcx.try_get_global_alloc(a_allocid), + self.tcx.try_get_global_alloc(b_allocid), + ) { + if a_allocid == b_allocid { + debug_assert_eq!( + a_did, b_did, + "different static item DefIds had same AllocId? {a_allocid:?} == {b_allocid:?}, {a_did:?} != {b_did:?}" + ); + // Comparing two pointers into the same static. As per + // https://doc.rust-lang.org/nightly/reference/items/static-items.html#r-items.static.intro + // a static cannot be duplicated, so if two pointers are into the same + // static, they are equal if and only if their offsets are equal. + (a_offset == b_offset) as u8 + } else { + debug_assert_ne!( + a_did, b_did, + "same static item DefId had two different AllocIds? {a_allocid:?} != {b_allocid:?}, {a_did:?} == {b_did:?}" + ); + // Comparing two pointers into the different statics. + // We can never determine for sure that two pointers into different statics + // are *equal*, but we can know that they are *inequal* if they are both + // strictly in-bounds (i.e. in-bounds and not one-past-the-end) of + // their respective static, as different non-zero-sized statics cannot + // overlap or be deduplicated as per + // https://doc.rust-lang.org/nightly/reference/items/static-items.html#r-items.static.intro + // (non-deduplication), and + // https://doc.rust-lang.org/nightly/reference/items/static-items.html#r-items.static.storage-disjointness + // (non-overlapping). + if a_offset < a_info.size && b_offset < b_info.size { + 0 + } else { + // Otherwise, conservatively say we don't know. + // There are some cases we could still return `0` for, e.g. + // if the pointers being equal would require their statics to overlap + // one or more bytes, but for simplicity we currently only check + // strictly in-bounds pointers. + 2 + } + } + } else { + // All other cases we conservatively say we don't know. + // + // For comparing statics to non-statics, as per https://doc.rust-lang.org/nightly/reference/items/static-items.html#r-items.static.storage-disjointness + // immutable statics can overlap with other kinds of allocations sometimes. + // + // FIXME: We could be more decisive for (non-zero-sized) mutable statics, + // which cannot overlap with other kinds of allocations. + // + // Functions and vtables can be duplicated and deduplicated, so we + // cannot be sure of runtime equality of pointers to the same one, or the + // runtime inequality of pointers to different ones (see e.g. #73722), + // so comparing those should return 2, whether they are the same allocation + // or not. + // + // `GlobalAlloc::TypeId` exists mostly to prevent consteval from comparing + // `TypeId`s, so comparing those should always return 2, whether they are the + // same allocation or not. + // + // FIXME: We could revisit comparing pointers into the same + // `GlobalAlloc::Memory` once https://github.com/rust-lang/rust/issues/128775 + // is fixed (but they can be deduplicated, so comparing pointers into different + // ones should return 2). + 2 + } } - // Other ways of comparing integers and pointers can never be known for sure. - (Scalar::Int { .. }, Scalar::Ptr(..)) | (Scalar::Ptr(..), Scalar::Int { .. }) => 2, - // FIXME: return a `1` for when both sides are the same pointer, *except* that - // some things (like functions and vtables) do not have stable addresses - // so we need to be careful around them (see e.g. #73722). - // FIXME: return `0` for at least some comparisons where we can reliably - // determine the result of runtime inequality tests at compile-time. - // Examples include comparison of addresses in different static items. - (Scalar::Ptr(..), Scalar::Ptr(..)) => 2, }) } } diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index 37c6c4a61d8d..7c41258ebfe5 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -1,6 +1,7 @@ use rustc_abi::{BackendRepr, FieldIdx, VariantIdx}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId, ValTreeCreationError}; +use rustc_middle::traits::ObligationCause; use rustc_middle::ty::layout::{LayoutCx, TyAndLayout}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, mir}; @@ -196,6 +197,7 @@ fn reconstruct_place_meta<'tcx>( // Traverse the type, and update `last_valtree` as we go. let tail = tcx.struct_tail_raw( layout.ty, + &ObligationCause::dummy(), |ty| ty, || { let branches = last_valtree.unwrap_branch(); diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs index a4148cb145ff..a0958a2b9ef3 100644 --- a/compiler/rustc_const_eval/src/errors.rs +++ b/compiler/rustc_const_eval/src/errors.rs @@ -51,6 +51,15 @@ pub(crate) struct ConstHeapPtrInFinal { pub span: Span, } +#[derive(Diagnostic)] +#[diag(const_eval_partial_pointer_in_final)] +#[note] +pub(crate) struct PartialPtrInFinal { + #[primary_span] + pub span: Span, + pub kind: InternKind, +} + #[derive(Diagnostic)] #[diag(const_eval_unstable_in_stable_exposed)] pub(crate) struct UnstableInStableExposed { @@ -657,9 +666,9 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { PartialPointer => const_eval_validation_partial_pointer, MutableRefToImmutable => const_eval_validation_mutable_ref_to_immutable, MutableRefInConst => const_eval_validation_mutable_ref_in_const, - NullFnPtr => const_eval_validation_null_fn_ptr, + NullFnPtr { .. } => const_eval_validation_null_fn_ptr, NeverVal => const_eval_validation_never_val, - NullablePtrOutOfRange { .. } => const_eval_validation_nullable_ptr_out_of_range, + NonnullPtrMaybeNull { .. } => const_eval_validation_nonnull_ptr_out_of_range, PtrOutOfRange { .. } => const_eval_validation_ptr_out_of_range, OutOfRange { .. } => const_eval_validation_out_of_range, UnsafeCellInImmutable => const_eval_validation_unsafe_cell, @@ -687,8 +696,8 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { } UnalignedPtr { ptr_kind: PointerKind::Box, .. } => const_eval_validation_unaligned_box, - NullPtr { ptr_kind: PointerKind::Box } => const_eval_validation_null_box, - NullPtr { ptr_kind: PointerKind::Ref(_) } => const_eval_validation_null_ref, + NullPtr { ptr_kind: PointerKind::Box, .. } => const_eval_validation_null_box, + NullPtr { ptr_kind: PointerKind::Ref(_), .. } => const_eval_validation_null_ref, DanglingPtrNoProvenance { ptr_kind: PointerKind::Box, .. } => { const_eval_validation_dangling_box_no_provenance } @@ -795,9 +804,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { | InvalidFnPtr { value } => { err.arg("value", value); } - NullablePtrOutOfRange { range, max_value } | PtrOutOfRange { range, max_value } => { - add_range_arg(range, max_value, err) - } + PtrOutOfRange { range, max_value } => add_range_arg(range, max_value, err), OutOfRange { range, max_value, value } => { err.arg("value", value); add_range_arg(range, max_value, err); @@ -813,10 +820,12 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> { err.arg("vtable_dyn_type", vtable_dyn_type.to_string()); err.arg("expected_dyn_type", expected_dyn_type.to_string()); } - NullPtr { .. } - | MutableRefToImmutable + NullPtr { maybe, .. } | NullFnPtr { maybe } => { + err.arg("maybe", maybe); + } + MutableRefToImmutable | MutableRefInConst - | NullFnPtr + | NonnullPtrMaybeNull | NeverVal | UnsafeCellInImmutable | InvalidMetaSliceTooLarge { .. } @@ -836,8 +845,7 @@ impl ReportErrorExt for UnsupportedOpInfo { UnsupportedOpInfo::Unsupported(s) => s.clone().into(), UnsupportedOpInfo::ExternTypeField => const_eval_extern_type_field, UnsupportedOpInfo::UnsizedLocal => const_eval_unsized_local, - UnsupportedOpInfo::OverwritePartialPointer(_) => const_eval_partial_pointer_overwrite, - UnsupportedOpInfo::ReadPartialPointer(_) => const_eval_partial_pointer_copy, + UnsupportedOpInfo::ReadPartialPointer(_) => const_eval_partial_pointer_read, UnsupportedOpInfo::ReadPointerAsInt(_) => const_eval_read_pointer_as_int, UnsupportedOpInfo::ThreadLocalStatic(_) => const_eval_thread_local_static, UnsupportedOpInfo::ExternStatic(_) => const_eval_extern_static, @@ -848,7 +856,7 @@ impl ReportErrorExt for UnsupportedOpInfo { use UnsupportedOpInfo::*; use crate::fluent_generated::*; - if let ReadPointerAsInt(_) | OverwritePartialPointer(_) | ReadPartialPointer(_) = self { + if let ReadPointerAsInt(_) | ReadPartialPointer(_) = self { diag.help(const_eval_ptr_as_bytes_1); diag.help(const_eval_ptr_as_bytes_2); } @@ -860,7 +868,7 @@ impl ReportErrorExt for UnsupportedOpInfo { | UnsupportedOpInfo::ExternTypeField | Unsupported(_) | ReadPointerAsInt(_) => {} - OverwritePartialPointer(ptr) | ReadPartialPointer(ptr) => { + ReadPartialPointer(ptr) => { diag.arg("ptr", ptr); } ThreadLocalStatic(did) | ExternStatic(did) => rustc_middle::ty::tls::with(|tcx| { diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index 5b3adba02659..312ebe7ddd09 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -11,6 +11,7 @@ use rustc_middle::ty::{self, AdtDef, Instance, Ty, VariantDef}; use rustc_middle::{bug, mir, span_bug}; use rustc_span::sym; use rustc_target::callconv::{ArgAbi, FnAbi, PassMode}; +use tracing::field::Empty; use tracing::{info, instrument, trace}; use super::{ @@ -18,15 +19,17 @@ use super::{ Projectable, Provenance, ReturnAction, ReturnContinuation, Scalar, StackPopInfo, interp_ok, throw_ub, throw_ub_custom, throw_unsup_format, }; -use crate::fluent_generated as fluent; +use crate::interpret::EnteredTraceSpan; +use crate::{enter_trace_span, fluent_generated as fluent}; /// An argument passed to a function. #[derive(Clone, Debug)] pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> { /// Pass a copy of the given operand. Copy(OpTy<'tcx, Prov>), - /// Allow for the argument to be passed in-place: destroy the value originally stored at that place and - /// make the place inaccessible for the duration of the function call. + /// Allow for the argument to be passed in-place: destroy the value originally stored at that + /// place and make the place inaccessible for the duration of the function call. This *must* be + /// an in-memory place so that we can do the proper alias checks. InPlace(MPlaceTy<'tcx, Prov>), } @@ -124,7 +127,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } else if all_fields_1zst(def.variant(var1))? { def.variant(var0) } else { - // No varant is all-1-ZST, so no NPO. + // No variant is all-1-ZST, so no NPO. return interp_ok(layout); }; // The "relevant" variant must have exactly one field, and its type is the "inner" type. @@ -344,6 +347,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { destination: &PlaceTy<'tcx, M::Provenance>, mut cont: ReturnContinuation, ) -> InterpResult<'tcx> { + let _trace = enter_trace_span!(M, step::init_stack_frame, %instance, tracing_separate_thread = Empty); + // Compute callee information. // FIXME: for variadic support, do we have to somehow determine callee's extra_args? let callee_fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; @@ -375,6 +380,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } + // *Before* pushing the new frame, determine whether the return destination is in memory. + // Need to use `place_to_op` to be *sure* we get the mplace if there is one. + let destination_mplace = self.place_to_op(destination)?.as_mplace_or_imm().left(); + + // Push the "raw" frame -- this leaves locals uninitialized. self.push_stack_frame_raw(instance, body, destination, cont)?; // If an error is raised here, pop the frame again to get an accurate backtrace. @@ -492,7 +502,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Protect return place for in-place return value passing. // We only need to protect anything if this is actually an in-memory place. - if let Left(mplace) = destination.as_mplace_or_local() { + if let Some(mplace) = destination_mplace { M::protect_in_place_function_argument(self, &mplace)?; } @@ -523,7 +533,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { target: Option, unwind: mir::UnwindAction, ) -> InterpResult<'tcx> { - trace!("init_fn_call: {:#?}", fn_val); + let _trace = + enter_trace_span!(M, step::init_fn_call, tracing_separate_thread = Empty, ?fn_val) + .or_if_tracing_disabled(|| trace!("init_fn_call: {:#?}", fn_val)); let instance = match fn_val { FnVal::Instance(instance) => instance, @@ -646,7 +658,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let val = self.read_immediate(&receiver)?; break self.ref_to_mplace(&val)?; } - ty::Dynamic(.., ty::Dyn) => break receiver.assert_mem_place(), // no immediate unsized values + ty::Dynamic(..) => break receiver.assert_mem_place(), // no immediate unsized values _ => { // Not there yet, search for the only non-ZST field. // (The rules for `DispatchFromDyn` ensure there's exactly one such field.) @@ -663,7 +675,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // (For that reason we also cannot use `unpack_dyn_trait`.) let receiver_tail = self.tcx.struct_tail_for_codegen(receiver_place.layout.ty, self.typing_env); - let ty::Dynamic(receiver_trait, _, ty::Dyn) = receiver_tail.kind() else { + let ty::Dynamic(receiver_trait, _) = receiver_tail.kind() else { span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail) }; assert!(receiver_place.layout.is_unsized()); @@ -725,18 +737,21 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) { let tcx = *self.tcx; - let trait_def_id = tcx.trait_of_assoc(def_id).unwrap(); - let virtual_trait_ref = ty::TraitRef::from_method(tcx, trait_def_id, virtual_instance.args); + let trait_def_id = tcx.parent(def_id); + let virtual_trait_ref = ty::TraitRef::from_assoc(tcx, trait_def_id, virtual_instance.args); let existential_trait_ref = ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref); let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty); - let concrete_method = Instance::expect_resolve_for_vtable( - tcx, - self.typing_env, - def_id, - virtual_instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args), - self.cur_span(), - ); + let concrete_method = { + let _trace = enter_trace_span!(M, resolve::expect_resolve_for_vtable, ?def_id); + Instance::expect_resolve_for_vtable( + tcx, + self.typing_env, + def_id, + virtual_instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args), + self.cur_span(), + ) + }; assert_eq!(concrete_instance, concrete_method); } @@ -807,7 +822,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // instead we do the virtual call stuff ourselves. It's easier here than in `eval_fn_call` // since we can just get a place of the underlying type and use `mplace_to_ref`. let place = match place.layout.ty.kind() { - ty::Dynamic(data, _, ty::Dyn) => { + ty::Dynamic(data, _) => { // Dropping a trait object. Need to find actual drop fn. self.unpack_dyn_trait(&place, data)? } @@ -819,7 +834,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { place } }; - let instance = ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty); + let instance = { + let _trace = + enter_trace_span!(M, resolve::resolve_drop_in_place, ty = ?place.layout.ty); + ty::Instance::resolve_drop_in_place(*self.tcx, place.layout.ty) + }; let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; let arg = self.mplace_to_ref(&place)?; diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs index de4fbc7b4752..cf5ee03bedae 100644 --- a/compiler/rustc_const_eval/src/interpret/cast.rs +++ b/compiler/rustc_const_eval/src/interpret/cast.rs @@ -16,8 +16,8 @@ use super::{ FnVal, ImmTy, Immediate, InterpCx, Machine, OpTy, PlaceTy, err_inval, interp_ok, throw_ub, throw_ub_custom, }; -use crate::fluent_generated as fluent; use crate::interpret::Writeable; +use crate::{enter_trace_span, fluent_generated as fluent}; impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { pub fn cast( @@ -81,13 +81,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // The src operand does not matter, just its type match *src.layout.ty.kind() { ty::FnDef(def_id, args) => { - let instance = ty::Instance::resolve_for_fn_ptr( - *self.tcx, - self.typing_env, - def_id, - args, - ) - .ok_or_else(|| err_inval!(TooGeneric))?; + let instance = { + let _trace = enter_trace_span!(M, resolve::resolve_for_fn_ptr, ?def_id); + ty::Instance::resolve_for_fn_ptr( + *self.tcx, + self.typing_env, + def_id, + args, + ) + .ok_or_else(|| err_inval!(TooGeneric))? + }; let fn_ptr = self.fn_ptr(FnVal::Instance(instance)); self.write_pointer(fn_ptr, dest)?; @@ -114,12 +117,15 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // The src operand does not matter, just its type match *src.layout.ty.kind() { ty::Closure(def_id, args) => { - let instance = ty::Instance::resolve_closure( - *self.tcx, - def_id, - args, - ty::ClosureKind::FnOnce, - ); + let instance = { + let _trace = enter_trace_span!(M, resolve::resolve_closure, ?def_id); + ty::Instance::resolve_closure( + *self.tcx, + def_id, + args, + ty::ClosureKind::FnOnce, + ) + }; let fn_ptr = self.fn_ptr(FnVal::Instance(instance)); self.write_pointer(fn_ptr, dest)?; } @@ -127,7 +133,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } - CastKind::Transmute => { + CastKind::Transmute | CastKind::Subtype => { assert!(src.layout.is_sized()); assert!(dest.layout.is_sized()); assert_eq!(cast_ty, dest.layout.ty); // we otherwise ignore `cast_ty` enirely... @@ -380,7 +386,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ); self.write_immediate(val, dest) } - (ty::Dynamic(data_a, _, ty::Dyn), ty::Dynamic(data_b, _, ty::Dyn)) => { + (ty::Dynamic(data_a, _), ty::Dynamic(data_b, _)) => { let val = self.read_immediate(src)?; // MIR building generates odd NOP casts, prevent them from causing unexpected trouble. // See . @@ -430,7 +436,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let new_vptr = self.get_vtable_ptr(ty, data_b)?; self.write_immediate(Immediate::new_dyn_trait(old_data, new_vptr, self), dest) } - (_, &ty::Dynamic(data, _, ty::Dyn)) => { + (_, &ty::Dynamic(data, _)) => { // Initial cast from sized to dyn trait let vtable = self.get_vtable_ptr(src_pointee_ty, data)?; let ptr = self.read_pointer(src)?; @@ -460,6 +466,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> InterpResult<'tcx> { trace!("Unsizing {:?} of type {} into {}", *src, src.layout.ty, cast_ty.ty); match (src.layout.ty.kind(), cast_ty.ty.kind()) { + (&ty::Pat(_, s_pat), &ty::Pat(cast_ty, c_pat)) if s_pat == c_pat => { + let src = self.project_field(src, FieldIdx::ZERO)?; + let dest = self.project_field(dest, FieldIdx::ZERO)?; + let cast_ty = self.layout_of(cast_ty)?; + self.unsize_into(&src, cast_ty, &dest) + } (&ty::Ref(_, s, _), &ty::Ref(_, c, _) | &ty::RawPtr(c, _)) | (&ty::RawPtr(s, _), &ty::RawPtr(c, _)) => self.unsize_into_ptr(src, dest, s, c), (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs index 0a8591ca140d..0e4a98f0941a 100644 --- a/compiler/rustc_const_eval/src/interpret/eval_context.rs +++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs @@ -4,6 +4,7 @@ use either::{Left, Right}; use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout}; use rustc_errors::DiagCtxtHandle; use rustc_hir::def_id::DefId; +use rustc_hir::limit::Limit; use rustc_middle::mir::interpret::{ErrorHandled, InvalidMetaKind, ReportedErrorInfo}; use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::layout::{ @@ -12,7 +13,6 @@ use rustc_middle::ty::layout::{ }; use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeFoldable, TypingEnv, Variance}; use rustc_middle::{mir, span_bug}; -use rustc_session::Limit; use rustc_span::Span; use rustc_target::callconv::FnAbi; use tracing::{debug, trace}; @@ -113,7 +113,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// See [LayoutOf::layout_of] for the original documentation. #[inline(always)] pub fn layout_of(&self, ty: Ty<'tcx>) -> >::LayoutOfResult { - let _span = enter_trace_span!(M, "InterpCx::layout_of", ty = ?ty.kind()); + let _trace = enter_trace_span!(M, layouting::layout_of, ty = ?ty.kind()); LayoutOf::layout_of(self, ty) } @@ -126,7 +126,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { sig: ty::PolyFnSig<'tcx>, extra_args: &'tcx ty::List>, ) -> >::FnAbiOfResult { - let _span = enter_trace_span!(M, "InterpCx::fn_abi_of_fn_ptr", ?sig, ?extra_args); + let _trace = enter_trace_span!(M, layouting::fn_abi_of_fn_ptr, ?sig, ?extra_args); FnAbiOf::fn_abi_of_fn_ptr(self, sig, extra_args) } @@ -139,7 +139,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { instance: ty::Instance<'tcx>, extra_args: &'tcx ty::List>, ) -> >::FnAbiOfResult { - let _span = enter_trace_span!(M, "InterpCx::fn_abi_of_instance", ?instance, ?extra_args); + let _trace = enter_trace_span!(M, layouting::fn_abi_of_instance, ?instance, ?extra_args); FnAbiOf::fn_abi_of_instance(self, instance, extra_args) } } @@ -322,11 +322,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { frame: &Frame<'tcx, M::Provenance, M::FrameExtra>, value: T, ) -> Result { - let _span = enter_trace_span!( + let _trace = enter_trace_span!( M, "instantiate_from_frame_and_normalize_erasing_regions", - "{}", - frame.instance + %frame.instance ); frame .instance @@ -344,6 +343,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { def: DefId, args: GenericArgsRef<'tcx>, ) -> InterpResult<'tcx, ty::Instance<'tcx>> { + let _trace = enter_trace_span!(M, resolve::try_resolve, def = ?def); trace!("resolve: {:?}, {:#?}", def, args); trace!("typing_env: {:#?}", self.typing_env); trace!("args: {:#?}", args); @@ -469,7 +469,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } interp_ok(Some((full_size, full_align))) } - ty::Dynamic(expected_trait, _, ty::Dyn) => { + ty::Dynamic(expected_trait, _) => { let vtable = metadata.unwrap_meta().to_pointer(self)?; // Read size and align from vtable (already checks size). interp_ok(Some(self.get_vtable_size_and_align(vtable, Some(expected_trait))?)) @@ -582,6 +582,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { span: Span, layout: Option>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let _trace = enter_trace_span!(M, const_eval::eval_mir_constant, ?val); let const_val = val.eval(*self.tcx, self.typing_env, span).map_err(|err| { if M::ALL_CONSTS_ARE_PRECHECKED { match err { diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs index bb59b9f54186..5d510a983526 100644 --- a/compiler/rustc_const_eval/src/interpret/intern.rs +++ b/compiler/rustc_const_eval/src/interpret/intern.rs @@ -19,9 +19,12 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; use rustc_hir as hir; use rustc_hir::definitions::{DefPathData, DisambiguatorState}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; -use rustc_middle::mir::interpret::{ConstAllocation, CtfeProvenance, InterpResult}; +use rustc_middle::mir::interpret::{ + AllocBytes, ConstAllocation, CtfeProvenance, InterpResult, Provenance, +}; use rustc_middle::query::TyCtxtAt; use rustc_middle::span_bug; +use rustc_middle::ty::TyCtxt; use rustc_middle::ty::layout::TyAndLayout; use rustc_span::def_id::LocalDefId; use tracing::{instrument, trace}; @@ -52,6 +55,45 @@ impl HasStaticRootDefId for const_eval::CompileTimeMachine<'_> { } } +fn prepare_alloc<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>( + tcx: TyCtxt<'tcx>, + kind: MemoryKind, + alloc: &mut Allocation, + mutability: Mutability, +) -> Result<(), InternError> { + match kind { + MemoryKind::Machine(const_eval::MemoryKind::Heap { was_made_global }) => { + if !was_made_global { + // Attempting to intern a `const_allocate`d pointer that was not made global via + // `const_make_global`. + tcx.dcx().delayed_bug("non-global heap allocation in const value"); + return Err(InternError::ConstAllocNotGlobal); + } + } + MemoryKind::Stack | MemoryKind::CallerLocation => {} + } + + if !alloc.provenance_merge_bytes(&tcx) { + // Per-byte provenance is not supported by backends, so we cannot accept it here. + tcx.dcx().delayed_bug("partial pointer in const value"); + return Err(InternError::PartialPointer); + } + + // Set allocation mutability as appropriate. This is used by LLVM to put things into + // read-only memory, and also by Miri when evaluating other globals that + // access this one. + match mutability { + Mutability::Not => { + alloc.mutability = Mutability::Not; + } + Mutability::Mut => { + // This must be already mutable, we won't "un-freeze" allocations ever. + assert_eq!(alloc.mutability, Mutability::Mut); + } + } + Ok(()) +} + /// Intern an allocation. Returns `Err` if the allocation does not exist in the local memory. /// /// `mutability` can be used to force immutable interning: if it is `Mutability::Not`, the @@ -72,31 +114,13 @@ fn intern_shallow<'tcx, M: CompileTimeMachine<'tcx>>( return Err(InternError::DanglingPointer); }; - match kind { - MemoryKind::Machine(const_eval::MemoryKind::Heap { was_made_global }) => { - if !was_made_global { - // Attempting to intern a `const_allocate`d pointer that was not made global via - // `const_make_global`. We want to error here, but we have to first put the - // allocation back into the `alloc_map` to keep things in a consistent state. - ecx.memory.alloc_map.insert(alloc_id, (kind, alloc)); - return Err(InternError::ConstAllocNotGlobal); - } - } - MemoryKind::Stack | MemoryKind::CallerLocation => {} + if let Err(err) = prepare_alloc(*ecx.tcx, kind, &mut alloc, mutability) { + // We want to error here, but we have to first put the + // allocation back into the `alloc_map` to keep things in a consistent state. + ecx.memory.alloc_map.insert(alloc_id, (kind, alloc)); + return Err(err); } - // Set allocation mutability as appropriate. This is used by LLVM to put things into - // read-only memory, and also by Miri when evaluating other globals that - // access this one. - match mutability { - Mutability::Not => { - alloc.mutability = Mutability::Not; - } - Mutability::Mut => { - // This must be already mutable, we won't "un-freeze" allocations ever. - assert_eq!(alloc.mutability, Mutability::Mut); - } - } // link the alloc id to the actual allocation let alloc = ecx.tcx.mk_const_alloc(alloc); if let Some(static_id) = ecx.machine.static_def_id() { @@ -166,6 +190,7 @@ pub enum InternError { BadMutablePointer, DanglingPointer, ConstAllocNotGlobal, + PartialPointer, } /// Intern `ret` and everything it references. @@ -221,13 +246,11 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx>>( let mut todo: Vec<_> = if is_static { // Do not steal the root allocation, we need it later to create the return value of `eval_static_initializer`. // But still change its mutability to match the requested one. - let alloc = ecx.memory.alloc_map.get_mut(&base_alloc_id).unwrap(); - alloc.1.mutability = base_mutability; - alloc.1.provenance().ptrs().iter().map(|&(_, prov)| prov).collect() + let (kind, alloc) = ecx.memory.alloc_map.get_mut(&base_alloc_id).unwrap(); + prepare_alloc(*ecx.tcx, *kind, alloc, base_mutability)?; + alloc.provenance().ptrs().iter().map(|&(_, prov)| prov).collect() } else { - intern_shallow(ecx, base_alloc_id, base_mutability, Some(&mut disambiguator)) - .unwrap() - .collect() + intern_shallow(ecx, base_alloc_id, base_mutability, Some(&mut disambiguator))?.collect() }; // We need to distinguish "has just been interned" from "was already in `tcx`", // so we track this in a separate set. @@ -235,7 +258,6 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx>>( // Whether we encountered a bad mutable pointer. // We want to first report "dangling" and then "mutable", so we need to delay reporting these // errors. - let mut result = Ok(()); let mut found_bad_mutable_ptr = false; // Keep interning as long as there are things to intern. @@ -310,20 +332,15 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx>>( // okay with losing some potential for immutability here. This can anyway only affect // `static mut`. just_interned.insert(alloc_id); - match intern_shallow(ecx, alloc_id, inner_mutability, Some(&mut disambiguator)) { - Ok(nested) => todo.extend(nested), - Err(err) => { - ecx.tcx.dcx().delayed_bug("error during const interning"); - result = Err(err); - } - } + let next = intern_shallow(ecx, alloc_id, inner_mutability, Some(&mut disambiguator))?; + todo.extend(next); } - if found_bad_mutable_ptr && result.is_ok() { + if found_bad_mutable_ptr { // We found a mutable pointer inside a const where inner allocations should be immutable, // and there was no other error. This should usually never happen! However, this can happen // in unleash-miri mode, so report it as a normal error then. if ecx.tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you { - result = Err(InternError::BadMutablePointer); + return Err(InternError::BadMutablePointer); } else { span_bug!( ecx.tcx.span, @@ -331,7 +348,7 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx>>( ); } } - result + Ok(()) } /// Intern `ret`. This function assumes that `ret` references no other allocation. diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 5e3d0a15d8bc..f0f06d469c1e 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -2,6 +2,8 @@ //! looking at their MIR. Intrinsics/functions supported here are shared by CTFE //! and miri. +mod simd; + use std::assert_matches::assert_matches; use rustc_abi::{FieldIdx, HasDataLayout, Size}; @@ -9,8 +11,8 @@ use rustc_apfloat::ieee::{Double, Half, Quad, Single}; use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint}; use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic}; use rustc_middle::ty::layout::TyAndLayout; -use rustc_middle::ty::{Ty, TyCtxt}; -use rustc_middle::{bug, ty}; +use rustc_middle::ty::{FloatTy, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug, ty}; use rustc_span::{Symbol, sym}; use tracing::trace; @@ -121,6 +123,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> InterpResult<'tcx, bool> { let instance_args = instance.args; let intrinsic_name = self.tcx.item_name(instance.def_id()); + + if intrinsic_name.as_str().starts_with("simd_") { + return self.eval_simd_intrinsic(intrinsic_name, instance_args, args, dest, ret); + } + let tcx = self.tcx.tcx; match intrinsic_name { @@ -149,6 +156,24 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let b_ty = self.read_type_id(&args[1])?; self.write_scalar(Scalar::from_bool(a_ty == b_ty), dest)?; } + sym::size_of => { + let tp_ty = instance.args.type_at(0); + let layout = self.layout_of(tp_ty)?; + if !layout.is_sized() { + span_bug!(self.cur_span(), "unsized type for `size_of`"); + } + let val = layout.size.bytes(); + self.write_scalar(Scalar::from_target_usize(val, self), dest)?; + } + sym::align_of => { + let tp_ty = instance.args.type_at(0); + let layout = self.layout_of(tp_ty)?; + if !layout.is_sized() { + span_bug!(self.cur_span(), "unsized type for `align_of`"); + } + let val = layout.align.bytes(); + self.write_scalar(Scalar::from_target_usize(val, self), dest)?; + } sym::variant_count => { let tp_ty = instance.args.type_at(0); let ty = match tp_ty.kind() { @@ -181,7 +206,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { | ty::Ref(_, _, _) | ty::FnDef(_, _) | ty::FnPtr(..) - | ty::Dynamic(_, _, _) + | ty::Dynamic(_, _) | ty::Closure(_, _) | ty::CoroutineClosure(_, _) | ty::Coroutine(_, _) @@ -454,37 +479,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.exact_div(&val, &size, dest)?; } - sym::simd_insert => { - let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let elem = &args[2]; - let (input, input_len) = self.project_to_simd(&args[0])?; - let (dest, dest_len) = self.project_to_simd(dest)?; - assert_eq!(input_len, dest_len, "Return vector length must match input length"); - // Bounds are not checked by typeck so we have to do it ourselves. - if index >= input_len { - throw_ub_format!( - "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}" - ); - } - - for i in 0..dest_len { - let place = self.project_index(&dest, i)?; - let value = - if i == index { elem.clone() } else { self.project_index(&input, i)? }; - self.copy_op(&value, &place)?; - } - } - sym::simd_extract => { - let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let (input, input_len) = self.project_to_simd(&args[0])?; - // Bounds are not checked by typeck so we have to do it ourselves. - if index >= input_len { - throw_ub_format!( - "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}" - ); - } - self.copy_op(&self.project_index(&input, index)?, dest)?; - } sym::black_box => { // These just return their argument self.copy_op(&args[0], dest)?; @@ -636,6 +630,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { dest, rustc_apfloat::Round::NearestTiesToEven, )?, + sym::fmaf16 => self.fma_intrinsic::(args, dest)?, + sym::fmaf32 => self.fma_intrinsic::(args, dest)?, + sym::fmaf64 => self.fma_intrinsic::(args, dest)?, + sym::fmaf128 => self.fma_intrinsic::(args, dest)?, + sym::fmuladdf16 => self.float_muladd_intrinsic::(args, dest)?, + sym::fmuladdf32 => self.float_muladd_intrinsic::(args, dest)?, + sym::fmuladdf64 => self.float_muladd_intrinsic::(args, dest)?, + sym::fmuladdf128 => self.float_muladd_intrinsic::(args, dest)?, // Unsupported intrinsic: skip the return_to_block below. _ => return interp_ok(false), @@ -870,7 +872,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { .compute_size_in_bytes(layout.size, count) .ok_or_else(|| err_ub_custom!(fluent::const_eval_size_overflow, name = name))?; - let bytes = std::iter::repeat(byte).take(len.bytes_usize()); + let bytes = std::iter::repeat_n(byte, len.bytes_usize()); self.write_bytes_ptr(dst, bytes) } @@ -1035,4 +1037,104 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(res, dest)?; interp_ok(()) } + + fn fma_intrinsic( + &mut self, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx, ()> + where + F: rustc_apfloat::Float + rustc_apfloat::FloatConvert + Into>, + { + let a: F = self.read_scalar(&args[0])?.to_float()?; + let b: F = self.read_scalar(&args[1])?.to_float()?; + let c: F = self.read_scalar(&args[2])?.to_float()?; + + let res = a.mul_add(b, c).value; + let res = self.adjust_nan(res, &[a, b, c]); + self.write_scalar(res, dest)?; + interp_ok(()) + } + + fn float_muladd_intrinsic( + &mut self, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + ) -> InterpResult<'tcx, ()> + where + F: rustc_apfloat::Float + rustc_apfloat::FloatConvert + Into>, + { + let a: F = self.read_scalar(&args[0])?.to_float()?; + let b: F = self.read_scalar(&args[1])?.to_float()?; + let c: F = self.read_scalar(&args[2])?.to_float()?; + + let fuse = M::float_fuse_mul_add(self); + + let res = if fuse { a.mul_add(b, c).value } else { ((a * b).value + c).value }; + let res = self.adjust_nan(res, &[a, b, c]); + self.write_scalar(res, dest)?; + interp_ok(()) + } + + /// Converts `src` from floating point to integer type `dest_ty` + /// after rounding with mode `round`. + /// Returns `None` if `f` is NaN or out of range. + pub fn float_to_int_checked( + &self, + src: &ImmTy<'tcx, M::Provenance>, + cast_to: TyAndLayout<'tcx>, + round: rustc_apfloat::Round, + ) -> InterpResult<'tcx, Option>> { + fn float_to_int_inner<'tcx, F: rustc_apfloat::Float, M: Machine<'tcx>>( + ecx: &InterpCx<'tcx, M>, + src: F, + cast_to: TyAndLayout<'tcx>, + round: rustc_apfloat::Round, + ) -> (Scalar, rustc_apfloat::Status) { + let int_size = cast_to.layout.size; + match cast_to.ty.kind() { + // Unsigned + ty::Uint(_) => { + let res = src.to_u128_r(int_size.bits_usize(), round, &mut false); + (Scalar::from_uint(res.value, int_size), res.status) + } + // Signed + ty::Int(_) => { + let res = src.to_i128_r(int_size.bits_usize(), round, &mut false); + (Scalar::from_int(res.value, int_size), res.status) + } + // Nothing else + _ => span_bug!( + ecx.cur_span(), + "attempted float-to-int conversion with non-int output type {}", + cast_to.ty, + ), + } + } + + let ty::Float(fty) = src.layout.ty.kind() else { + bug!("float_to_int_checked: non-float input type {}", src.layout.ty) + }; + + let (val, status) = match fty { + FloatTy::F16 => float_to_int_inner(self, src.to_scalar().to_f16()?, cast_to, round), + FloatTy::F32 => float_to_int_inner(self, src.to_scalar().to_f32()?, cast_to, round), + FloatTy::F64 => float_to_int_inner(self, src.to_scalar().to_f64()?, cast_to, round), + FloatTy::F128 => float_to_int_inner(self, src.to_scalar().to_f128()?, cast_to, round), + }; + + if status.intersects( + rustc_apfloat::Status::INVALID_OP + | rustc_apfloat::Status::OVERFLOW + | rustc_apfloat::Status::UNDERFLOW, + ) { + // Floating point value is NaN (flagged with INVALID_OP) or outside the range + // of values of the integer type (flagged with OVERFLOW or UNDERFLOW). + interp_ok(None) + } else { + // Floating point value can be represented by the integer type after rounding. + // The INEXACT flag is ignored on purpose to allow rounding. + interp_ok(Some(ImmTy::from_scalar(val, cast_to))) + } + } } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs new file mode 100644 index 000000000000..0dba66ae9372 --- /dev/null +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -0,0 +1,782 @@ +use either::Either; +use rustc_abi::Endian; +use rustc_apfloat::{Float, Round}; +use rustc_middle::mir::interpret::{InterpErrorKind, UndefinedBehaviorInfo}; +use rustc_middle::ty::FloatTy; +use rustc_middle::{bug, err_ub_format, mir, span_bug, throw_unsup_format, ty}; +use rustc_span::{Symbol, sym}; +use tracing::trace; + +use super::{ + ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Provenance, Scalar, Size, interp_ok, + throw_ub_format, +}; +use crate::interpret::Writeable; + +#[derive(Copy, Clone)] +pub(crate) enum MinMax { + Min, + Max, +} + +impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { + /// Returns `true` if emulation happened. + /// Here we implement the intrinsics that are common to all CTFE instances; individual machines can add their own + /// intrinsic handling. + pub fn eval_simd_intrinsic( + &mut self, + intrinsic_name: Symbol, + generic_args: ty::GenericArgsRef<'tcx>, + args: &[OpTy<'tcx, M::Provenance>], + dest: &PlaceTy<'tcx, M::Provenance>, + ret: Option, + ) -> InterpResult<'tcx, bool> { + let dest = dest.force_mplace(self)?; + + match intrinsic_name { + sym::simd_insert => { + let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); + let elem = &args[2]; + let (input, input_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + assert_eq!(input_len, dest_len, "Return vector length must match input length"); + // Bounds are not checked by typeck so we have to do it ourselves. + if index >= input_len { + throw_ub_format!( + "`simd_insert` index {index} is out-of-bounds of vector with length {input_len}" + ); + } + + for i in 0..dest_len { + let place = self.project_index(&dest, i)?; + let value = + if i == index { elem.clone() } else { self.project_index(&input, i)? }; + self.copy_op(&value, &place)?; + } + } + sym::simd_extract => { + let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); + let (input, input_len) = self.project_to_simd(&args[0])?; + // Bounds are not checked by typeck so we have to do it ourselves. + if index >= input_len { + throw_ub_format!( + "`simd_extract` index {index} is out-of-bounds of vector with length {input_len}" + ); + } + self.copy_op(&self.project_index(&input, index)?, &dest)?; + } + sym::simd_neg + | sym::simd_fabs + | sym::simd_ceil + | sym::simd_floor + | sym::simd_round + | sym::simd_round_ties_even + | sym::simd_trunc + | sym::simd_ctlz + | sym::simd_ctpop + | sym::simd_cttz + | sym::simd_bswap + | sym::simd_bitreverse => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, op_len); + + #[derive(Copy, Clone)] + enum Op { + MirOp(mir::UnOp), + Abs, + Round(rustc_apfloat::Round), + Numeric(Symbol), + } + let which = match intrinsic_name { + sym::simd_neg => Op::MirOp(mir::UnOp::Neg), + sym::simd_fabs => Op::Abs, + sym::simd_ceil => Op::Round(rustc_apfloat::Round::TowardPositive), + sym::simd_floor => Op::Round(rustc_apfloat::Round::TowardNegative), + sym::simd_round => Op::Round(rustc_apfloat::Round::NearestTiesToAway), + sym::simd_round_ties_even => Op::Round(rustc_apfloat::Round::NearestTiesToEven), + sym::simd_trunc => Op::Round(rustc_apfloat::Round::TowardZero), + sym::simd_ctlz => Op::Numeric(sym::ctlz), + sym::simd_ctpop => Op::Numeric(sym::ctpop), + sym::simd_cttz => Op::Numeric(sym::cttz), + sym::simd_bswap => Op::Numeric(sym::bswap), + sym::simd_bitreverse => Op::Numeric(sym::bitreverse), + _ => unreachable!(), + }; + + for i in 0..dest_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + let dest = self.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // this already does NaN adjustments + self.unary_op(mir_op, &op)?.to_scalar() + } + Op::Abs => { + // Works for f32 and f64. + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!( + self.cur_span(), + "{} operand is not a float", + intrinsic_name + ) + }; + let op = op.to_scalar(); + // "Bitwise" operation, no NaN adjustments + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => Scalar::from_f32(op.to_f32()?.abs()), + FloatTy::F64 => Scalar::from_f64(op.to_f64()?.abs()), + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Round(rounding) => { + let ty::Float(float_ty) = op.layout.ty.kind() else { + span_bug!( + self.cur_span(), + "{} operand is not a float", + intrinsic_name + ) + }; + match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let f = op.to_scalar().to_f32()?; + let res = f.round_to_integral(rounding).value; + let res = self.adjust_nan(res, &[f]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let f = op.to_scalar().to_f64()?; + let res = f.round_to_integral(rounding).value; + let res = self.adjust_nan(res, &[f]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + } + } + Op::Numeric(name) => { + self.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? + } + }; + self.write_scalar(val, &dest)?; + } + } + sym::simd_add + | sym::simd_sub + | sym::simd_mul + | sym::simd_div + | sym::simd_rem + | sym::simd_shl + | sym::simd_shr + | sym::simd_and + | sym::simd_or + | sym::simd_xor + | sym::simd_eq + | sym::simd_ne + | sym::simd_lt + | sym::simd_le + | sym::simd_gt + | sym::simd_ge + | sym::simd_fmax + | sym::simd_fmin + | sym::simd_saturating_add + | sym::simd_saturating_sub + | sym::simd_arith_offset => { + use mir::BinOp; + + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, left_len); + assert_eq!(dest_len, right_len); + + enum Op { + MirOp(BinOp), + SaturatingOp(BinOp), + FMinMax(MinMax), + WrappingOffset, + } + let which = match intrinsic_name { + sym::simd_add => Op::MirOp(BinOp::Add), + sym::simd_sub => Op::MirOp(BinOp::Sub), + sym::simd_mul => Op::MirOp(BinOp::Mul), + sym::simd_div => Op::MirOp(BinOp::Div), + sym::simd_rem => Op::MirOp(BinOp::Rem), + sym::simd_shl => Op::MirOp(BinOp::ShlUnchecked), + sym::simd_shr => Op::MirOp(BinOp::ShrUnchecked), + sym::simd_and => Op::MirOp(BinOp::BitAnd), + sym::simd_or => Op::MirOp(BinOp::BitOr), + sym::simd_xor => Op::MirOp(BinOp::BitXor), + sym::simd_eq => Op::MirOp(BinOp::Eq), + sym::simd_ne => Op::MirOp(BinOp::Ne), + sym::simd_lt => Op::MirOp(BinOp::Lt), + sym::simd_le => Op::MirOp(BinOp::Le), + sym::simd_gt => Op::MirOp(BinOp::Gt), + sym::simd_ge => Op::MirOp(BinOp::Ge), + sym::simd_fmax => Op::FMinMax(MinMax::Max), + sym::simd_fmin => Op::FMinMax(MinMax::Min), + sym::simd_saturating_add => Op::SaturatingOp(BinOp::Add), + sym::simd_saturating_sub => Op::SaturatingOp(BinOp::Sub), + sym::simd_arith_offset => Op::WrappingOffset, + _ => unreachable!(), + }; + + for i in 0..dest_len { + let left = self.read_immediate(&self.project_index(&left, i)?)?; + let right = self.read_immediate(&self.project_index(&right, i)?)?; + let dest = self.project_index(&dest, i)?; + let val = match which { + Op::MirOp(mir_op) => { + // this does NaN adjustments. + let val = self.binary_op(mir_op, &left, &right).map_err_kind(|kind| { + match kind { + InterpErrorKind::UndefinedBehavior(UndefinedBehaviorInfo::ShiftOverflow { shift_amount, .. }) => { + // this resets the interpreter backtrace, but it's not worth avoiding that. + let shift_amount = match shift_amount { + Either::Left(v) => v.to_string(), + Either::Right(v) => v.to_string(), + }; + err_ub_format!("overflowing shift by {shift_amount} in `{intrinsic_name}` in lane {i}") + } + kind => kind + } + })?; + if matches!( + mir_op, + BinOp::Eq + | BinOp::Ne + | BinOp::Lt + | BinOp::Le + | BinOp::Gt + | BinOp::Ge + ) { + // Special handling for boolean-returning operations + assert_eq!(val.layout.ty, self.tcx.types.bool); + let val = val.to_scalar().to_bool().unwrap(); + bool_to_simd_element(val, dest.layout.size) + } else { + assert_ne!(val.layout.ty, self.tcx.types.bool); + assert_eq!(val.layout.ty, dest.layout.ty); + val.to_scalar() + } + } + Op::SaturatingOp(mir_op) => self.saturating_arith(mir_op, &left, &right)?, + Op::WrappingOffset => { + let ptr = left.to_scalar().to_pointer(self)?; + let offset_count = right.to_scalar().to_target_isize(self)?; + let pointee_ty = left.layout.ty.builtin_deref(true).unwrap(); + + let pointee_size = + i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap(); + let offset_bytes = offset_count.wrapping_mul(pointee_size); + let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self); + Scalar::from_maybe_pointer(offset_ptr, self) + } + Op::FMinMax(op) => self.fminmax_op(op, &left, &right)?, + }; + self.write_scalar(val, &dest)?; + } + } + sym::simd_reduce_and + | sym::simd_reduce_or + | sym::simd_reduce_xor + | sym::simd_reduce_any + | sym::simd_reduce_all + | sym::simd_reduce_max + | sym::simd_reduce_min => { + use mir::BinOp; + + let (op, op_len) = self.project_to_simd(&args[0])?; + + let imm_from_bool = |b| { + ImmTy::from_scalar( + Scalar::from_bool(b), + self.layout_of(self.tcx.types.bool).unwrap(), + ) + }; + + enum Op { + MirOp(BinOp), + MirOpBool(BinOp), + MinMax(MinMax), + } + let which = match intrinsic_name { + sym::simd_reduce_and => Op::MirOp(BinOp::BitAnd), + sym::simd_reduce_or => Op::MirOp(BinOp::BitOr), + sym::simd_reduce_xor => Op::MirOp(BinOp::BitXor), + sym::simd_reduce_any => Op::MirOpBool(BinOp::BitOr), + sym::simd_reduce_all => Op::MirOpBool(BinOp::BitAnd), + sym::simd_reduce_max => Op::MinMax(MinMax::Max), + sym::simd_reduce_min => Op::MinMax(MinMax::Min), + _ => unreachable!(), + }; + + // Initialize with first lane, then proceed with the rest. + let mut res = self.read_immediate(&self.project_index(&op, 0)?)?; + if matches!(which, Op::MirOpBool(_)) { + // Convert to `bool` scalar. + res = imm_from_bool(simd_element_to_bool(res)?); + } + for i in 1..op_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + res = match which { + Op::MirOp(mir_op) => self.binary_op(mir_op, &res, &op)?, + Op::MirOpBool(mir_op) => { + let op = imm_from_bool(simd_element_to_bool(op)?); + self.binary_op(mir_op, &res, &op)? + } + Op::MinMax(mmop) => { + if matches!(res.layout.ty.kind(), ty::Float(_)) { + ImmTy::from_scalar(self.fminmax_op(mmop, &res, &op)?, res.layout) + } else { + // Just boring integers, so NaNs to worry about + let mirop = match mmop { + MinMax::Min => BinOp::Le, + MinMax::Max => BinOp::Ge, + }; + if self.binary_op(mirop, &res, &op)?.to_scalar().to_bool()? { + res + } else { + op + } + } + } + }; + } + self.write_immediate(*res, &dest)?; + } + sym::simd_reduce_add_ordered | sym::simd_reduce_mul_ordered => { + use mir::BinOp; + + let (op, op_len) = self.project_to_simd(&args[0])?; + let init = self.read_immediate(&args[1])?; + + let mir_op = match intrinsic_name { + sym::simd_reduce_add_ordered => BinOp::Add, + sym::simd_reduce_mul_ordered => BinOp::Mul, + _ => unreachable!(), + }; + + let mut res = init; + for i in 0..op_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + res = self.binary_op(mir_op, &res, &op)?; + } + self.write_immediate(*res, &dest)?; + } + sym::simd_select => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let (yes, yes_len) = self.project_to_simd(&args[1])?; + let (no, no_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + for i in 0..dest_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let yes = self.read_immediate(&self.project_index(&yes, i)?)?; + let no = self.read_immediate(&self.project_index(&no, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { yes } else { no }; + self.write_immediate(*val, &dest)?; + } + } + // Variant of `select` that takes a bitmask rather than a "vector of bool". + sym::simd_select_bitmask => { + let mask = &args[0]; + let (yes, yes_len) = self.project_to_simd(&args[1])?; + let (no, no_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + let bitmask_len = dest_len.next_multiple_of(8); + if bitmask_len > 64 { + throw_unsup_format!( + "simd_select_bitmask: vectors larger than 64 elements are currently not supported" + ); + } + + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + // Read the mask, either as an integer or as an array. + let mask: u64 = match mask.layout.ty.kind() { + ty::Uint(_) => { + // Any larger integer type is fine. + assert!(mask.layout.size.bits() >= bitmask_len); + self.read_scalar(mask)?.to_bits(mask.layout.size)?.try_into().unwrap() + } + ty::Array(elem, _len) if elem == &self.tcx.types.u8 => { + // The array must have exactly the right size. + assert_eq!(mask.layout.size.bits(), bitmask_len); + // Read the raw bytes. + let mask = mask.assert_mem_place(); // arrays cannot be immediate + let mask_bytes = + self.read_bytes_ptr_strip_provenance(mask.ptr(), mask.layout.size)?; + // Turn them into a `u64` in the right way. + let mask_size = mask.layout.size.bytes_usize(); + let mut mask_arr = [0u8; 8]; + match self.tcx.data_layout.endian { + Endian::Little => { + // Fill the first N bytes. + mask_arr[..mask_size].copy_from_slice(mask_bytes); + u64::from_le_bytes(mask_arr) + } + Endian::Big => { + // Fill the last N bytes. + let i = mask_arr.len().strict_sub(mask_size); + mask_arr[i..].copy_from_slice(mask_bytes); + u64::from_be_bytes(mask_arr) + } + } + } + _ => bug!("simd_select_bitmask: invalid mask type {}", mask.layout.ty), + }; + + let dest_len = u32::try_from(dest_len).unwrap(); + for i in 0..dest_len { + let bit_i = simd_bitmask_index(i, dest_len, self.tcx.data_layout.endian); + let mask = mask & 1u64.strict_shl(bit_i); + let yes = self.read_immediate(&self.project_index(&yes, i.into())?)?; + let no = self.read_immediate(&self.project_index(&no, i.into())?)?; + let dest = self.project_index(&dest, i.into())?; + + let val = if mask != 0 { yes } else { no }; + self.write_immediate(*val, &dest)?; + } + // The remaining bits of the mask are ignored. + } + // Converts a "vector of bool" into a bitmask. + sym::simd_bitmask => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let bitmask_len = op_len.next_multiple_of(8); + if bitmask_len > 64 { + throw_unsup_format!( + "simd_bitmask: vectors larger than 64 elements are currently not supported" + ); + } + + let op_len = u32::try_from(op_len).unwrap(); + let mut res = 0u64; + for i in 0..op_len { + let op = self.read_immediate(&self.project_index(&op, i.into())?)?; + if simd_element_to_bool(op)? { + let bit_i = simd_bitmask_index(i, op_len, self.tcx.data_layout.endian); + res |= 1u64.strict_shl(bit_i); + } + } + // Write the result, depending on the `dest` type. + // Returns either an unsigned integer or array of `u8`. + match dest.layout.ty.kind() { + ty::Uint(_) => { + // Any larger integer type is fine, it will be zero-extended. + assert!(dest.layout.size.bits() >= bitmask_len); + self.write_scalar(Scalar::from_uint(res, dest.layout.size), &dest)?; + } + ty::Array(elem, _len) if elem == &self.tcx.types.u8 => { + // The array must have exactly the right size. + assert_eq!(dest.layout.size.bits(), bitmask_len); + // We have to write the result byte-for-byte. + let res_size = dest.layout.size.bytes_usize(); + let res_bytes; + let res_bytes_slice = match self.tcx.data_layout.endian { + Endian::Little => { + res_bytes = res.to_le_bytes(); + &res_bytes[..res_size] // take the first N bytes + } + Endian::Big => { + res_bytes = res.to_be_bytes(); + &res_bytes[res_bytes.len().strict_sub(res_size)..] // take the last N bytes + } + }; + self.write_bytes_ptr(dest.ptr(), res_bytes_slice.iter().cloned())?; + } + _ => bug!("simd_bitmask: invalid return type {}", dest.layout.ty), + } + } + sym::simd_cast + | sym::simd_as + | sym::simd_cast_ptr + | sym::simd_with_exposed_provenance => { + let (op, op_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, op_len); + + let unsafe_cast = intrinsic_name == sym::simd_cast; + let safe_cast = intrinsic_name == sym::simd_as; + let ptr_cast = intrinsic_name == sym::simd_cast_ptr; + let from_exposed_cast = intrinsic_name == sym::simd_with_exposed_provenance; + + for i in 0..dest_len { + let op = self.read_immediate(&self.project_index(&op, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = match (op.layout.ty.kind(), dest.layout.ty.kind()) { + // Int-to-(int|float): always safe + (ty::Int(_) | ty::Uint(_), ty::Int(_) | ty::Uint(_) | ty::Float(_)) + if safe_cast || unsafe_cast => + self.int_to_int_or_float(&op, dest.layout)?, + // Float-to-float: always safe + (ty::Float(_), ty::Float(_)) if safe_cast || unsafe_cast => + self.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in safe mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if safe_cast => + self.float_to_float_or_int(&op, dest.layout)?, + // Float-to-int in unchecked mode + (ty::Float(_), ty::Int(_) | ty::Uint(_)) if unsafe_cast => { + self.float_to_int_checked(&op, dest.layout, Round::TowardZero)? + .ok_or_else(|| { + err_ub_format!( + "`simd_cast` intrinsic called on {op} which cannot be represented in target type `{:?}`", + dest.layout.ty + ) + })? + } + // Ptr-to-ptr cast + (ty::RawPtr(..), ty::RawPtr(..)) if ptr_cast => + self.ptr_to_ptr(&op, dest.layout)?, + // Int->Ptr casts + (ty::Int(_) | ty::Uint(_), ty::RawPtr(..)) if from_exposed_cast => + self.pointer_with_exposed_provenance_cast(&op, dest.layout)?, + // Error otherwise + _ => + throw_unsup_format!( + "Unsupported SIMD cast from element type {from_ty} to {to_ty}", + from_ty = op.layout.ty, + to_ty = dest.layout.ty, + ), + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_shuffle_const_generic => { + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch(); + let index_len = index.len(); + + assert_eq!(left_len, right_len); + assert_eq!(u64::try_from(index_len).unwrap(), dest_len); + + for i in 0..dest_len { + let src_index: u64 = + index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into(); + let dest = self.project_index(&dest, i)?; + + let val = if src_index < left_len { + self.read_immediate(&self.project_index(&left, src_index)?)? + } else if src_index < left_len.strict_add(right_len) { + let right_idx = src_index.strict_sub(left_len); + self.read_immediate(&self.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle_const_generic` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_shuffle => { + let (left, left_len) = self.project_to_simd(&args[0])?; + let (right, right_len) = self.project_to_simd(&args[1])?; + let (index, index_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(left_len, right_len); + assert_eq!(index_len, dest_len); + + for i in 0..dest_len { + let src_index: u64 = self + .read_immediate(&self.project_index(&index, i)?)? + .to_scalar() + .to_u32()? + .into(); + let dest = self.project_index(&dest, i)?; + + let val = if src_index < left_len { + self.read_immediate(&self.project_index(&left, src_index)?)? + } else if src_index < left_len.strict_add(right_len) { + let right_idx = src_index.strict_sub(left_len); + self.read_immediate(&self.project_index(&right, right_idx)?)? + } else { + throw_ub_format!( + "`simd_shuffle` index {src_index} is out-of-bounds for 2 vectors with length {dest_len}" + ); + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_gather => { + let (passthru, passthru_len) = self.project_to_simd(&args[0])?; + let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?; + let (mask, mask_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, passthru_len); + assert_eq!(dest_len, ptrs_len); + assert_eq!(dest_len, mask_len); + + for i in 0..dest_len { + let passthru = self.read_immediate(&self.project_index(&passthru, i)?)?; + let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?; + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + let place = self.deref_pointer(&ptr)?; + self.read_immediate(&place)? + } else { + passthru + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_scatter => { + let (value, value_len) = self.project_to_simd(&args[0])?; + let (ptrs, ptrs_len) = self.project_to_simd(&args[1])?; + let (mask, mask_len) = self.project_to_simd(&args[2])?; + + assert_eq!(ptrs_len, value_len); + assert_eq!(ptrs_len, mask_len); + + for i in 0..ptrs_len { + let value = self.read_immediate(&self.project_index(&value, i)?)?; + let ptr = self.read_immediate(&self.project_index(&ptrs, i)?)?; + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + + if simd_element_to_bool(mask)? { + let place = self.deref_pointer(&ptr)?; + self.write_immediate(*value, &place)?; + } + } + } + sym::simd_masked_load => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let ptr = self.read_pointer(&args[1])?; + let (default, default_len) = self.project_to_simd(&args[2])?; + let (dest, dest_len) = self.project_to_simd(&dest)?; + + assert_eq!(dest_len, mask_len); + assert_eq!(dest_len, default_len); + + for i in 0..dest_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let default = self.read_immediate(&self.project_index(&default, i)?)?; + let dest = self.project_index(&dest, i)?; + + let val = if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + let ptr = ptr.wrapping_offset(dest.layout.size * i, self); + let place = self.ptr_to_mplace(ptr, dest.layout); + self.read_immediate(&place)? + } else { + default + }; + self.write_immediate(*val, &dest)?; + } + } + sym::simd_masked_store => { + let (mask, mask_len) = self.project_to_simd(&args[0])?; + let ptr = self.read_pointer(&args[1])?; + let (vals, vals_len) = self.project_to_simd(&args[2])?; + + assert_eq!(mask_len, vals_len); + + for i in 0..vals_len { + let mask = self.read_immediate(&self.project_index(&mask, i)?)?; + let val = self.read_immediate(&self.project_index(&vals, i)?)?; + + if simd_element_to_bool(mask)? { + // Size * u64 is implemented as always checked + let ptr = ptr.wrapping_offset(val.layout.size * i, self); + let place = self.ptr_to_mplace(ptr, val.layout); + self.write_immediate(*val, &place)? + }; + } + } + + // Unsupported intrinsic: skip the return_to_block below. + _ => return interp_ok(false), + } + + trace!("{:?}", self.dump_place(&dest.clone().into())); + self.return_to_block(ret)?; + interp_ok(true) + } + + fn fminmax_op( + &self, + op: MinMax, + left: &ImmTy<'tcx, Prov>, + right: &ImmTy<'tcx, Prov>, + ) -> InterpResult<'tcx, Scalar> { + assert_eq!(left.layout.ty, right.layout.ty); + let ty::Float(float_ty) = left.layout.ty.kind() else { + bug!("fmax operand is not a float") + }; + let left = left.to_scalar(); + let right = right.to_scalar(); + interp_ok(match float_ty { + FloatTy::F16 => unimplemented!("f16_f128"), + FloatTy::F32 => { + let left = left.to_f32()?; + let right = right.to_f32()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = self.adjust_nan(res, &[left, right]); + Scalar::from_f32(res) + } + FloatTy::F64 => { + let left = left.to_f64()?; + let right = right.to_f64()?; + let res = match op { + MinMax::Min => left.min(right), + MinMax::Max => left.max(right), + }; + let res = self.adjust_nan(res, &[left, right]); + Scalar::from_f64(res) + } + FloatTy::F128 => unimplemented!("f16_f128"), + }) + } +} + +fn simd_bitmask_index(idx: u32, vec_len: u32, endianness: Endian) -> u32 { + assert!(idx < vec_len); + match endianness { + Endian::Little => idx, + #[expect(clippy::arithmetic_side_effects)] // idx < vec_len + Endian::Big => vec_len - 1 - idx, // reverse order of bits + } +} + +fn bool_to_simd_element(b: bool, size: Size) -> Scalar { + // SIMD uses all-1 as pattern for "true". In two's complement, + // -1 has all its bits set to one and `from_int` will truncate or + // sign-extend it to `size` as required. + let val = if b { -1 } else { 0 }; + Scalar::from_int(val, size) +} + +fn simd_element_to_bool(elem: ImmTy<'_, Prov>) -> InterpResult<'_, bool> { + assert!( + matches!(elem.layout.ty.kind(), ty::Int(_) | ty::Uint(_)), + "SIMD mask element type must be an integer, but this is `{}`", + elem.layout.ty + ); + let val = elem.to_scalar().to_int(elem.layout.size)?; + interp_ok(match val { + 0 => false, + -1 => true, + _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), + }) +} diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index e22629993fba..1725635e0b47 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -289,6 +289,9 @@ pub trait Machine<'tcx>: Sized { a } + /// Determines whether the `fmuladd` intrinsics fuse the multiply-add or use separate operations. + fn float_fuse_mul_add(_ecx: &mut InterpCx<'tcx, Self>) -> bool; + /// Called before a basic block terminator is executed. #[inline] fn before_terminator(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> { @@ -672,6 +675,11 @@ pub macro compile_time_machine(<$tcx: lifetime>) { match fn_val {} } + #[inline(always)] + fn float_fuse_mul_add(_ecx: &mut InterpCx<$tcx, Self>) -> bool { + true + } + #[inline(always)] fn ub_checks(_ecx: &InterpCx<$tcx, Self>) -> InterpResult<$tcx, bool> { // We can't look at `tcx.sess` here as that can differ across crates, which can lead to diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index 47bebf5371a9..323e1cefd586 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -937,8 +937,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // (both global from `alloc_map` and local from `extra_fn_ptr_map`) if let Some(fn_val) = self.get_fn_alloc(id) { let align = match fn_val { - FnVal::Instance(instance) => { - self.tcx.codegen_instance_attrs(instance.def).alignment.unwrap_or(Align::ONE) + FnVal::Instance(_instance) => { + // FIXME: Until we have a clear design for the effects of align(N) functions + // on the address of function pointers, we don't consider the align(N) + // attribute on functions in the interpreter. + // See for more context. + Align::ONE } // Machine-specific extra functions currently do not support alignment restrictions. FnVal::Other(_) => Align::ONE, @@ -949,6 +953,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // # Global allocations if let Some(global_alloc) = self.tcx.try_get_global_alloc(id) { + // NOTE: `static` alignment from attributes has already been applied to the allocation. let (size, align) = global_alloc.size_and_align(*self.tcx, self.typing_env); let mutbl = global_alloc.mutability(*self.tcx, self.typing_env); let kind = match global_alloc { @@ -1310,29 +1315,20 @@ impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> } /// Mark the given sub-range (relative to this allocation reference) as uninitialized. - pub fn write_uninit(&mut self, range: AllocRange) -> InterpResult<'tcx> { + pub fn write_uninit(&mut self, range: AllocRange) { let range = self.range.subrange(range); - self.alloc - .write_uninit(&self.tcx, range) - .map_err(|e| e.to_interp_error(self.alloc_id)) - .into() + self.alloc.write_uninit(&self.tcx, range); } /// Mark the entire referenced range as uninitialized - pub fn write_uninit_full(&mut self) -> InterpResult<'tcx> { - self.alloc - .write_uninit(&self.tcx, self.range) - .map_err(|e| e.to_interp_error(self.alloc_id)) - .into() + pub fn write_uninit_full(&mut self) { + self.alloc.write_uninit(&self.tcx, self.range); } /// Remove all provenance in the reference range. - pub fn clear_provenance(&mut self) -> InterpResult<'tcx> { - self.alloc - .clear_provenance(&self.tcx, self.range) - .map_err(|e| e.to_interp_error(self.alloc_id)) - .into() + pub fn clear_provenance(&mut self) { + self.alloc.clear_provenance(&self.tcx, self.range); } } @@ -1423,11 +1419,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Side-step AllocRef and directly access the underlying bytes more efficiently. // (We are staying inside the bounds here and all bytes do get overwritten so all is good.) - let alloc_id = alloc_ref.alloc_id; - let bytes = alloc_ref - .alloc - .get_bytes_unchecked_for_overwrite(&alloc_ref.tcx, alloc_ref.range) - .map_err(move |e| e.to_interp_error(alloc_id))?; + let bytes = + alloc_ref.alloc.get_bytes_unchecked_for_overwrite(&alloc_ref.tcx, alloc_ref.range); // `zip` would stop when the first iterator ends; we want to definitely // cover all of `bytes`. for dest in bytes { @@ -1511,7 +1504,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // This will also error if copying partial provenance is not supported. let provenance = src_alloc .provenance() - .prepare_copy(src_range, dest_offset, num_copies, self) + .prepare_copy(src_range, self) .map_err(|e| e.to_interp_error(src_alloc_id))?; // Prepare a copy of the initialization mask. let init = src_alloc.init_mask().prepare_copy(src_range); @@ -1530,10 +1523,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { dest_range, )?; // Yes we do overwrite all bytes in `dest_bytes`. - let dest_bytes = dest_alloc - .get_bytes_unchecked_for_overwrite_ptr(&tcx, dest_range) - .map_err(|e| e.to_interp_error(dest_alloc_id))? - .as_mut_ptr(); + let dest_bytes = + dest_alloc.get_bytes_unchecked_for_overwrite_ptr(&tcx, dest_range).as_mut_ptr(); if init.no_bytes_init() { // Fast path: If all bytes are `uninit` then there is nothing to copy. The target range @@ -1542,9 +1533,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // This also avoids writing to the target bytes so that the backing allocation is never // touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary // operating system this can avoid physically allocating the page. - dest_alloc - .write_uninit(&tcx, dest_range) - .map_err(|e| e.to_interp_error(dest_alloc_id))?; + dest_alloc.write_uninit(&tcx, dest_range); // `write_uninit` also resets the provenance, so we are done. return interp_ok(()); } @@ -1601,7 +1590,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { num_copies, ); // copy the provenance to the destination - dest_alloc.provenance_apply_copy(provenance); + dest_alloc.provenance_apply_copy(provenance, alloc_range(dest_offset, size), num_copies); interp_ok(()) } diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 21afd082a055..560b0e1ae4e6 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -13,6 +13,7 @@ use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter}; use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt}; use rustc_middle::{bug, mir, span_bug, ty}; use rustc_span::DUMMY_SP; +use tracing::field::Empty; use tracing::trace; use super::{ @@ -20,6 +21,7 @@ use super::{ OffsetMode, PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub, from_known_layout, interp_ok, mir_assign_valid_types, throw_ub, }; +use crate::enter_trace_span; /// An `Immediate` represents a single immediate self-contained Rust value. /// @@ -173,6 +175,16 @@ impl Immediate { } interp_ok(()) } + + pub fn has_provenance(&self) -> bool { + match self { + Immediate::Scalar(scalar) => matches!(scalar, Scalar::Ptr { .. }), + Immediate::ScalarPair(s1, s2) => { + matches!(s1, Scalar::Ptr { .. }) || matches!(s2, Scalar::Ptr { .. }) + } + Immediate::Uninit => false, + } + } } // ScalarPair needs a type to interpret, so we often have an immediate and a type together @@ -186,18 +198,18 @@ pub struct ImmTy<'tcx, Prov: Provenance = CtfeProvenance> { impl std::fmt::Display for ImmTy<'_, Prov> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { /// Helper function for printing a scalar to a FmtPrinter - fn p<'a, 'tcx, Prov: Provenance>( - cx: &mut FmtPrinter<'a, 'tcx>, + fn print_scalar<'a, 'tcx, Prov: Provenance>( + p: &mut FmtPrinter<'a, 'tcx>, s: Scalar, ty: Ty<'tcx>, ) -> Result<(), std::fmt::Error> { match s { - Scalar::Int(int) => cx.pretty_print_const_scalar_int(int, ty, true), + Scalar::Int(int) => p.pretty_print_const_scalar_int(int, ty, true), Scalar::Ptr(ptr, _sz) => { // Just print the ptr value. `pretty_print_const_scalar_ptr` would also try to // print what is points to, which would fail since it has no access to the local // memory. - cx.pretty_print_const_pointer(ptr, ty) + p.pretty_print_const_pointer(ptr, ty) } } } @@ -205,8 +217,9 @@ impl std::fmt::Display for ImmTy<'_, Prov> { match self.imm { Immediate::Scalar(s) => { if let Some(ty) = tcx.lift(self.layout.ty) { - let s = - FmtPrinter::print_string(tcx, Namespace::ValueNS, |cx| p(cx, s, ty))?; + let s = FmtPrinter::print_string(tcx, Namespace::ValueNS, |p| { + print_scalar(p, s, ty) + })?; f.write_str(&s)?; return Ok(()); } @@ -770,6 +783,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { mir_place: mir::Place<'tcx>, layout: Option>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let _trace = enter_trace_span!( + M, + step::eval_place_to_op, + ?mir_place, + tracing_separate_thread = Empty + ); + // Do not use the layout passed in as argument if the base we are looking at // here is not the entire place. let layout = if mir_place.projection.is_empty() { layout } else { None }; @@ -813,6 +833,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { mir_op: &mir::Operand<'tcx>, layout: Option>, ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> { + let _trace = + enter_trace_span!(M, step::eval_operand, ?mir_op, tracing_separate_thread = Empty); + use rustc_middle::mir::Operand::*; let op = match mir_op { // FIXME: do some more logic on `move` to invalidate the old location diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index 74f8a0a7b093..58b90abf0129 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -517,20 +517,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let usize_layout = || self.layout_of(self.tcx.types.usize).unwrap(); interp_ok(match null_op { - SizeOf => { - if !layout.is_sized() { - span_bug!(self.cur_span(), "unsized type for `NullaryOp::SizeOf`"); - } - let val = layout.size.bytes(); - ImmTy::from_uint(val, usize_layout()) - } - AlignOf => { - if !layout.is_sized() { - span_bug!(self.cur_span(), "unsized type for `NullaryOp::AlignOf`"); - } - let val = layout.align.abi.bytes(); - ImmTy::from_uint(val, usize_layout()) - } OffsetOf(fields) => { let val = self.tcx.offset_of_subfield(self.typing_env, layout, fields.iter()).bytes(); diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index e2284729efdc..cd34892f029c 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -9,6 +9,7 @@ use rustc_abi::{BackendRepr, HasDataLayout, Size}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::{bug, mir, span_bug}; +use tracing::field::Empty; use tracing::{instrument, trace}; use super::{ @@ -16,6 +17,7 @@ use super::{ InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer, Projectable, Provenance, Scalar, alloc_range, interp_ok, mir_assign_valid_types, }; +use crate::enter_trace_span; #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] /// Information required for the sound usage of a `MemPlace`. @@ -232,6 +234,12 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> { } /// A place is either an mplace or some local. + /// + /// Note that the return value can be different even for logically identical places! + /// Specifically, if a local is stored in-memory, this may return `Local` or `MPlaceTy` + /// depending on how the place was constructed. In other words, seeing `Local` here does *not* + /// imply that this place does not point to memory. Every caller must therefore always handle + /// both cases. #[inline(always)] pub fn as_mplace_or_local( &self, @@ -524,6 +532,9 @@ where &self, mir_place: mir::Place<'tcx>, ) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> { + let _trace = + enter_trace_span!(M, step::eval_place, ?mir_place, tracing_separate_thread = Empty); + let mut place = self.local_to_place(mir_place.local)?; // Using `try_fold` turned out to be bad for performance, hence the loop. for elem in mir_place.projection.iter() { @@ -700,7 +711,7 @@ where match value { Immediate::Scalar(scalar) => { - alloc.write_scalar(alloc_range(Size::ZERO, scalar.size()), scalar) + alloc.write_scalar(alloc_range(Size::ZERO, scalar.size()), scalar)?; } Immediate::ScalarPair(a_val, b_val) => { let BackendRepr::ScalarPair(a, b) = layout.backend_repr else { @@ -720,10 +731,10 @@ where alloc.write_scalar(alloc_range(Size::ZERO, a_val.size()), a_val)?; alloc.write_scalar(alloc_range(b_offset, b_val.size()), b_val)?; // We don't have to reset padding here, `write_immediate` will anyway do a validation run. - interp_ok(()) } Immediate::Uninit => alloc.write_uninit_full(), } + interp_ok(()) } pub fn write_uninit( @@ -743,7 +754,7 @@ where // Zero-sized access return interp_ok(()); }; - alloc.write_uninit_full()?; + alloc.write_uninit_full(); } } interp_ok(()) @@ -754,6 +765,13 @@ where &mut self, dest: &impl Writeable<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { + // If this is an efficiently represented local variable without provenance, skip the + // `as_mplace_or_mutable_local` that would otherwise force this local into memory. + if let Right(imm) = dest.to_op(self)?.as_mplace_or_imm() { + if !imm.has_provenance() { + return interp_ok(()); + } + } match self.as_mplace_or_mutable_local(&dest.to_place())? { Right((local_val, _local_layout, local)) => { local_val.clear_provenance()?; @@ -767,7 +785,7 @@ where // Zero-sized access return interp_ok(()); }; - alloc.clear_provenance()?; + alloc.clear_provenance(); } } interp_ok(()) @@ -840,7 +858,7 @@ where /// Also, if you use this you are responsible for validating that things get copied at the /// right type. #[instrument(skip(self), level = "trace")] - fn copy_op_no_validate( + pub(super) fn copy_op_no_validate( &mut self, src: &impl Projectable<'tcx, M::Provenance>, dest: &impl Writeable<'tcx, M::Provenance>, diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index f72c44180814..027e634ef7f7 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -2,7 +2,7 @@ //! //! OpTy and PlaceTy generally work by "let's see if we are actually an MPlaceTy, and do something custom if not". //! For PlaceTy, the custom thing is basically always to call `force_allocation` and then use the MPlaceTy logic anyway. -//! For OpTy, the custom thing on field pojections has to be pretty clever (since `Operand::Immediate` can have fields), +//! For OpTy, the custom thing on field projections has to be pretty clever (since `Operand::Immediate` can have fields), //! but for array/slice operations it only has to worry about `Operand::Uninit`. That makes the value part trivial, //! but we still need to do bounds checking and adjust the layout. To not duplicate that with MPlaceTy, we actually //! implement the logic on OpTy, and MPlaceTy calls that. @@ -199,6 +199,15 @@ where base.offset_with_meta(offset, OffsetMode::Inbounds, meta, field_layout, self) } + /// Projects multiple fields at once. See [`Self::project_field`] for details. + pub fn project_fields, const N: usize>( + &self, + base: &P, + fields: [FieldIdx; N], + ) -> InterpResult<'tcx, [P; N]> { + fields.try_map(|field| self.project_field(base, field)) + } + /// Downcasting to an enum variant. pub fn project_downcast>( &self, @@ -386,8 +395,6 @@ where span_bug!(self.cur_span(), "OpaqueCast({ty}) encountered after borrowck") } UnwrapUnsafeBinder(target) => base.transmute(self.layout_of(target)?, self)?, - // We don't want anything happening here, this is here as a dummy. - Subtype(_) => base.transmute(base.layout(), self)?, Field(field, _) => self.project_field(base, field)?, Downcast(_, variant) => self.project_downcast(base, variant)?, Deref => self.deref_pointer(&base.to_op(self)?)?.into(), diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs index 2e99bb4209f2..1c1c59da9d88 100644 --- a/compiler/rustc_const_eval/src/interpret/stack.rs +++ b/compiler/rustc_const_eval/src/interpret/stack.rs @@ -20,7 +20,7 @@ use super::{ MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout, interp_ok, throw_ub, throw_unsup, }; -use crate::errors; +use crate::{enter_trace_span, errors}; // The Phantomdata exists to prevent this type from being `Send`. If it were sent across a thread // boundary and dropped in the other thread, it would exit the span in the other thread. @@ -386,6 +386,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Make sure all the constants required by this frame evaluate successfully (post-monomorphization check). for &const_ in body.required_consts() { + // We can't use `eval_mir_constant` here as that assumes that all required consts have + // already been checked, so we need a separate tracing call. + let _trace = enter_trace_span!(M, const_eval::required_consts, ?const_.const_); let c = self.instantiate_from_current_frame_and_normalize_erasing_regions(const_.const_)?; c.eval(*self.tcx, self.typing_env, const_.span).map_err(|err| { @@ -397,11 +400,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Finish things up. M::after_stack_push(self)?; self.frame_mut().loc = Left(mir::Location::START); - // `tracing_separate_thread` is used to instruct the chrome_tracing [tracing::Layer] in Miri + // `tracing_separate_thread` is used to instruct the tracing_chrome [tracing::Layer] in Miri // to put the "frame" span on a separate trace thread/line than other spans, to make the - // visualization in https://ui.perfetto.dev easier to interpret. It is set to a value of + // visualization in easier to interpret. It is set to a value of // [tracing::field::Empty] so that other tracing layers (e.g. the logger) will ignore it. - let span = info_span!("frame", tracing_separate_thread = Empty, "{}", instance); + let span = info_span!("frame", tracing_separate_thread = Empty, frame = %instance); self.frame_mut().tracing_span.enter(span); interp_ok(()) @@ -506,7 +509,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { | ty::Never | ty::Error(_) => true, - ty::Str | ty::Slice(_) | ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => false, + ty::Str | ty::Slice(_) | ty::Dynamic(_, _) | ty::Foreign(..) => false, ty::Tuple(tys) => tys.last().is_none_or(|ty| is_very_trivially_sized(*ty)), diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 629dcc3523ca..1766bbe92480 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -2,20 +2,25 @@ //! //! The main entry point is the `step` method. +use std::iter; + use either::Either; use rustc_abi::{FIRST_VARIANT, FieldIdx}; +use rustc_data_structures::fx::FxHashSet; use rustc_index::IndexSlice; use rustc_middle::ty::{self, Instance, Ty}; use rustc_middle::{bug, mir, span_bug}; use rustc_span::source_map::Spanned; use rustc_target::callconv::FnAbi; +use tracing::field::Empty; use tracing::{info, instrument, trace}; use super::{ FnArg, FnVal, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemPlaceMeta, PlaceTy, - Projectable, Scalar, interp_ok, throw_ub, throw_unsup_format, + Projectable, interp_ok, throw_ub, throw_unsup_format, }; -use crate::util; +use crate::interpret::EnteredTraceSpan; +use crate::{enter_trace_span, util}; struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { callee: FnVal<'tcx, M::ExtraFnVal>, @@ -74,7 +79,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// /// This does NOT move the statement counter forward, the caller has to do that! pub fn eval_statement(&mut self, stmt: &mir::Statement<'tcx>) -> InterpResult<'tcx> { - info!("{:?}", stmt); + let _trace = enter_trace_span!( + M, + step::eval_statement, + stmt = ?stmt.kind, + span = ?stmt.source_info.span, + tracing_separate_thread = Empty, + ) + .or_if_tracing_disabled(|| info!(stmt = ?stmt.kind)); use rustc_middle::mir::StatementKind::*; @@ -86,11 +98,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_discriminant(*variant_index, &dest)?; } - Deinit(place) => { - let dest = self.eval_place(**place)?; - self.write_uninit(&dest)?; - } - // Mark locals as alive StorageLive(local) => { self.storage_live(*local)?; @@ -176,10 +183,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.copy_op(&op, &dest)?; } - CopyForDeref(place) => { - let op = self.eval_place_to_op(place, Some(dest.layout))?; - self.copy_op(&op, &dest)?; - } + CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"), BinaryOp(bin_op, box (ref left, ref right)) => { let layout = util::binop_left_homogeneous(bin_op).then_some(dest.layout); @@ -213,12 +217,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_repeat(operand, &dest)?; } - Len(place) => { - let src = self.eval_place(place)?; - let len = src.len(self)?; - self.write_scalar(Scalar::from_target_usize(len, self), &dest)?; - } - Ref(_, borrow_kind, place) => { let src = self.eval_place(place)?; let place = self.force_allocation(&src)?; @@ -298,7 +296,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { operands: &IndexSlice>, dest: &PlaceTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx> { - self.write_uninit(dest)?; // make sure all the padding ends up as uninit let (variant_index, variant_dest, active_field_index) = match *kind { mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { let variant_dest = self.project_downcast(dest, variant_index)?; @@ -334,9 +331,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let field_index = active_field_index.unwrap_or(field_index); let field_dest = self.project_field(&variant_dest, field_index)?; let op = self.eval_operand(operand, Some(field_dest.layout))?; - self.copy_op(&op, &field_dest)?; + // We validate manually below so we don't have to do it here. + self.copy_op_no_validate(&op, &field_dest, /*allow_transmute*/ false)?; } - self.write_discriminant(variant_index, dest) + self.write_discriminant(variant_index, dest)?; + // Validate that the entire thing is valid, and reset padding that might be in between the + // fields. + if M::enforce_validity(self, dest.layout()) { + self.validate_operand( + dest, + M::enforce_validity_recursively(self, dest.layout()), + /*reset_provenance_and_padding*/ true, + )?; + } + interp_ok(()) } /// Repeats `operand` into the destination. `dest` must have array type, and that type @@ -380,8 +388,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Evaluate the arguments of a function call fn eval_fn_call_argument( - &self, + &mut self, op: &mir::Operand<'tcx>, + move_definitely_disjoint: bool, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match op { mir::Operand::Copy(_) | mir::Operand::Constant(_) => { @@ -390,24 +399,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { FnArg::Copy(op) } mir::Operand::Move(place) => { - // If this place lives in memory, preserve its location. - // We call `place_to_op` which will be an `MPlaceTy` whenever there exists - // an mplace for this place. (This is in contrast to `PlaceTy::as_mplace_or_local` - // which can return a local even if that has an mplace.) let place = self.eval_place(*place)?; - let op = self.place_to_op(&place)?; - - match op.as_mplace_or_imm() { - Either::Left(mplace) => FnArg::InPlace(mplace), - Either::Right(_imm) => { - // This argument doesn't live in memory, so there's no place - // to make inaccessible during the call. - // We rely on there not being any stray `PlaceTy` that would let the - // caller directly access this local! - // This is also crucial for tail calls, where we want the `FnArg` to - // stay valid when the old stack frame gets popped. - FnArg::Copy(op) + if move_definitely_disjoint { + // We still have to ensure that no *other* pointers are used to access this place, + // so *if* it is in memory then we have to treat it as `InPlace`. + // Use `place_to_op` to guarantee that we notice it being in memory. + let op = self.place_to_op(&place)?; + match op.as_mplace_or_imm() { + Either::Left(mplace) => FnArg::InPlace(mplace), + Either::Right(_imm) => FnArg::Copy(op), } + } else { + // We have to force this into memory to detect aliasing among `Move` arguments. + FnArg::InPlace(self.force_allocation(&place)?) } } }) @@ -416,18 +420,53 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { /// Shared part of `Call` and `TailCall` implementation — finding and evaluating all the /// necessary information about callee and arguments to make a call. fn eval_callee_and_args( - &self, + &mut self, terminator: &mir::Terminator<'tcx>, func: &mir::Operand<'tcx>, args: &[Spanned>], + dest: &mir::Place<'tcx>, ) -> InterpResult<'tcx, EvaluatedCalleeAndArgs<'tcx, M>> { let func = self.eval_operand(func, None)?; + + // Evaluating function call arguments. The tricky part here is dealing with `Move` + // arguments: we have to ensure no two such arguments alias. This would be most easily done + // by just forcing them all into memory and then doing the usual in-place argument + // protection, but then we'd force *a lot* of arguments into memory. So we do some syntactic + // pre-processing here where if all `move` arguments are syntactically distinct local + // variables (and none is indirect), we can skip the in-memory forcing. + // We have to include `dest` in that list so that we can detect aliasing of an in-place + // argument with the return place. + let move_definitely_disjoint = 'move_definitely_disjoint: { + let mut previous_locals = FxHashSet::::default(); + for place in args + .iter() + .filter_map(|a| { + // We only have to care about `Move` arguments. + if let mir::Operand::Move(place) = &a.node { Some(place) } else { None } + }) + .chain(iter::once(dest)) + { + if place.is_indirect_first_projection() { + // An indirect in-place argument could alias with anything else... + break 'move_definitely_disjoint false; + } + if !previous_locals.insert(place.local) { + // This local is the base for two arguments! They might overlap. + break 'move_definitely_disjoint false; + } + } + // We found no violation so they are all definitely disjoint. + true + }; let args = args .iter() - .map(|arg| self.eval_fn_call_argument(&arg.node)) + .map(|arg| self.eval_fn_call_argument(&arg.node, move_definitely_disjoint)) .collect::>>()?; - let fn_sig_binder = func.layout.ty.fn_sig(*self.tcx); + let fn_sig_binder = { + let _trace = enter_trace_span!(M, "fn_sig", ty = ?func.layout.ty.kind()); + func.layout.ty.fn_sig(*self.tcx) + }; let fn_sig = self.tcx.normalize_erasing_late_bound_regions(self.typing_env, fn_sig_binder); let extra_args = &args[fn_sig.inputs().len()..]; let extra_args = @@ -456,7 +495,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } fn eval_terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> { - info!("{:?}", terminator.kind); + let _trace = enter_trace_span!( + M, + step::eval_terminator, + terminator = ?terminator.kind, + span = ?terminator.source_info.span, + tracing_separate_thread = Empty, + ) + .or_if_tracing_disabled(|| info!(terminator = ?terminator.kind)); use rustc_middle::mir::TerminatorKind::*; match terminator.kind { @@ -503,7 +549,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let old_loc = self.frame().loc; let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args)?; + self.eval_callee_and_args(terminator, func, args, &destination)?; let destination = self.eval_place(destination)?; self.init_fn_call( @@ -526,7 +572,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let old_frame_idx = self.frame_idx(); let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = - self.eval_callee_and_args(terminator, func, args)?; + self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?; self.init_fn_tail_call(callee, (fn_sig.abi, fn_abi), &args, with_caller_location)?; @@ -544,7 +590,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { "Async Drop must be expanded or reset to sync in runtime MIR" ); let place = self.eval_place(place)?; - let instance = Instance::resolve_drop_in_place(*self.tcx, place.layout.ty); + let instance = { + let _trace = + enter_trace_span!(M, resolve::resolve_drop_in_place, ty = ?place.layout.ty); + Instance::resolve_drop_in_place(*self.tcx, place.layout.ty) + }; if let ty::InstanceKind::DropGlue(_, None) = instance.def { // This is the branch we enter if and only if the dropped type has no drop glue // whatsoever. This can happen as a result of monomorphizing a drop of a diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs index e4b5c82853a2..d982ed961674 100644 --- a/compiler/rustc_const_eval/src/interpret/traits.rs +++ b/compiler/rustc_const_eval/src/interpret/traits.rs @@ -23,7 +23,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> InterpResult<'tcx, Pointer>> { trace!("get_vtable(ty={ty:?}, dyn_ty={dyn_ty:?})"); - let (ty, dyn_ty) = self.tcx.erase_regions((ty, dyn_ty)); + let (ty, dyn_ty) = self.tcx.erase_and_anonymize_regions((ty, dyn_ty)); // All vtables must be monomorphic, bail out otherwise. ensure_monomorphic_enough(*self.tcx, ty)?; @@ -53,8 +53,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ) -> &'tcx [VtblEntry<'tcx>] { if let Some(trait_) = trait_ { let trait_ref = trait_.with_self_ty(*self.tcx, dyn_ty); - let trait_ref = - self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_ref)); + let trait_ref = self.tcx.erase_and_anonymize_regions( + self.tcx.instantiate_bound_regions_with_erased(trait_ref), + ); self.tcx.vtable_entries(trait_ref) } else { TyCtxt::COMMON_VTABLE_ENTRIES @@ -108,7 +109,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { expected_trait: &'tcx ty::List>, ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> { assert!( - matches!(mplace.layout.ty.kind(), ty::Dynamic(_, _, ty::Dyn)), + matches!(mplace.layout.ty.kind(), ty::Dynamic(_, _)), "`unpack_dyn_trait` only makes sense on `dyn*` types" ); let vtable = mplace.meta().unwrap_meta().to_pointer(self)?; diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs index 72650d545c3c..1e18a22be81c 100644 --- a/compiler/rustc_const_eval/src/interpret/util.rs +++ b/compiler/rustc_const_eval/src/interpret/util.rs @@ -1,6 +1,6 @@ use rustc_hir::def_id::LocalDefId; use rustc_middle::mir; -use rustc_middle::mir::interpret::{AllocInit, Allocation, InterpResult, Pointer}; +use rustc_middle::mir::interpret::{AllocInit, Allocation, GlobalAlloc, InterpResult, Pointer}; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{TyCtxt, TypeVisitable, TypeVisitableExt}; use tracing::debug; @@ -38,7 +38,14 @@ pub(crate) fn create_static_alloc<'tcx>( static_def_id: LocalDefId, layout: TyAndLayout<'tcx>, ) -> InterpResult<'tcx, MPlaceTy<'tcx>> { - let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit, ())?; + // Inherit size and align from the `GlobalAlloc::Static` so we can avoid duplicating + // the alignment attribute logic. + let (size, align) = + GlobalAlloc::Static(static_def_id.into()).size_and_align(*ecx.tcx, ecx.typing_env); + assert_eq!(size, layout.size); + assert!(align >= layout.align.abi); + + let alloc = Allocation::try_new(size, align, AllocInit::Uninit, ())?; let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into()); assert_eq!(ecx.machine.static_root_ids, None); ecx.machine.static_root_ids = Some((alloc_id, static_def_id)); @@ -48,18 +55,104 @@ pub(crate) fn create_static_alloc<'tcx>( /// A marker trait returned by [crate::interpret::Machine::enter_trace_span], identifying either a /// real [tracing::span::EnteredSpan] in case tracing is enabled, or the dummy type `()` when -/// tracing is disabled. -pub trait EnteredTraceSpan {} -impl EnteredTraceSpan for () {} -impl EnteredTraceSpan for tracing::span::EnteredSpan {} - -/// Shortand for calling [crate::interpret::Machine::enter_trace_span] on a [tracing::info_span]. -/// This is supposed to be compiled out when [crate::interpret::Machine::enter_trace_span] has the -/// default implementation (i.e. when it does not actually enter the span but instead returns `()`). -/// Note: the result of this macro **must be used** because the span is exited when it's dropped. -#[macro_export] -macro_rules! enter_trace_span { - ($machine:ident, $($tt:tt)*) => { - $machine::enter_trace_span(|| tracing::info_span!($($tt)*)) +/// tracing is disabled. Also see [crate::enter_trace_span!] below. +pub trait EnteredTraceSpan { + /// Allows executing an alternative function when tracing is disabled. Useful for example if you + /// want to open a trace span when tracing is enabled, and alternatively just log a line when + /// tracing is disabled. + fn or_if_tracing_disabled(self, f: impl FnOnce()) -> Self; +} +impl EnteredTraceSpan for () { + fn or_if_tracing_disabled(self, f: impl FnOnce()) -> Self { + f(); // tracing is disabled, execute the function + self } } +impl EnteredTraceSpan for tracing::span::EnteredSpan { + fn or_if_tracing_disabled(self, _f: impl FnOnce()) -> Self { + self // tracing is enabled, don't execute anything + } +} + +/// Shorthand for calling [crate::interpret::Machine::enter_trace_span] on a [tracing::info_span!]. +/// This is supposed to be compiled out when [crate::interpret::Machine::enter_trace_span] has the +/// default implementation (i.e. when it does not actually enter the span but instead returns `()`). +/// This macro takes a type implementing the [crate::interpret::Machine] trait as its first argument +/// and otherwise accepts the same syntax as [tracing::span!] (see some tips below). +/// Note: the result of this macro **must be used** because the span is exited when it's dropped. +/// +/// ### Syntax accepted by this macro +/// +/// The full documentation for the [tracing::span!] syntax can be found at [tracing] under "Using the +/// Macros". A few possibly confusing syntaxes are listed here: +/// ```rust +/// # use rustc_const_eval::enter_trace_span; +/// # type M = rustc_const_eval::const_eval::CompileTimeMachine<'static>; +/// # let my_display_var = String::new(); +/// # let my_debug_var = String::new(); +/// // logs a span named "hello" with a field named "arg" of value 42 (works only because +/// // 42 implements the tracing::Value trait, otherwise use one of the options below) +/// let _trace = enter_trace_span!(M, "hello", arg = 42); +/// // logs a field called "my_display_var" using the Display implementation +/// let _trace = enter_trace_span!(M, "hello", %my_display_var); +/// // logs a field called "my_debug_var" using the Debug implementation +/// let _trace = enter_trace_span!(M, "hello", ?my_debug_var); +/// ``` +/// +/// ### `NAME::SUBNAME` syntax +/// +/// In addition to the syntax accepted by [tracing::span!], this macro optionally allows passing +/// the span name (i.e. the first macro argument) in the form `NAME::SUBNAME` (without quotes) to +/// indicate that the span has name "NAME" (usually the name of the component) and has an additional +/// more specific name "SUBNAME" (usually the function name). The latter is passed to the [tracing] +/// infrastructure as a span field with the name "NAME". This allows not being distracted by +/// subnames when looking at the trace in , but when deeper introspection +/// is needed within a component, it's still possible to view the subnames directly in the UI by +/// selecting a span, clicking on the "NAME" argument on the right, and clicking on "Visualize +/// argument values". +/// ```rust +/// # use rustc_const_eval::enter_trace_span; +/// # type M = rustc_const_eval::const_eval::CompileTimeMachine<'static>; +/// // for example, the first will expand to the second +/// let _trace = enter_trace_span!(M, borrow_tracker::on_stack_pop, /* ... */); +/// let _trace = enter_trace_span!(M, "borrow_tracker", borrow_tracker = "on_stack_pop", /* ... */); +/// ``` +/// +/// ### `tracing_separate_thread` parameter +/// +/// This macro was introduced to obtain better traces of Miri without impacting release performance. +/// Miri saves traces using the `tracing_chrome` `tracing::Layer` so that they can be visualized +/// in . To instruct `tracing_chrome` to put some spans on a separate trace +/// thread/line than other spans when viewed in , you can pass +/// `tracing_separate_thread = tracing::field::Empty` to the tracing macros. This is useful to +/// separate out spans which just indicate the current step or program frame being processed by the +/// interpreter. You should use a value of [tracing::field::Empty] so that other tracing layers +/// (e.g. the logger) will ignore the `tracing_separate_thread` field. For example: +/// ```rust +/// # use rustc_const_eval::enter_trace_span; +/// # type M = rustc_const_eval::const_eval::CompileTimeMachine<'static>; +/// let _trace = enter_trace_span!(M, step::eval_statement, tracing_separate_thread = tracing::field::Empty); +/// ``` +/// +/// ### Executing something else when tracing is disabled +/// +/// [crate::interpret::Machine::enter_trace_span] returns [EnteredTraceSpan], on which you can call +/// [EnteredTraceSpan::or_if_tracing_disabled], to e.g. log a line as an alternative to the tracing +/// span for when tracing is disabled. For example: +/// ```rust +/// # use rustc_const_eval::enter_trace_span; +/// # use rustc_const_eval::interpret::EnteredTraceSpan; +/// # type M = rustc_const_eval::const_eval::CompileTimeMachine<'static>; +/// let _trace = enter_trace_span!(M, step::eval_statement) +/// .or_if_tracing_disabled(|| tracing::info!("eval_statement")); +/// ``` +#[macro_export] +macro_rules! enter_trace_span { + ($machine:ty, $name:ident :: $subname:ident $($tt:tt)*) => { + $crate::enter_trace_span!($machine, stringify!($name), $name = %stringify!($subname) $($tt)*) + }; + + ($machine:ty, $($tt:tt)*) => { + <$machine as $crate::interpret::Machine>::enter_trace_span(|| tracing::info_span!($($tt)*)) + }; +} diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index ed48f53c3105..89a3303eb390 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -449,7 +449,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { ) -> InterpResult<'tcx> { let tail = self.ecx.tcx.struct_tail_for_codegen(pointee.ty, self.ecx.typing_env); match tail.kind() { - ty::Dynamic(data, _, ty::Dyn) => { + ty::Dynamic(data, _) => { let vtable = meta.unwrap_meta().to_pointer(self.ecx)?; // Make sure it is a genuine vtable pointer for the right trait. try_validation!( @@ -511,7 +511,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { CheckInAllocMsg::Dereferenceable, // will anyway be replaced by validity message ), self.path, - Ub(DanglingIntPointer { addr: 0, .. }) => NullPtr { ptr_kind }, + Ub(DanglingIntPointer { addr: 0, .. }) => NullPtr { ptr_kind, maybe: false }, Ub(DanglingIntPointer { addr: i, .. }) => DanglingPtrNoProvenance { ptr_kind, // FIXME this says "null pointer" when null but we need translate @@ -538,8 +538,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { ); // Make sure this is non-null. We checked dereferenceability above, but if `size` is zero // that does not imply non-null. - if self.ecx.scalar_may_be_null(Scalar::from_maybe_pointer(place.ptr(), self.ecx))? { - throw_validation_failure!(self.path, NullPtr { ptr_kind }) + let scalar = Scalar::from_maybe_pointer(place.ptr(), self.ecx); + if self.ecx.scalar_may_be_null(scalar)? { + let maybe = !M::Provenance::OFFSET_IS_ADDR && matches!(scalar, Scalar::Ptr(..)); + throw_validation_failure!(self.path, NullPtr { ptr_kind, maybe }) } // Do not allow references to uninhabited types. if place.layout.is_uninhabited() { @@ -755,9 +757,12 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { ); // FIXME: Check if the signature matches } else { - // Otherwise (for standalone Miri), we have to still check it to be non-null. + // Otherwise (for standalone Miri and for `-Zextra-const-ub-checks`), + // we have to still check it to be non-null. if self.ecx.scalar_may_be_null(scalar)? { - throw_validation_failure!(self.path, NullFnPtr); + let maybe = + !M::Provenance::OFFSET_IS_ADDR && matches!(scalar, Scalar::Ptr(..)); + throw_validation_failure!(self.path, NullFnPtr { maybe }); } } if self.reset_provenance_and_padding { @@ -819,10 +824,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { if start == 1 && end == max_value { // Only null is the niche. So make sure the ptr is NOT null. if self.ecx.scalar_may_be_null(scalar)? { - throw_validation_failure!( - self.path, - NullablePtrOutOfRange { range: valid_range, max_value } - ) + throw_validation_failure!(self.path, NonnullPtrMaybeNull) } else { return interp_ok(()); } @@ -949,7 +951,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> { let padding_size = offset - padding_cleared_until; let range = alloc_range(padding_start, padding_size); trace!("reset_padding on {}: resetting padding range {range:?}", mplace.layout.ty); - alloc.write_uninit(range)?; + alloc.write_uninit(range); } padding_cleared_until = offset + size; } @@ -1239,7 +1241,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt, if self.reset_provenance_and_padding { // We can't share this with above as above, we might be looking at read-only memory. let mut alloc = self.ecx.get_ptr_alloc_mut(mplace.ptr(), size)?.expect("we already excluded size 0"); - alloc.clear_provenance()?; + alloc.clear_provenance(); // Also, mark this as containing data, not padding. self.add_data_range(mplace.ptr(), size); } @@ -1259,9 +1261,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt, // When you extend this match, make sure to also add tests to // tests/ui/type/pattern_types/validity.rs(( match **pat { - // Range patterns are precisely reflected into `valid_range` and thus + // Range and non-null patterns are precisely reflected into `valid_range` and thus // handled fully by `visit_scalar` (called below). ty::PatternKind::Range { .. } => {}, + ty::PatternKind::NotNull => {}, // FIXME(pattern_types): check that the value is covered by one of the variants. // For now, we rely on layout computation setting the scalar's `valid_range` to @@ -1415,10 +1418,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { recursive: bool, reset_provenance_and_padding: bool, ) -> InterpResult<'tcx> { - let _span = enter_trace_span!( + let _trace = enter_trace_span!( M, "validate_operand", - "recursive={recursive}, reset_provenance_and_padding={reset_provenance_and_padding}, val={val:?}" + recursive, + reset_provenance_and_padding, + ?val, ); // Note that we *could* actually be in CTFE here with `-Zextra-const-ub-checks`, but it's diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index a27b66461315..b5de10c7dcd1 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -90,7 +90,7 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { // Special treatment for special types, where the (static) layout is not sufficient. match *ty.kind() { // If it is a trait object, switch to the real type that was used to create it. - ty::Dynamic(data, _, ty::Dyn) => { + ty::Dynamic(data, _) => { // Dyn types. This is unsized, and the actual dynamic type of the data is given by the // vtable stored in the place metadata. // unsized values are never immediate, so we can assert_mem_place @@ -121,25 +121,24 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { // `Box` has two fields: the pointer we care about, and the allocator. assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields"); - let (unique_ptr, alloc) = ( - self.ecx().project_field(v, FieldIdx::ZERO)?, - self.ecx().project_field(v, FieldIdx::ONE)?, - ); + let [unique_ptr, alloc] = + self.ecx().project_fields(v, [FieldIdx::ZERO, FieldIdx::ONE])?; + // Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`... // (which means another 2 fields, the second of which is a `PhantomData`) assert_eq!(unique_ptr.layout().fields.count(), 2); - let (nonnull_ptr, phantom) = ( - self.ecx().project_field(&unique_ptr, FieldIdx::ZERO)?, - self.ecx().project_field(&unique_ptr, FieldIdx::ONE)?, - ); + let [nonnull_ptr, phantom] = + self.ecx().project_fields(&unique_ptr, [FieldIdx::ZERO, FieldIdx::ONE])?; assert!( phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()), "2nd field of `Unique` should be PhantomData but is {:?}", phantom.layout().ty, ); + // ... that contains a `NonNull`... (gladly, only a single field here) assert_eq!(nonnull_ptr.layout().fields.count(), 1); let raw_ptr = self.ecx().project_field(&nonnull_ptr, FieldIdx::ZERO)?; // the actual raw ptr + // ... whose only field finally is a raw ptr we can dereference. self.visit_box(ty, &raw_ptr)?; diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index bf7a79dcb20f..9c0ec4e51a0c 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -2,6 +2,7 @@ #![allow(internal_features)] #![allow(rustc::diagnostic_outside_of_impl)] #![doc(rust_logo)] +#![feature(array_try_map)] #![feature(assert_matches)] #![feature(box_patterns)] #![feature(decl_macro)] @@ -9,7 +10,6 @@ #![feature(never_type)] #![feature(rustdoc_internals)] #![feature(slice_ptr_get)] -#![feature(strict_overflow_ops)] #![feature(trait_alias)] #![feature(try_blocks)] #![feature(unqualified_local_imports)] diff --git a/compiler/rustc_const_eval/src/util/alignment.rs b/compiler/rustc_const_eval/src/util/alignment.rs index 9507b24f603e..9aafc7efd8a6 100644 --- a/compiler/rustc_const_eval/src/util/alignment.rs +++ b/compiler/rustc_const_eval/src/util/alignment.rs @@ -37,7 +37,7 @@ where debug!( "is_disaligned({:?}) - align = {}, packed = {}; not disaligned", place, - layout.align.abi.bytes(), + layout.align.bytes(), pack.bytes() ); false diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs index c437934eaabe..4e7c8310007b 100644 --- a/compiler/rustc_const_eval/src/util/caller_location.rs +++ b/compiler/rustc_const_eval/src/util/caller_location.rs @@ -42,12 +42,12 @@ fn alloc_caller_location<'tcx>( let location = ecx.allocate(loc_layout, MemoryKind::CallerLocation).unwrap(); // Initialize fields. - ecx.write_immediate(filename, &ecx.project_field(&location, FieldIdx::from_u32(0)).unwrap()) - .expect("writing to memory we just allocated cannot fail"); - ecx.write_scalar(line, &ecx.project_field(&location, FieldIdx::from_u32(1)).unwrap()) - .expect("writing to memory we just allocated cannot fail"); - ecx.write_scalar(col, &ecx.project_field(&location, FieldIdx::from_u32(2)).unwrap()) + let [filename_field, line_field, col_field] = + ecx.project_fields(&location, [0, 1, 2].map(FieldIdx::from_u32)).unwrap(); + ecx.write_immediate(filename, &filename_field) .expect("writing to memory we just allocated cannot fail"); + ecx.write_scalar(line, &line_field).expect("writing to memory we just allocated cannot fail"); + ecx.write_scalar(col, &col_field).expect("writing to memory we just allocated cannot fail"); location } @@ -61,7 +61,7 @@ pub(crate) fn const_caller_location_provider( trace!("const_caller_location: {}:{}:{}", file, line, col); let mut ecx = mk_eval_cx_to_read_const_val( tcx, - rustc_span::DUMMY_SP, // FIXME: use a proper span here? + rustc_span::DUMMY_SP, // This interpreter cannot fail, so the span is irrelevant. ty::TypingEnv::fully_monomorphized(), CanAccessMutGlobal::No, ); diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs index b1f295987505..34bd8423e8e1 100644 --- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs +++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs @@ -59,7 +59,7 @@ fn check_validity_requirement_strict<'tcx>( if kind == ValidityRequirement::Zero { cx.write_bytes_ptr( allocated.ptr(), - std::iter::repeat(0_u8).take(ty.layout.size().bytes_usize()), + std::iter::repeat_n(0_u8, ty.layout.size().bytes_usize()), ) .expect("failed to write bytes for zero valid check"); } @@ -129,7 +129,7 @@ fn check_validity_requirement_lax<'tcx>( if let Some(pointee) = this.ty.builtin_deref(false) { let pointee = cx.layout_of(pointee)?; // We need to ensure that the LLVM attributes `aligned` and `dereferenceable(size)` are satisfied. - if pointee.align.abi.bytes() > 1 { + if pointee.align.bytes() > 1 { // 0x01-filling is not aligned. return Ok(false); } diff --git a/compiler/rustc_const_eval/src/util/compare_types.rs b/compiler/rustc_const_eval/src/util/compare_types.rs index 9eed1a20f152..8db056ed8737 100644 --- a/compiler/rustc_const_eval/src/util/compare_types.rs +++ b/compiler/rustc_const_eval/src/util/compare_types.rs @@ -43,5 +43,5 @@ pub fn relate_types<'tcx>( Ok(()) => {} Err(_) => return false, }; - ocx.select_all_or_error().is_empty() + ocx.evaluate_obligations_error_on_ambiguity().is_empty() } diff --git a/compiler/rustc_const_eval/src/util/type_name.rs b/compiler/rustc_const_eval/src/util/type_name.rs index e8f2728a7728..db651811551f 100644 --- a/compiler/rustc_const_eval/src/util/type_name.rs +++ b/compiler/rustc_const_eval/src/util/type_name.rs @@ -1,24 +1,27 @@ use std::fmt::Write; use rustc_data_structures::intern::Interned; -use rustc_hir::def_id::CrateNum; +use rustc_hir::def_id::{CrateNum, DefId}; use rustc_hir::definitions::DisambiguatedDefPathData; use rustc_middle::bug; -use rustc_middle::ty::print::{PrettyPrinter, Print, PrintError, Printer}; -use rustc_middle::ty::{self, GenericArg, GenericArgKind, Ty, TyCtxt}; +use rustc_middle::ty::print::{PrettyPrinter, PrintError, Printer}; +use rustc_middle::ty::{self, GenericArg, Ty, TyCtxt}; -struct AbsolutePathPrinter<'tcx> { +struct TypeNamePrinter<'tcx> { tcx: TyCtxt<'tcx>, path: String, } -impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { +impl<'tcx> Printer<'tcx> for TypeNamePrinter<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } fn print_region(&mut self, _region: ty::Region<'_>) -> Result<(), PrintError> { - Ok(()) + // FIXME: most regions have been erased by the time this code runs. + // Just printing `'_` is a bit hacky but gives mostly good results, and + // doing better is difficult. See `should_print_optional_region`. + write!(self, "'_") } fn print_type(&mut self, ty: Ty<'tcx>) -> Result<(), PrintError> { @@ -38,7 +41,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { | ty::FnPtr(..) | ty::Never | ty::Tuple(_) - | ty::Dynamic(_, _, _) + | ty::Dynamic(_, _) | ty::UnsafeBinder(_) => self.pretty_print_type(ty), // Placeholders (all printed as `_` to uniformize them). @@ -73,27 +76,26 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { self.pretty_print_dyn_existential(predicates) } - fn path_crate(&mut self, cnum: CrateNum) -> Result<(), PrintError> { + fn print_crate_name(&mut self, cnum: CrateNum) -> Result<(), PrintError> { self.path.push_str(self.tcx.crate_name(cnum).as_str()); Ok(()) } - fn path_qualified( + fn print_path_with_qualified( &mut self, self_ty: Ty<'tcx>, trait_ref: Option>, ) -> Result<(), PrintError> { - self.pretty_path_qualified(self_ty, trait_ref) + self.pretty_print_path_with_qualified(self_ty, trait_ref) } - fn path_append_impl( + fn print_path_with_impl( &mut self, print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>, - _disambiguated_data: &DisambiguatedDefPathData, self_ty: Ty<'tcx>, trait_ref: Option>, ) -> Result<(), PrintError> { - self.pretty_path_append_impl( + self.pretty_print_path_with_impl( |cx| { print_prefix(cx)?; @@ -106,7 +108,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { ) } - fn path_append( + fn print_path_with_simple( &mut self, print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>, disambiguated_data: &DisambiguatedDefPathData, @@ -118,38 +120,60 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> { Ok(()) } - fn path_generic_args( + fn print_path_with_generic_args( &mut self, print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>, args: &[GenericArg<'tcx>], ) -> Result<(), PrintError> { print_prefix(self)?; - let args = - args.iter().cloned().filter(|arg| !matches!(arg.kind(), GenericArgKind::Lifetime(_))); - if args.clone().next().is_some() { - self.generic_delimiters(|cx| cx.comma_sep(args)) + if !args.is_empty() { + self.generic_delimiters(|cx| cx.comma_sep(args.iter().copied())) } else { Ok(()) } } + + fn print_coroutine_with_kind( + &mut self, + def_id: DefId, + parent_args: &'tcx [GenericArg<'tcx>], + kind: Ty<'tcx>, + ) -> Result<(), PrintError> { + self.print_def_path(def_id, parent_args)?; + + let ty::Coroutine(_, args) = self.tcx.type_of(def_id).instantiate_identity().kind() else { + // Could be `ty::Error`. + return Ok(()); + }; + + let default_kind = args.as_coroutine().kind_ty(); + + match kind.to_opt_closure_kind() { + _ if kind == default_kind => { + // No need to mark the closure if it's the deduced coroutine kind. + } + Some(ty::ClosureKind::Fn) | None => { + // Should never happen. Just don't mark anything rather than panicking. + } + Some(ty::ClosureKind::FnMut) => self.path.push_str("::{{call_mut}}"), + Some(ty::ClosureKind::FnOnce) => self.path.push_str("::{{call_once}}"), + } + + Ok(()) + } } -impl<'tcx> PrettyPrinter<'tcx> for AbsolutePathPrinter<'tcx> { - fn should_print_region(&self, _region: ty::Region<'_>) -> bool { - false - } - fn comma_sep(&mut self, mut elems: impl Iterator) -> Result<(), PrintError> - where - T: Print<'tcx, Self>, - { - if let Some(first) = elems.next() { - first.print(self)?; - for elem in elems { - self.path.push_str(", "); - elem.print(self)?; - } +impl<'tcx> PrettyPrinter<'tcx> for TypeNamePrinter<'tcx> { + fn should_print_optional_region(&self, region: ty::Region<'_>) -> bool { + // Bound regions are always printed (as `'_`), which gives some idea that they are special, + // even though the `for` is omitted by the pretty printer. + // E.g. `for<'a, 'b> fn(&'a u32, &'b u32)` is printed as "fn(&'_ u32, &'_ u32)". + let kind = region.kind(); + match region.kind() { + ty::ReErased | ty::ReEarlyParam(_) | ty::ReStatic => false, + ty::ReBound(..) => true, + _ => panic!("type_name unhandled region: {kind:?}"), } - Ok(()) } fn generic_delimiters( @@ -171,7 +195,7 @@ impl<'tcx> PrettyPrinter<'tcx> for AbsolutePathPrinter<'tcx> { } } -impl Write for AbsolutePathPrinter<'_> { +impl Write for TypeNamePrinter<'_> { fn write_str(&mut self, s: &str) -> std::fmt::Result { self.path.push_str(s); Ok(()) @@ -179,7 +203,7 @@ impl Write for AbsolutePathPrinter<'_> { } pub fn type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> String { - let mut printer = AbsolutePathPrinter { tcx, path: String::new() }; - printer.print_type(ty).unwrap(); - printer.path + let mut p = TypeNamePrinter { tcx, path: String::new() }; + p.print_type(ty).unwrap(); + p.path } diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 17204883fb03..c8296e05f6bd 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -13,6 +13,7 @@ ena = "0.14.3" indexmap = "2.4.0" jobserver_crate = { version = "0.1.28", package = "jobserver" } measureme = "12.0.1" +parking_lot = "0.12" rustc-hash = "2.0.0" rustc-stable-hash = { version = "0.1.0", features = ["nightly"] } rustc_arena = { path = "../rustc_arena" } @@ -34,9 +35,6 @@ version = "0.15.2" default-features = false features = ["nightly"] # for may_dangle -[dependencies.parking_lot] -version = "0.12" - [target.'cfg(windows)'.dependencies.windows] version = "0.61.0" features = [ diff --git a/compiler/rustc_data_structures/src/frozen.rs b/compiler/rustc_data_structures/src/frozen.rs index 73190574667f..4a60d17de2ab 100644 --- a/compiler/rustc_data_structures/src/frozen.rs +++ b/compiler/rustc_data_structures/src/frozen.rs @@ -46,7 +46,7 @@ //! Frozen::freeze(new_bar)`). /// An owned immutable value. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Frozen(T); impl Frozen { diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs index 518817ea0f53..1882e6e835a4 100644 --- a/compiler/rustc_data_structures/src/graph/scc/mod.rs +++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs @@ -289,7 +289,7 @@ enum NodeState { #[derive(Copy, Clone, Debug)] enum WalkReturn { /// The walk found a cycle, but the entire component is not known to have - /// been fully walked yet. We only know the minimum depth of this + /// been fully walked yet. We only know the minimum depth of this /// component in a minimum spanning tree of the graph. This component /// is tentatively represented by the state of the first node of this /// cycle we met, which is at `min_depth`. diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 53178d09348d..132f04b05100 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -34,6 +34,7 @@ #![feature(sized_hierarchy)] #![feature(test)] #![feature(thread_id_value)] +#![feature(trusted_len)] #![feature(type_alias_impl_trait)] #![feature(unwrap_infallible)] // tidy-alphabetical-end @@ -43,6 +44,8 @@ use std::fmt; pub use atomic_ref::AtomicRef; pub use ena::{snapshot_vec, undo_log, unify}; pub use rustc_index::static_assert_size; +// Re-export some data-structure crates which are part of our public API. +pub use {either, indexmap, smallvec, thin_vec}; pub mod aligned; pub mod base_n; @@ -77,6 +80,7 @@ pub mod thinvec; pub mod thousands; pub mod transitive_relation; pub mod unhash; +pub mod union_find; pub mod unord; pub mod vec_cache; pub mod work_queue; diff --git a/compiler/rustc_data_structures/src/sorted_map.rs b/compiler/rustc_data_structures/src/sorted_map.rs index c002d47815b1..15e3e6ea4c32 100644 --- a/compiler/rustc_data_structures/src/sorted_map.rs +++ b/compiler/rustc_data_structures/src/sorted_map.rs @@ -1,6 +1,7 @@ use std::borrow::Borrow; use std::cmp::Ordering; use std::fmt::Debug; +use std::iter::TrustedLen; use std::mem; use std::ops::{Bound, Index, IndexMut, RangeBounds}; @@ -215,36 +216,40 @@ impl SortedMap { /// It is up to the caller to make sure that the elements are sorted by key /// and that there are no duplicates. #[inline] - pub fn insert_presorted(&mut self, elements: Vec<(K, V)>) { - if elements.is_empty() { + pub fn insert_presorted( + &mut self, + // We require `TrustedLen` to ensure that the `splice` below is actually efficient. + mut elements: impl Iterator + DoubleEndedIterator + TrustedLen, + ) { + let Some(first) = elements.next() else { return; - } + }; - debug_assert!(elements.array_windows().all(|[fst, snd]| fst.0 < snd.0)); - - let start_index = self.lookup_index_for(&elements[0].0); + let start_index = self.lookup_index_for(&first.0); let elements = match start_index { Ok(index) => { - let mut elements = elements.into_iter(); - self.data[index] = elements.next().unwrap(); - elements + self.data[index] = first; // overwrite first element + elements.chain(None) // insert the rest below } Err(index) => { - if index == self.data.len() || elements.last().unwrap().0 < self.data[index].0 { + let last = elements.next_back(); + if index == self.data.len() + || last.as_ref().is_none_or(|l| l.0 < self.data[index].0) + { // We can copy the whole range without having to mix with // existing elements. - self.data.splice(index..index, elements); + self.data + .splice(index..index, std::iter::once(first).chain(elements).chain(last)); return; } - let mut elements = elements.into_iter(); - self.data.insert(index, elements.next().unwrap()); - elements + self.data.insert(index, first); + elements.chain(last) // insert the rest below } }; - // Insert the rest + // Insert the rest. This is super inefficicent since each insertion copies the entire tail. for (k, v) in elements { self.insert(k, v); } diff --git a/compiler/rustc_data_structures/src/sorted_map/tests.rs b/compiler/rustc_data_structures/src/sorted_map/tests.rs index ea4d2f1feacc..17d0d3cb1701 100644 --- a/compiler/rustc_data_structures/src/sorted_map/tests.rs +++ b/compiler/rustc_data_structures/src/sorted_map/tests.rs @@ -171,7 +171,7 @@ fn test_insert_presorted_non_overlapping() { map.insert(2, 0); map.insert(8, 0); - map.insert_presorted(vec![(3, 0), (7, 0)]); + map.insert_presorted(vec![(3, 0), (7, 0)].into_iter()); let expected = vec![2, 3, 7, 8]; assert_eq!(keys(map), expected); @@ -183,7 +183,7 @@ fn test_insert_presorted_first_elem_equal() { map.insert(2, 2); map.insert(8, 8); - map.insert_presorted(vec![(2, 0), (7, 7)]); + map.insert_presorted(vec![(2, 0), (7, 7)].into_iter()); let expected = vec![(2, 0), (7, 7), (8, 8)]; assert_eq!(elements(map), expected); @@ -195,7 +195,7 @@ fn test_insert_presorted_last_elem_equal() { map.insert(2, 2); map.insert(8, 8); - map.insert_presorted(vec![(3, 3), (8, 0)]); + map.insert_presorted(vec![(3, 3), (8, 0)].into_iter()); let expected = vec![(2, 2), (3, 3), (8, 0)]; assert_eq!(elements(map), expected); @@ -207,7 +207,7 @@ fn test_insert_presorted_shuffle() { map.insert(2, 2); map.insert(7, 7); - map.insert_presorted(vec![(1, 1), (3, 3), (8, 8)]); + map.insert_presorted(vec![(1, 1), (3, 3), (8, 8)].into_iter()); let expected = vec![(1, 1), (2, 2), (3, 3), (7, 7), (8, 8)]; assert_eq!(elements(map), expected); @@ -219,7 +219,7 @@ fn test_insert_presorted_at_end() { map.insert(1, 1); map.insert(2, 2); - map.insert_presorted(vec![(3, 3), (8, 8)]); + map.insert_presorted(vec![(3, 3), (8, 8)].into_iter()); let expected = vec![(1, 1), (2, 2), (3, 3), (8, 8)]; assert_eq!(elements(map), expected); diff --git a/compiler/rustc_mir_transform/src/coverage/counters/union_find.rs b/compiler/rustc_data_structures/src/union_find.rs similarity index 92% rename from compiler/rustc_mir_transform/src/coverage/counters/union_find.rs rename to compiler/rustc_data_structures/src/union_find.rs index a826a953fa67..ef73cd7ab40a 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters/union_find.rs +++ b/compiler/rustc_data_structures/src/union_find.rs @@ -9,7 +9,7 @@ mod tests; /// Simple implementation of a union-find data structure, i.e. a disjoint-set /// forest. #[derive(Debug)] -pub(crate) struct UnionFind { +pub struct UnionFind { table: IndexVec>, } @@ -28,7 +28,7 @@ struct UnionFindEntry { impl UnionFind { /// Creates a new disjoint-set forest containing the keys `0..num_keys`. /// Initially, every key is part of its own one-element set. - pub(crate) fn new(num_keys: usize) -> Self { + pub fn new(num_keys: usize) -> Self { // Initially, every key is the root of its own set, so its parent is itself. Self { table: IndexVec::from_fn_n(|key| UnionFindEntry { parent: key, rank: 0 }, num_keys) } } @@ -38,7 +38,7 @@ impl UnionFind { /// /// Also updates internal data structures to make subsequent `find` /// operations faster. - pub(crate) fn find(&mut self, key: Key) -> Key { + pub fn find(&mut self, key: Key) -> Key { // Loop until we find a key that is its own parent. let mut curr = key; while let parent = self.table[curr].parent @@ -60,7 +60,7 @@ impl UnionFind { /// Merges the set containing `a` and the set containing `b` into one set. /// /// Returns the common root of both keys, after the merge. - pub(crate) fn unify(&mut self, a: Key, b: Key) -> Key { + pub fn unify(&mut self, a: Key, b: Key) -> Key { let mut a = self.find(a); let mut b = self.find(b); @@ -90,7 +90,7 @@ impl UnionFind { /// Takes a "snapshot" of the current state of this disjoint-set forest, in /// the form of a vector that directly maps each key to its current root. - pub(crate) fn snapshot(&mut self) -> IndexVec { + pub fn snapshot(&mut self) -> IndexVec { self.table.indices().map(|key| self.find(key)).collect() } } diff --git a/compiler/rustc_mir_transform/src/coverage/counters/union_find/tests.rs b/compiler/rustc_data_structures/src/union_find/tests.rs similarity index 100% rename from compiler/rustc_mir_transform/src/coverage/counters/union_find/tests.rs rename to compiler/rustc_data_structures/src/union_find/tests.rs diff --git a/compiler/rustc_driver/Cargo.toml b/compiler/rustc_driver/Cargo.toml index e3ee83512952..519097198276 100644 --- a/compiler/rustc_driver/Cargo.toml +++ b/compiler/rustc_driver/Cargo.toml @@ -10,3 +10,8 @@ crate-type = ["dylib"] # tidy-alphabetical-start rustc_driver_impl = { path = "../rustc_driver_impl" } # tidy-alphabetical-end + +[build-dependencies] +# tidy-alphabetical-start +rustc_windows_rc = { path = "../rustc_windows_rc" } +# tidy-alphabetical-end diff --git a/compiler/rustc_driver/build.rs b/compiler/rustc_driver/build.rs new file mode 100644 index 000000000000..ba44fe7a86ed --- /dev/null +++ b/compiler/rustc_driver/build.rs @@ -0,0 +1,21 @@ +use std::{env, path}; + +use rustc_windows_rc::{VersionInfoFileType, compile_windows_resource_file}; + +fn main() { + let target_os = env::var("CARGO_CFG_TARGET_OS"); + let target_env = env::var("CARGO_CFG_TARGET_ENV"); + if Ok("windows") == target_os.as_deref() && Ok("msvc") == target_env.as_deref() { + set_windows_dll_options(); + } else { + // Avoid rerunning the build script every time. + println!("cargo:rerun-if-changed=build.rs"); + } +} + +fn set_windows_dll_options() { + let stem = path::PathBuf::from("rustc_driver_resource"); + let file_description = "rustc_driver"; + let res_file = compile_windows_resource_file(&stem, file_description, VersionInfoFileType::Dll); + println!("cargo:rustc-link-arg={}", res_file.display()); +} diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml index 0d6b49607eb4..46efa50cff36 100644 --- a/compiler/rustc_driver_impl/Cargo.toml +++ b/compiler/rustc_driver_impl/Cargo.toml @@ -4,8 +4,8 @@ version = "0.0.0" edition = "2024" [dependencies] -jiff = { version = "0.2.5", default-features = false, features = ["std"] } # tidy-alphabetical-start +jiff = { version = "0.2.5", default-features = false, features = ["std"] } rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } rustc_ast_lowering = { path = "../rustc_ast_lowering" } @@ -74,6 +74,7 @@ ctrlc = "3.4.4" # tidy-alphabetical-start check_only = ['rustc_interface/check_only'] llvm = ['rustc_interface/llvm'] +llvm_enzyme = ['rustc_interface/llvm_enzyme'] max_level_info = ['rustc_log/max_level_info'] rustc_randomized_layouts = [ 'rustc_index/rustc_randomized_layouts', diff --git a/compiler/rustc_driver_impl/messages.ftl b/compiler/rustc_driver_impl/messages.ftl index 2c6a0291ac29..b62cdc35f513 100644 --- a/compiler/rustc_driver_impl/messages.ftl +++ b/compiler/rustc_driver_impl/messages.ftl @@ -14,7 +14,7 @@ driver_impl_ice_version = rustc {$version} running on {$triple} driver_impl_rlink_corrupt_file = corrupt metadata encountered in `{$file}` -driver_impl_rlink_empty_version_number = The input does not contain version number +driver_impl_rlink_empty_version_number = the input does not contain version number driver_impl_rlink_encoding_version_mismatch = .rlink file was produced with encoding version `{$version_array}`, but the current version is `{$rlink_version}` @@ -24,6 +24,6 @@ driver_impl_rlink_rustc_version_mismatch = .rlink file was produced by rustc ver driver_impl_rlink_unable_to_read = failed to read rlink file: `{$err}` -driver_impl_rlink_wrong_file_type = The input does not look like a .rlink file +driver_impl_rlink_wrong_file_type = the input does not look like a .rlink file driver_impl_unstable_feature_usage = cannot dump feature usage metrics: {$error} diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index f3ed60421056..c926a7c742a0 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -51,6 +51,7 @@ use rustc_lint::unerased_lint_store; use rustc_metadata::creader::MetadataLoader; use rustc_metadata::locator; use rustc_middle::ty::TyCtxt; +use rustc_parse::lexer::StripTokens; use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal}; use rustc_session::config::{ CG_OPTIONS, CrateType, ErrorOutputType, Input, OptionDesc, OutFileName, OutputType, Sysroot, @@ -205,7 +206,7 @@ impl Callbacks for TimePassesCallbacks { // time because it will mess up the --print output. See #64339. // self.time_passes = (config.opts.prints.is_empty() && config.opts.unstable_opts.time_passes) - .then(|| config.opts.unstable_opts.time_passes_format); + .then_some(config.opts.unstable_opts.time_passes_format); config.opts.trimmed_def_paths = true; } } @@ -268,7 +269,6 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send)) make_codegen_backend: None, registry: diagnostics_registry(), using_internal_features: &USING_INTERNAL_FEATURES, - expanded_args: args, }; callbacks.config(&mut config); @@ -438,8 +438,9 @@ fn make_input(early_dcx: &EarlyDiagCtxt, free_matches: &[String]) -> Option() + .expect("UNSTABLE_RUSTDOC_TEST_LINE needs to be a number"); FileName::doc_test_source_code(PathBuf::from(path), line) } Err(_) => FileName::anon_source_code(&input), @@ -473,8 +474,7 @@ fn handle_explain(early_dcx: &EarlyDiagCtxt, registry: Registry, code: &str, col let mut text = String::new(); // Slice off the leading newline and print. for line in description.lines() { - let indent_level = - line.find(|c: char| !c.is_whitespace()).unwrap_or_else(|| line.len()); + let indent_level = line.find(|c: char| !c.is_whitespace()).unwrap_or(line.len()); let dedented_line = &line[indent_level..]; if dedented_line.starts_with("```") { is_in_code_block = !is_in_code_block; @@ -520,11 +520,11 @@ fn show_md_content_with_pager(content: &str, color: ColorConfig) { }; // Try to prettify the raw markdown text. The result can be used by the pager or on stdout. - let pretty_data = { + let mut pretty_data = { let mdstream = markdown::MdStream::parse_str(content); let bufwtr = markdown::create_stdout_bufwtr(); - let mut mdbuf = bufwtr.buffer(); - if mdstream.write_termcolor_buf(&mut mdbuf).is_ok() { Some((bufwtr, mdbuf)) } else { None } + let mut mdbuf = Vec::new(); + if mdstream.write_anstream_buf(&mut mdbuf).is_ok() { Some((bufwtr, mdbuf)) } else { None } }; // Try to print via the pager, pretty output if possible. @@ -545,8 +545,8 @@ fn show_md_content_with_pager(content: &str, color: ColorConfig) { } // The pager failed. Try to print pretty output to stdout. - if let Some((bufwtr, mdbuf)) = &pretty_data - && bufwtr.print(&mdbuf).is_ok() + if let Some((bufwtr, mdbuf)) = &mut pretty_data + && bufwtr.write_all(&mdbuf).is_ok() { return; } @@ -597,8 +597,7 @@ fn process_rlink(sess: &Session, compiler: &interface::Compiler) { fn list_metadata(sess: &Session, metadata_loader: &dyn MetadataLoader) { match sess.io.input { - Input::File(ref ifile) => { - let path = &(*ifile); + Input::File(ref path) => { let mut v = Vec::new(); locator::list_file_metadata( &sess.target, @@ -667,6 +666,10 @@ fn print_crate_info( TargetSpecJson => { println_info!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap()); } + TargetSpecJsonSchema => { + let schema = rustc_target::spec::json_schema(); + println_info!("{}", serde_json::to_string_pretty(&schema).unwrap()); + } AllTargetSpecsJson => { let mut targets = BTreeMap::new(); for name in rustc_target::spec::TARGETS { @@ -683,7 +686,8 @@ fn print_crate_info( }; let t_outputs = rustc_interface::util::build_output_filenames(attrs, sess); let crate_name = passes::get_crate_name(sess, attrs); - let crate_types = collect_crate_types(sess, attrs); + let crate_types = + collect_crate_types(sess, &codegen_backend.supported_crate_types(sess), attrs); for &style in &crate_types { let fname = rustc_session::output::filename_for_input( sess, style, crate_name, &t_outputs, @@ -828,7 +832,7 @@ fn print_crate_info( SupportedCrateTypes => { let supported_crate_types = CRATE_TYPES .iter() - .filter(|(_, crate_type)| !invalid_output_for_target(&sess, *crate_type)) + .filter(|(_, crate_type)| !invalid_output_for_target(sess, *crate_type)) .filter(|(_, crate_type)| *crate_type != CrateType::Sdylib) .map(|(crate_type_sym, _)| *crate_type_sym) .collect::>(); @@ -1288,10 +1292,15 @@ fn warn_on_confusing_output_filename_flag( fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> { let mut parser = unwrap_or_emit_fatal(match &sess.io.input { - Input::File(file) => new_parser_from_file(&sess.psess, file, None), - Input::Str { name, input } => { - new_parser_from_source_str(&sess.psess, name.clone(), input.clone()) + Input::File(file) => { + new_parser_from_file(&sess.psess, file, StripTokens::ShebangAndFrontmatter, None) } + Input::Str { name, input } => new_parser_from_source_str( + &sess.psess, + name.clone(), + input.clone(), + StripTokens::ShebangAndFrontmatter, + ), }); parser.parse_inner_attributes() } @@ -1424,7 +1433,7 @@ pub fn install_ice_hook(bug_report_url: &'static str, extra_info: fn(&DiagCtxt)) eprintln!(); if let Some(ice_path) = ice_path() - && let Ok(mut out) = File::options().create(true).append(true).open(&ice_path) + && let Ok(mut out) = File::options().create(true).append(true).open(ice_path) { // The current implementation always returns `Some`. let location = info.location().unwrap(); @@ -1500,7 +1509,7 @@ fn report_ice( let file = if let Some(path) = ice_path() { // Create the ICE dump target file. - match crate::fs::File::options().create(true).append(true).open(&path) { + match crate::fs::File::options().create(true).append(true).open(path) { Ok(mut file) => { dcx.emit_note(session_diagnostics::IcePath { path: path.clone() }); if FIRST_PANIC.swap(false, Ordering::SeqCst) { diff --git a/compiler/rustc_driver_impl/src/print.rs b/compiler/rustc_driver_impl/src/print.rs index 70de55320f7a..3f107eb7a61a 100644 --- a/compiler/rustc_driver_impl/src/print.rs +++ b/compiler/rustc_driver_impl/src/print.rs @@ -14,7 +14,7 @@ macro_rules! safe_println { } pub(crate) fn print(args: fmt::Arguments<'_>) { - if let Err(_) = io::stdout().write_fmt(args) { + if io::stdout().write_fmt(args).is_err() { rustc_errors::FatalError.raise(); } } diff --git a/compiler/rustc_error_codes/src/error_codes/E0458.md b/compiler/rustc_error_codes/src/error_codes/E0458.md index 1b280cba44f5..651bc3788796 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0458.md +++ b/compiler/rustc_error_codes/src/error_codes/E0458.md @@ -1,8 +1,10 @@ +#### Note: this error code is no longer emitted by the compiler. + An unknown "kind" was specified for a link attribute. Erroneous code example: -```compile_fail,E0458 +```ignore (no longer emitted) #[link(kind = "wonderful_unicorn")] extern "C" {} // error: unknown kind: `wonderful_unicorn` ``` diff --git a/compiler/rustc_error_codes/src/error_codes/E0518.md b/compiler/rustc_error_codes/src/error_codes/E0518.md index f04329bc4e61..87dc231578ac 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0518.md +++ b/compiler/rustc_error_codes/src/error_codes/E0518.md @@ -1,9 +1,11 @@ +#### Note: this error code is no longer emitted by the compiler. + An `#[inline(..)]` attribute was incorrectly placed on something other than a function or method. Example of erroneous code: -```compile_fail,E0518 +```ignore (no longer emitted) #[inline(always)] struct Foo; diff --git a/compiler/rustc_error_codes/src/error_codes/E0562.md b/compiler/rustc_error_codes/src/error_codes/E0562.md index 95f038df56d6..af7b219fb120 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0562.md +++ b/compiler/rustc_error_codes/src/error_codes/E0562.md @@ -1,5 +1,4 @@ -Abstract return types (written `impl Trait` for some trait `Trait`) are only -allowed as function and inherent impl return types. +`impl Trait` is only allowed as a function return and argument type. Erroneous code example: @@ -14,7 +13,7 @@ fn main() { } ``` -Make sure `impl Trait` only appears in return-type position. +Make sure `impl Trait` appears in a function signature. ``` fn count_to_n(n: usize) -> impl Iterator { @@ -28,6 +27,6 @@ fn main() { } ``` -See [RFC 1522] for more details. +See the [reference] for more details on `impl Trait`. -[RFC 1522]: https://github.com/rust-lang/rfcs/blob/master/text/1522-conservative-impl-trait.md +[reference]: https://doc.rust-lang.org/stable/reference/types/impl-trait.html diff --git a/compiler/rustc_error_codes/src/error_codes/E0578.md b/compiler/rustc_error_codes/src/error_codes/E0578.md index fca89757287f..78fabe855bb6 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0578.md +++ b/compiler/rustc_error_codes/src/error_codes/E0578.md @@ -1,8 +1,10 @@ +#### Note: this error code is no longer emitted by the compiler. + A module cannot be found and therefore, the visibility cannot be determined. Erroneous code example: -```compile_fail,E0578,edition2018 +```ignore (no longer emitted) foo!(); pub (in ::Sea) struct Shark; // error! diff --git a/compiler/rustc_error_codes/src/error_codes/E0608.md b/compiler/rustc_error_codes/src/error_codes/E0608.md index d0ebc3a26f08..3c29484f575c 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0608.md +++ b/compiler/rustc_error_codes/src/error_codes/E0608.md @@ -1,5 +1,5 @@ -An attempt to use index on a type which doesn't implement the `std::ops::Index` -trait was performed. +Attempted to index a value whose type doesn't implement the +`std::ops::Index` trait. Erroneous code example: @@ -7,8 +7,8 @@ Erroneous code example: 0u8[2]; // error: cannot index into a value of type `u8` ``` -To be able to index into a type it needs to implement the `std::ops::Index` -trait. Example: +Only values with types that implement the `std::ops::Index` trait +can be indexed with square brackets. Example: ``` let v: Vec = vec![0, 1, 2, 3]; @@ -16,3 +16,10 @@ let v: Vec = vec![0, 1, 2, 3]; // The `Vec` type implements the `Index` trait so you can do: println!("{}", v[2]); ``` + +Tuples and structs are indexed with dot (`.`), not with brackets (`[]`), +and tuple element names are their positions: +```ignore(pseudo code) +// this (pseudo code) expression is true for any tuple: +tuple == (tuple.0, tuple.1, ...) +``` diff --git a/compiler/rustc_error_codes/src/error_codes/E0701.md b/compiler/rustc_error_codes/src/error_codes/E0701.md index 4965e6431059..e1be0e915f44 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0701.md +++ b/compiler/rustc_error_codes/src/error_codes/E0701.md @@ -1,9 +1,11 @@ +#### Note: this error code is no longer emitted by the compiler. + This error indicates that a `#[non_exhaustive]` attribute was incorrectly placed on something other than a struct or enum. Erroneous code example: -```compile_fail,E0701 +```ignore (no longer emitted) #[non_exhaustive] trait Foo { } ``` diff --git a/compiler/rustc_error_codes/src/error_codes/E0719.md b/compiler/rustc_error_codes/src/error_codes/E0719.md index cd981db1058a..6aec38b42a3b 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0719.md +++ b/compiler/rustc_error_codes/src/error_codes/E0719.md @@ -1,4 +1,4 @@ -An associated type value was specified more than once. +An associated item was specified more than once in a trait object. Erroneous code example: @@ -7,21 +7,15 @@ trait FooTrait {} trait BarTrait {} // error: associated type `Item` in trait `Iterator` is specified twice -struct Foo> { f: T } +type Foo = dyn Iterator; ``` -`Item` in trait `Iterator` cannot be specified multiple times for struct `Foo`. -To fix this, create a new trait that is a combination of the desired traits and -specify the associated type with the new trait. +To fix this, remove the duplicate specifier: Corrected example: ``` -trait FooTrait {} -trait BarTrait {} -trait FooBarTrait: FooTrait + BarTrait {} - -struct Foo> { f: T } // ok! +type Foo = dyn Iterator; // ok! ``` For more information about associated types, see [the book][bk-at]. For more diff --git a/compiler/rustc_error_codes/src/error_codes/E0739.md b/compiler/rustc_error_codes/src/error_codes/E0739.md index 406d3d52779d..5403405ca9dc 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0739.md +++ b/compiler/rustc_error_codes/src/error_codes/E0739.md @@ -1,8 +1,10 @@ +#### Note: this error code is no longer emitted by the compiler. + `#[track_caller]` must be applied to a function Erroneous code example: -```compile_fail,E0739 +```ignore (no longer emitted) #[track_caller] struct Bar { a: u8, diff --git a/compiler/rustc_error_codes/src/error_codes/E0755.md b/compiler/rustc_error_codes/src/error_codes/E0755.md index b67f078c78ec..bd93626a8db4 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0755.md +++ b/compiler/rustc_error_codes/src/error_codes/E0755.md @@ -1,8 +1,10 @@ +#### Note: this error code is no longer emitted by the compiler. + The `ffi_pure` attribute was used on a non-foreign function. Erroneous code example: -```compile_fail,E0755 +```ignore (no longer emitted) #![feature(ffi_pure)] #[unsafe(ffi_pure)] // error! diff --git a/compiler/rustc_error_codes/src/error_codes/E0756.md b/compiler/rustc_error_codes/src/error_codes/E0756.md index aadde038d12c..daafc2a5ac09 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0756.md +++ b/compiler/rustc_error_codes/src/error_codes/E0756.md @@ -1,9 +1,11 @@ +#### Note: this error code is no longer emitted by the compiler. + The `ffi_const` attribute was used on something other than a foreign function declaration. Erroneous code example: -```compile_fail,E0756 +```ignore (no longer emitted) #![feature(ffi_const)] #[unsafe(ffi_const)] // error! diff --git a/compiler/rustc_error_codes/src/error_codes/E0788.md b/compiler/rustc_error_codes/src/error_codes/E0788.md index ba138aed2d12..1afa961f9b7c 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0788.md +++ b/compiler/rustc_error_codes/src/error_codes/E0788.md @@ -1,3 +1,5 @@ +#### Note: this error code is no longer emitted by the compiler. + A `#[coverage(off|on)]` attribute was found in a position where it is not allowed. @@ -10,7 +12,7 @@ Coverage attributes can be applied to: Example of erroneous code: -```compile_fail,E0788 +```ignore (no longer emitted) unsafe extern "C" { #[coverage(off)] fn foreign_fn(); diff --git a/compiler/rustc_error_codes/src/error_codes/E0793.md b/compiler/rustc_error_codes/src/error_codes/E0793.md index ccd1b43bd194..2722dac104b7 100644 --- a/compiler/rustc_error_codes/src/error_codes/E0793.md +++ b/compiler/rustc_error_codes/src/error_codes/E0793.md @@ -1,4 +1,9 @@ -An unaligned reference to a field of a [packed] struct got created. +An unaligned reference to a field of a [packed] `struct` or `union` was created. + +The `#[repr(packed)]` attribute removes padding between fields, which can +cause fields to be stored at unaligned memory addresses. Creating references +to such fields violates Rust's memory safety guarantees and can lead to +undefined behavior in optimized code. Erroneous code example: @@ -45,9 +50,36 @@ unsafe { // For formatting, we can create a copy to avoid the direct reference. let copy = foo.field1; println!("{}", copy); + // Creating a copy can be written in a single line with curly braces. // (This is equivalent to the two lines above.) println!("{}", { foo.field1 }); + + // A reference to a field that will always be sufficiently aligned is safe: + println!("{}", foo.field2); +} +``` + +### Unions + +Although creating a reference to a `union` field is `unsafe`, this error +will still be triggered if the referenced field is not sufficiently +aligned. Use `addr_of!` and raw pointers in the same way as for struct fields. + +```compile_fail,E0793 +#[repr(packed)] +pub union Foo { + field1: u64, + field2: u8, +} + +unsafe { + let foo = Foo { field1: 0 }; + // Accessing the field directly is fine. + let val = foo.field1; + + // A reference to a packed union field causes an error. + let val = &foo.field1; // ERROR } ``` diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml index 0951859fa531..db22e065907e 100644 --- a/compiler/rustc_error_messages/Cargo.toml +++ b/compiler/rustc_error_messages/Cargo.toml @@ -7,10 +7,11 @@ edition = "2024" # tidy-alphabetical-start fluent-bundle = "0.16" fluent-syntax = "0.12" -icu_list = "1.2" -icu_locid = "1.2" -icu_provider_adapters = "1.2" +icu_list = { version = "2.0", default-features = false, features = ["alloc"] } +icu_locale = { version = "2.0", default-features = false } intl-memoizer = "0.5.1" +rustc_ast = { path = "../rustc_ast" } +rustc_ast_pretty = { path = "../rustc_ast_pretty" } rustc_baked_icu_data = { path = "../rustc_baked_icu_data" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_macros = { path = "../rustc_macros" } diff --git a/compiler/rustc_error_messages/src/diagnostic_impls.rs b/compiler/rustc_error_messages/src/diagnostic_impls.rs new file mode 100644 index 000000000000..3b664cce5776 --- /dev/null +++ b/compiler/rustc_error_messages/src/diagnostic_impls.rs @@ -0,0 +1,205 @@ +use std::backtrace::Backtrace; +use std::borrow::Cow; +use std::fmt; +use std::num::ParseIntError; +use std::path::{Path, PathBuf}; +use std::process::ExitStatus; + +use rustc_ast as ast; +use rustc_ast_pretty::pprust; +use rustc_span::edition::Edition; + +use crate::{DiagArgValue, IntoDiagArg}; + +pub struct DiagArgFromDisplay<'a>(pub &'a dyn fmt::Display); + +impl IntoDiagArg for DiagArgFromDisplay<'_> { + fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { + self.0.to_string().into_diag_arg(path) + } +} + +impl<'a> From<&'a dyn fmt::Display> for DiagArgFromDisplay<'a> { + fn from(t: &'a dyn fmt::Display) -> Self { + DiagArgFromDisplay(t) + } +} + +impl<'a, T: fmt::Display> From<&'a T> for DiagArgFromDisplay<'a> { + fn from(t: &'a T) -> Self { + DiagArgFromDisplay(t) + } +} + +impl<'a, T: Clone + IntoDiagArg> IntoDiagArg for &'a T { + fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { + self.clone().into_diag_arg(path) + } +} + +#[macro_export] +macro_rules! into_diag_arg_using_display { + ($( $ty:ty ),+ $(,)?) => { + $( + impl $crate::IntoDiagArg for $ty { + fn into_diag_arg(self, path: &mut Option) -> $crate::DiagArgValue { + self.to_string().into_diag_arg(path) + } + } + )+ + } +} + +macro_rules! into_diag_arg_for_number { + ($( $ty:ty ),+ $(,)?) => { + $( + impl $crate::IntoDiagArg for $ty { + fn into_diag_arg(self, path: &mut Option) -> $crate::DiagArgValue { + // Convert to a string if it won't fit into `Number`. + #[allow(irrefutable_let_patterns)] + if let Ok(n) = TryInto::::try_into(self) { + $crate::DiagArgValue::Number(n) + } else { + self.to_string().into_diag_arg(path) + } + } + } + )+ + } +} + +into_diag_arg_using_display!( + ast::ParamKindOrd, + std::io::Error, + Box, + std::num::NonZero, + Edition, + rustc_span::Ident, + rustc_span::MacroRulesNormalizedIdent, + ParseIntError, + ExitStatus, +); + +into_diag_arg_for_number!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128, isize, usize); + +impl IntoDiagArg for bool { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + if self { + DiagArgValue::Str(Cow::Borrowed("true")) + } else { + DiagArgValue::Str(Cow::Borrowed("false")) + } + } +} + +impl IntoDiagArg for char { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(format!("{self:?}"))) + } +} + +impl IntoDiagArg for Vec { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::StrListSepByAnd( + self.into_iter().map(|c| Cow::Owned(format!("{c:?}"))).collect(), + ) + } +} + +impl IntoDiagArg for rustc_span::Symbol { + fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { + self.to_ident_string().into_diag_arg(path) + } +} + +impl<'a> IntoDiagArg for &'a str { + fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { + self.to_string().into_diag_arg(path) + } +} + +impl IntoDiagArg for String { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self)) + } +} + +impl<'a> IntoDiagArg for Cow<'a, str> { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self.into_owned())) + } +} + +impl<'a> IntoDiagArg for &'a Path { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self.display().to_string())) + } +} + +impl IntoDiagArg for PathBuf { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self.display().to_string())) + } +} + +impl IntoDiagArg for ast::Expr { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(pprust::expr_to_string(&self))) + } +} + +impl IntoDiagArg for ast::Path { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(pprust::path_to_string(&self))) + } +} + +impl IntoDiagArg for ast::token::Token { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(pprust::token_to_string(&self)) + } +} + +impl IntoDiagArg for ast::token::TokenKind { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(pprust::token_kind_to_string(&self)) + } +} + +impl IntoDiagArg for std::ffi::CString { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned())) + } +} + +impl IntoDiagArg for rustc_data_structures::small_c_str::SmallCStr { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned())) + } +} + +impl IntoDiagArg for ast::Visibility { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + let s = pprust::vis_to_string(&self); + let s = s.trim_end().to_string(); + DiagArgValue::Str(Cow::Owned(s)) + } +} + +impl IntoDiagArg for Backtrace { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::from(self.to_string())) + } +} + +impl IntoDiagArg for ast::util::parser::ExprPrecedence { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Number(self as i32) + } +} + +impl IntoDiagArg for ast::FloatTy { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::Borrowed(self.name_str())) + } +} diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs index 4b3ecad307fe..7b7843f6cf31 100644 --- a/compiler/rustc_error_messages/src/lib.rs +++ b/compiler/rustc_error_messages/src/lib.rs @@ -15,7 +15,6 @@ use fluent_bundle::FluentResource; pub use fluent_bundle::types::FluentType; pub use fluent_bundle::{self, FluentArgs, FluentError, FluentValue}; use fluent_syntax::parser::ParserError; -use icu_provider_adapters::fallback::{LocaleFallbackProvider, LocaleFallbacker}; use intl_memoizer::concurrent::IntlLangMemoizer; use rustc_data_structures::sync::{DynSend, IntoDynSyncSend}; use rustc_macros::{Decodable, Encodable}; @@ -23,6 +22,9 @@ use rustc_span::Span; use tracing::{instrument, trace}; pub use unic_langid::{LanguageIdentifier, langid}; +mod diagnostic_impls; +pub use diagnostic_impls::DiagArgFromDisplay; + pub type FluentBundle = IntoDynSyncSend>; @@ -512,8 +514,8 @@ impl From> for MultiSpan { } } -fn icu_locale_from_unic_langid(lang: LanguageIdentifier) -> Option { - icu_locid::Locale::try_from_bytes(lang.to_string().as_bytes()).ok() +fn icu_locale_from_unic_langid(lang: LanguageIdentifier) -> Option { + icu_locale::Locale::try_from_str(&lang.to_string()).ok() } pub fn fluent_value_from_str_list_sep_by_and(l: Vec>) -> FluentValue<'_> { @@ -565,21 +567,15 @@ pub fn fluent_value_from_str_list_sep_by_and(l: Vec>) -> FluentValu where Self: Sized, { - let baked_data_provider = rustc_baked_icu_data::baked_data_provider(); - let locale_fallbacker = - LocaleFallbacker::try_new_with_any_provider(&baked_data_provider) - .expect("Failed to create fallback provider"); - let data_provider = - LocaleFallbackProvider::new_with_fallbacker(baked_data_provider, locale_fallbacker); let locale = icu_locale_from_unic_langid(lang) .unwrap_or_else(|| rustc_baked_icu_data::supported_locales::EN); - let list_formatter = - icu_list::ListFormatter::try_new_and_with_length_with_any_provider( - &data_provider, - &locale.into(), - icu_list::ListLength::Wide, - ) - .expect("Failed to create list formatter"); + let list_formatter = icu_list::ListFormatter::try_new_and_unstable( + &rustc_baked_icu_data::BakedDataProvider, + locale.into(), + icu_list::options::ListFormatterOptions::default() + .with_length(icu_list::options::ListLength::Wide), + ) + .expect("Failed to create list formatter"); Ok(MemoizableListFormatter(list_formatter)) } @@ -589,3 +585,53 @@ pub fn fluent_value_from_str_list_sep_by_and(l: Vec>) -> FluentValu FluentValue::Custom(Box::new(FluentStrListSepByAnd(l))) } + +/// Simplified version of `FluentArg` that can implement `Encodable` and `Decodable`. Collection of +/// `DiagArg` are converted to `FluentArgs` (consuming the collection) at the start of diagnostic +/// emission. +pub type DiagArg<'iter> = (&'iter DiagArgName, &'iter DiagArgValue); + +/// Name of a diagnostic argument. +pub type DiagArgName = Cow<'static, str>; + +/// Simplified version of `FluentValue` that can implement `Encodable` and `Decodable`. Converted +/// to a `FluentValue` by the emitter to be used in diagnostic translation. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)] +pub enum DiagArgValue { + Str(Cow<'static, str>), + // This gets converted to a `FluentNumber`, which is an `f64`. An `i32` + // safely fits in an `f64`. Any integers bigger than that will be converted + // to strings in `into_diag_arg` and stored using the `Str` variant. + Number(i32), + StrListSepByAnd(Vec>), +} + +/// Converts a value of a type into a `DiagArg` (typically a field of an `Diag` struct). +/// Implemented as a custom trait rather than `From` so that it is implemented on the type being +/// converted rather than on `DiagArgValue`, which enables types from other `rustc_*` crates to +/// implement this. +pub trait IntoDiagArg { + /// Convert `Self` into a `DiagArgValue` suitable for rendering in a diagnostic. + /// + /// It takes a `path` where "long values" could be written to, if the `DiagArgValue` is too big + /// for displaying on the terminal. This path comes from the `Diag` itself. When rendering + /// values that come from `TyCtxt`, like `Ty<'_>`, they can use `TyCtxt::short_string`. If a + /// value has no shortening logic that could be used, the argument can be safely ignored. + fn into_diag_arg(self, path: &mut Option) -> DiagArgValue; +} + +impl IntoDiagArg for DiagArgValue { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + self + } +} + +impl From for FluentValue<'static> { + fn from(val: DiagArgValue) -> Self { + match val { + DiagArgValue::Str(s) => From::from(s), + DiagArgValue::Number(n) => From::from(n), + DiagArgValue::StrListSepByAnd(l) => fluent_value_from_str_list_sep_by_and(l), + } + } +} diff --git a/compiler/rustc_errors/Cargo.toml b/compiler/rustc_errors/Cargo.toml index c4181a62a35d..6ade87ea3b25 100644 --- a/compiler/rustc_errors/Cargo.toml +++ b/compiler/rustc_errors/Cargo.toml @@ -5,29 +5,25 @@ edition = "2024" [dependencies] # tidy-alphabetical-start -annotate-snippets = "0.11" +annotate-snippets = "0.12.7" +anstream = "0.6.20" +anstyle = "1.0.13" derive_setters = "0.1.6" rustc_abi = { path = "../rustc_abi" } rustc_ast = { path = "../rustc_ast" } -rustc_ast_pretty = { path = "../rustc_ast_pretty" } -rustc_attr_data_structures = { path = "../rustc_attr_data_structures" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_error_codes = { path = "../rustc_error_codes" } rustc_error_messages = { path = "../rustc_error_messages" } rustc_fluent_macro = { path = "../rustc_fluent_macro" } rustc_hashes = { path = "../rustc_hashes" } -rustc_hir = { path = "../rustc_hir" } rustc_index = { path = "../rustc_index" } rustc_lexer = { path = "../rustc_lexer" } rustc_lint_defs = { path = "../rustc_lint_defs" } rustc_macros = { path = "../rustc_macros" } rustc_serialize = { path = "../rustc_serialize" } rustc_span = { path = "../rustc_span" } -rustc_target = { path = "../rustc_target" } -rustc_type_ir = { path = "../rustc_type_ir" } -serde = { version = "1.0.125", features = [ "derive" ] } +serde = { version = "1.0.125", features = ["derive"] } serde_json = "1.0.59" -termcolor = "1.2.0" termize = "0.2" tracing = "0.1" # tidy-alphabetical-end diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs index 2eb3c23259ff..854e3ddf15e4 100644 --- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs +++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs @@ -5,32 +5,70 @@ //! //! [annotate_snippets]: https://docs.rs/crate/annotate-snippets/ +use std::borrow::Cow; +use std::error::Report; +use std::fmt::Debug; +use std::io; +use std::io::Write; use std::sync::Arc; -use annotate_snippets::{Renderer, Snippet}; -use rustc_error_messages::FluentArgs; -use rustc_span::SourceFile; +use annotate_snippets::renderer::DEFAULT_TERM_WIDTH; +use annotate_snippets::{AnnotationKind, Group, Origin, Padding, Patch, Renderer, Snippet}; +use anstream::ColorChoice; +use derive_setters::Setters; +use rustc_data_structures::sync::IntoDynSyncSend; +use rustc_error_messages::{FluentArgs, SpanLabel}; +use rustc_lint_defs::pluralize; use rustc_span::source_map::SourceMap; +use rustc_span::{BytePos, FileName, Pos, SourceFile, Span}; +use tracing::debug; -use crate::emitter::FileWithAnnotatedLines; +use crate::emitter::{ + ConfusionType, Destination, MAX_SUGGESTIONS, OutputTheme, detect_confusion_type, is_different, + normalize_whitespace, should_show_source_code, +}; use crate::registry::Registry; -use crate::snippet::Line; use crate::translation::{Translator, to_fluent_args}; use crate::{ CodeSuggestion, DiagInner, DiagMessage, Emitter, ErrCode, Level, MultiSpan, Style, Subdiag, + SuggestionStyle, TerminalUrl, }; /// Generates diagnostics using annotate-snippet +#[derive(Setters)] pub struct AnnotateSnippetEmitter { - source_map: Option>, + #[setters(skip)] + dst: IntoDynSyncSend, + sm: Option>, + #[setters(skip)] translator: Translator, - - /// If true, hides the longer explanation text short_message: bool, - /// If true, will normalize line numbers with `LL` to prevent noise in UI test diffs. ui_testing: bool, + ignored_directories_in_source_blocks: Vec, + diagnostic_width: Option, macro_backtrace: bool, + track_diagnostics: bool, + terminal_url: TerminalUrl, + theme: OutputTheme, +} + +impl Debug for AnnotateSnippetEmitter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AnnotateSnippetEmitter") + .field("short_message", &self.short_message) + .field("ui_testing", &self.ui_testing) + .field( + "ignored_directories_in_source_blocks", + &self.ignored_directories_in_source_blocks, + ) + .field("diagnostic_width", &self.diagnostic_width) + .field("macro_backtrace", &self.macro_backtrace) + .field("track_diagnostics", &self.track_diagnostics) + .field("terminal_url", &self.terminal_url) + .field("theme", &self.theme) + .finish() + } } impl Emitter for AnnotateSnippetEmitter { @@ -38,6 +76,10 @@ impl Emitter for AnnotateSnippetEmitter { fn emit_diagnostic(&mut self, mut diag: DiagInner, _registry: &Registry) { let fluent_args = to_fluent_args(diag.args.iter()); + if self.track_diagnostics && diag.span.has_primary_spans() && !diag.span.is_dummy() { + diag.children.insert(0, diag.emitted_at_sub_diag()); + } + let mut suggestions = diag.suggestions.unwrap_tag(); self.primary_span_formatted(&mut diag.span, &mut suggestions, &fluent_args); @@ -55,12 +97,12 @@ impl Emitter for AnnotateSnippetEmitter { &diag.code, &diag.span, &diag.children, - &suggestions, + suggestions, ); } fn source_map(&self) -> Option<&SourceMap> { - self.source_map.as_deref() + self.sm.as_deref() } fn should_show_explain(&self) -> bool { @@ -70,128 +112,648 @@ impl Emitter for AnnotateSnippetEmitter { fn translator(&self) -> &Translator { &self.translator } + + fn supports_color(&self) -> bool { + false + } } -/// Provides the source string for the given `line` of `file` -fn source_string(file: Arc, line: &Line) -> String { - file.get_line(line.line_index - 1).map(|a| a.to_string()).unwrap_or_default() -} - -/// Maps [`crate::Level`] to [`annotate_snippets::Level`] -fn annotation_level_for_level(level: Level) -> annotate_snippets::Level { +fn annotation_level_for_level(level: Level) -> annotate_snippets::level::Level<'static> { match level { - Level::Bug | Level::Fatal | Level::Error | Level::DelayedBug => { - annotate_snippets::Level::Error + Level::Bug | Level::DelayedBug => { + annotate_snippets::Level::ERROR.with_name("error: internal compiler error") } - Level::ForceWarning | Level::Warning => annotate_snippets::Level::Warning, - Level::Note | Level::OnceNote => annotate_snippets::Level::Note, - Level::Help | Level::OnceHelp => annotate_snippets::Level::Help, - // FIXME(#59346): Not sure how to map this level - Level::FailureNote => annotate_snippets::Level::Error, + Level::Fatal | Level::Error => annotate_snippets::level::ERROR, + Level::ForceWarning | Level::Warning => annotate_snippets::Level::WARNING, + Level::Note | Level::OnceNote => annotate_snippets::Level::NOTE, + Level::Help | Level::OnceHelp => annotate_snippets::Level::HELP, + Level::FailureNote => annotate_snippets::Level::NOTE.no_name(), Level::Allow => panic!("Should not call with Allow"), Level::Expect => panic!("Should not call with Expect"), } } impl AnnotateSnippetEmitter { - pub fn new( - source_map: Option>, - translator: Translator, - short_message: bool, - macro_backtrace: bool, - ) -> Self { - Self { source_map, translator, short_message, ui_testing: false, macro_backtrace } - } - - /// Allows to modify `Self` to enable or disable the `ui_testing` flag. - /// - /// If this is set to true, line numbers will be normalized as `LL` in the output. - pub fn ui_testing(mut self, ui_testing: bool) -> Self { - self.ui_testing = ui_testing; - self + pub fn new(dst: Destination, translator: Translator) -> Self { + Self { + dst: IntoDynSyncSend(dst), + sm: None, + translator, + short_message: false, + ui_testing: false, + ignored_directories_in_source_blocks: Vec::new(), + diagnostic_width: None, + macro_backtrace: false, + track_diagnostics: false, + terminal_url: TerminalUrl::No, + theme: OutputTheme::Ascii, + } } fn emit_messages_default( &mut self, level: &Level, - messages: &[(DiagMessage, Style)], + msgs: &[(DiagMessage, Style)], args: &FluentArgs<'_>, code: &Option, msp: &MultiSpan, - _children: &[Subdiag], - _suggestions: &[CodeSuggestion], + children: &[Subdiag], + suggestions: Vec, ) { - let message = self.translator.translate_messages(messages, args); - if let Some(source_map) = &self.source_map { - // Make sure our primary file comes first - let primary_lo = if let Some(primary_span) = msp.primary_span().as_ref() { - if primary_span.is_dummy() { - // FIXME(#59346): Not sure when this is the case and what - // should be done if it happens - return; - } else { - source_map.lookup_char_pos(primary_span.lo()) - } - } else { - // FIXME(#59346): Not sure when this is the case and what - // should be done if it happens - return; - }; - let mut annotated_files = FileWithAnnotatedLines::collect_annotations(self, args, msp); - if let Ok(pos) = - annotated_files.binary_search_by(|x| x.file.name.cmp(&primary_lo.file.name)) - { - annotated_files.swap(0, pos); - } - // owned: file name, line source, line index, annotations - type Owned = (String, String, usize, Vec); - let annotated_files: Vec = annotated_files - .into_iter() - .flat_map(|annotated_file| { - let file = annotated_file.file; - annotated_file - .lines - .into_iter() - .map(|line| { - // Ensure the source file is present before we try - // to load a string from it. - // FIXME(#115869): support -Z ignore-directory-in-diagnostics-source-blocks - source_map.ensure_source_file_source_present(&file); - ( - format!("{}", source_map.filename_for_diagnostics(&file.name)), - source_string(Arc::clone(&file), &line), - line.line_index, - line.annotations, - ) - }) - .collect::>() - }) - .collect(); - let code = code.map(|code| code.to_string()); + let renderer = self.renderer(); + let annotation_level = annotation_level_for_level(*level); - let snippets = - annotated_files.iter().map(|(file_name, source, line_index, annotations)| { - Snippet::source(source) - .line_start(*line_index) - .origin(file_name) - // FIXME(#59346): Not really sure when `fold` should be true or false - .fold(false) - .annotations(annotations.iter().map(|annotation| { - annotation_level_for_level(*level) - .span(annotation.start_col.display..annotation.end_col.display) - .label(annotation.label.as_deref().unwrap_or_default()) - })) - }); - let mut message = annotation_level_for_level(*level).title(&message).snippets(snippets); - if let Some(code) = code.as_deref() { - message = message.id(code) + // If at least one portion of the message is styled, we need to + // "pre-style" the message + let mut title = if msgs.iter().any(|(_, style)| style != &crate::Style::NoStyle) { + annotation_level + .clone() + .secondary_title(Cow::Owned(self.pre_style_msgs(msgs, *level, args))) + } else { + annotation_level.clone().primary_title(self.translator.translate_messages(msgs, args)) + }; + + if let Some(c) = code { + title = title.id(c.to_string()); + if let TerminalUrl::Yes = self.terminal_url { + title = title.id_url(format!("https://doc.rust-lang.org/error_codes/{c}.html")); } - // FIXME(#59346): Figure out if we can _always_ print to stderr or not. - // `emitter.rs` has the `Destination` enum that lists various possible output - // destinations. - let renderer = Renderer::plain().anonymized_line_numbers(self.ui_testing); - eprintln!("{}", renderer.render(message)) } - // FIXME(#59346): Is it ok to return None if there's no source_map? + + let mut report = vec![]; + let mut group = Group::with_title(title); + + // If we don't have span information, emit and exit + let Some(sm) = self.sm.as_ref() else { + group = group.elements(children.iter().map(|c| { + let msg = self.translator.translate_messages(&c.messages, args).to_string(); + let level = annotation_level_for_level(c.level); + level.message(msg) + })); + + report.push(group); + if let Err(e) = emit_to_destination( + renderer.render(&report), + level, + &mut self.dst, + self.short_message, + ) { + panic!("failed to emit error: {e}"); + } + return; + }; + + let mut file_ann = collect_annotations(args, msp, sm, &self.translator); + + // Make sure our primary file comes first + let primary_span = msp.primary_span().unwrap_or_default(); + if !primary_span.is_dummy() { + let primary_lo = sm.lookup_char_pos(primary_span.lo()); + if let Ok(pos) = file_ann.binary_search_by(|(f, _)| f.name.cmp(&primary_lo.file.name)) { + file_ann.swap(0, pos); + } + + for (file_idx, (file, annotations)) in file_ann.into_iter().enumerate() { + if should_show_source_code(&self.ignored_directories_in_source_blocks, sm, &file) { + if let Some(snippet) = self.annotated_snippet(annotations, &file.name, sm) { + group = group.element(snippet); + } + // we can't annotate anything if the source is unavailable. + } else if !self.short_message { + // We'll just print unannotated messages + group = self.unannotated_messages( + annotations, + &file.name, + sm, + file_idx, + &mut report, + group, + &annotation_level, + ); + // If this is the last annotation for a file, and + // this is the last file, and the first child is a + // "secondary" message, we need to add padding + // ╭▸ /rustc/FAKE_PREFIX/library/core/src/clone.rs:236:13 + // │ + // ├ note: the late bound lifetime parameter + // │ (<- It adds *this*) + // ╰ warning: this was previously accepted + if let Some(c) = children.first() + && (!c.span.has_primary_spans() && !c.span.has_span_labels()) + { + group = group.element(Padding); + } + } + } + } + + for c in children { + let level = annotation_level_for_level(c.level); + + // If at least one portion of the message is styled, we need to + // "pre-style" the message + let msg = if c.messages.iter().any(|(_, style)| style != &crate::Style::NoStyle) { + Cow::Owned(self.pre_style_msgs(&c.messages, c.level, args)) + } else { + self.translator.translate_messages(&c.messages, args) + }; + + // This is a secondary message with no span info + if !c.span.has_primary_spans() && !c.span.has_span_labels() { + group = group.element(level.clone().message(msg)); + continue; + } + + report.push(std::mem::replace( + &mut group, + Group::with_title(level.clone().secondary_title(msg)), + )); + + let mut file_ann = collect_annotations(args, &c.span, sm, &self.translator); + let primary_span = c.span.primary_span().unwrap_or_default(); + if !primary_span.is_dummy() { + let primary_lo = sm.lookup_char_pos(primary_span.lo()); + if let Ok(pos) = + file_ann.binary_search_by(|(f, _)| f.name.cmp(&primary_lo.file.name)) + { + file_ann.swap(0, pos); + } + } + + for (file_idx, (file, annotations)) in file_ann.into_iter().enumerate() { + if should_show_source_code(&self.ignored_directories_in_source_blocks, sm, &file) { + if let Some(snippet) = self.annotated_snippet(annotations, &file.name, sm) { + group = group.element(snippet); + } + // we can't annotate anything if the source is unavailable. + } else if !self.short_message { + // We'll just print unannotated messages + group = self.unannotated_messages( + annotations, + &file.name, + sm, + file_idx, + &mut report, + group, + &level, + ); + } + } + } + + let suggestions_expected = suggestions + .iter() + .filter(|s| { + matches!( + s.style, + SuggestionStyle::HideCodeInline + | SuggestionStyle::ShowCode + | SuggestionStyle::ShowAlways + ) + }) + .count(); + for suggestion in suggestions { + match suggestion.style { + SuggestionStyle::CompletelyHidden => { + // do not display this suggestion, it is meant only for tools + } + SuggestionStyle::HideCodeAlways => { + let msg = self + .translator + .translate_messages(&[(suggestion.msg.to_owned(), Style::HeaderMsg)], args); + group = group.element(annotate_snippets::Level::HELP.message(msg)); + } + SuggestionStyle::HideCodeInline + | SuggestionStyle::ShowCode + | SuggestionStyle::ShowAlways => { + let substitutions = suggestion + .substitutions + .into_iter() + .filter_map(|mut subst| { + // Suggestions coming from macros can have malformed spans. This is a heavy + // handed approach to avoid ICEs by ignoring the suggestion outright. + let invalid = + subst.parts.iter().any(|item| sm.is_valid_span(item.span).is_err()); + if invalid { + debug!("suggestion contains an invalid span: {:?}", subst); + } + + // Assumption: all spans are in the same file, and all spans + // are disjoint. Sort in ascending order. + subst.parts.sort_by_key(|part| part.span.lo()); + // Verify the assumption that all spans are disjoint + assert_eq!( + subst.parts.array_windows().find(|[a, b]| a.span.overlaps(b.span)), + None, + "all spans must be disjoint", + ); + + // Account for cases where we are suggesting the same code that's already + // there. This shouldn't happen often, but in some cases for multipart + // suggestions it's much easier to handle it here than in the origin. + subst.parts.retain(|p| is_different(sm, &p.snippet, p.span)); + + let item_span = subst.parts.first()?; + let file = sm.lookup_source_file(item_span.span.lo()); + if !invalid + && should_show_source_code( + &self.ignored_directories_in_source_blocks, + sm, + &file, + ) + { + Some(subst) + } else { + None + } + }) + .collect::>(); + + if substitutions.is_empty() { + continue; + } + let mut msg = self + .translator + .translate_message(&suggestion.msg, args) + .map_err(Report::new) + .unwrap() + .to_string(); + + let lo = substitutions + .iter() + .find_map(|sub| sub.parts.first().map(|p| p.span.lo())) + .unwrap(); + let file = sm.lookup_source_file(lo); + + let filename = + sm.filename_for_diagnostics(&file.name).to_string_lossy().to_string(); + + let other_suggestions = substitutions.len().saturating_sub(MAX_SUGGESTIONS); + + let subs = substitutions + .into_iter() + .take(MAX_SUGGESTIONS) + .filter_map(|sub| { + let mut confusion_type = ConfusionType::None; + for part in &sub.parts { + let part_confusion = + detect_confusion_type(sm, &part.snippet, part.span); + confusion_type = confusion_type.combine(part_confusion); + } + + if !matches!(confusion_type, ConfusionType::None) { + msg.push_str(confusion_type.label_text()); + } + + let mut parts = sub + .parts + .into_iter() + .filter_map(|p| { + if is_different(sm, &p.snippet, p.span) { + Some((p.span, p.snippet)) + } else { + None + } + }) + .collect::>(); + + if parts.is_empty() { + None + } else { + let spans = parts.iter().map(|(span, _)| *span).collect::>(); + // The suggestion adds an entire line of code, ending on a newline, so we'll also + // print the *following* line, to provide context of what we're advising people to + // do. Otherwise you would only see contextless code that can be confused for + // already existing code, despite the colors and UI elements. + // We special case `#[derive(_)]\n` and other attribute suggestions, because those + // are the ones where context is most useful. + let fold = if let [(p, snippet)] = &mut parts[..] + && snippet.trim().starts_with("#[") + // This allows for spaces to come between the attribute and the newline + && snippet.trim().ends_with("]") + && snippet.ends_with('\n') + && p.hi() == p.lo() + && let Ok(b) = sm.span_to_prev_source(*p) + && let b = b.rsplit_once('\n').unwrap_or_else(|| ("", &b)).1 + && b.trim().is_empty() + { + // FIXME: This is a hack: + // The span for attribute suggestions often times points to the + // beginning of an item, disregarding leading whitespace. This + // causes the attribute to be properly indented, but leaves original + // item without indentation when rendered. + // This fixes that problem by adjusting the span to point to the start + // of the whitespace, and adds the whitespace to the replacement. + // + // Source: " extern "custom" fn negate(a: i64) -> i64 {\n" + // Span: 4..4 + // Replacement: "#[unsafe(naked)]\n" + // + // Before: + // help: convert this to an `#[unsafe(naked)]` function + // | + // LL + #[unsafe(naked)] + // LL | extern "custom" fn negate(a: i64) -> i64 { + // | + // + // After + // help: convert this to an `#[unsafe(naked)]` function + // | + // LL + #[unsafe(naked)] + // LL | extern "custom" fn negate(a: i64) -> i64 { + // | + if !b.is_empty() && !snippet.ends_with(b) { + snippet.insert_str(0, b); + let offset = BytePos(b.len() as u32); + *p = p.with_lo(p.lo() - offset).shrink_to_lo(); + } + false + } else { + true + }; + + if let Some((bounding_span, source, line_offset)) = + shrink_file(spans.as_slice(), &file.name, sm) + { + let adj_lo = bounding_span.lo().to_usize(); + Some( + Snippet::source(source) + .line_start(line_offset) + .path(filename.clone()) + .fold(fold) + .patches(parts.into_iter().map( + |(span, replacement)| { + let lo = + span.lo().to_usize().saturating_sub(adj_lo); + let hi = + span.hi().to_usize().saturating_sub(adj_lo); + + Patch::new(lo..hi, replacement) + }, + )), + ) + } else { + None + } + } + }) + .collect::>(); + if !subs.is_empty() { + report.push(std::mem::replace( + &mut group, + Group::with_title(annotate_snippets::Level::HELP.secondary_title(msg)), + )); + + group = group.elements(subs); + if other_suggestions > 0 { + group = group.element( + annotate_snippets::Level::NOTE.no_name().message(format!( + "and {} other candidate{}", + other_suggestions, + pluralize!(other_suggestions) + )), + ); + } + } + } + } + } + + // FIXME: This hack should be removed once annotate_snippets is the + // default emitter. + if suggestions_expected > 0 && report.is_empty() { + group = group.element(Padding); + } + + if !group.is_empty() { + report.push(group); + } + if let Err(e) = + emit_to_destination(renderer.render(&report), level, &mut self.dst, self.short_message) + { + panic!("failed to emit error: {e}"); + } + } + + fn renderer(&self) -> Renderer { + let width = if let Some(width) = self.diagnostic_width { + width + } else if self.ui_testing || cfg!(miri) { + DEFAULT_TERM_WIDTH + } else { + termize::dimensions().map(|(w, _)| w).unwrap_or(DEFAULT_TERM_WIDTH) + }; + let decor_style = match self.theme { + OutputTheme::Ascii => annotate_snippets::renderer::DecorStyle::Ascii, + OutputTheme::Unicode => annotate_snippets::renderer::DecorStyle::Unicode, + }; + + match self.dst.current_choice() { + ColorChoice::AlwaysAnsi | ColorChoice::Always | ColorChoice::Auto => Renderer::styled(), + ColorChoice::Never => Renderer::plain(), + } + .term_width(width) + .anonymized_line_numbers(self.ui_testing) + .decor_style(decor_style) + .short_message(self.short_message) + } + + fn pre_style_msgs( + &self, + msgs: &[(DiagMessage, Style)], + level: Level, + args: &FluentArgs<'_>, + ) -> String { + msgs.iter() + .filter_map(|(m, style)| { + let text = self.translator.translate_message(m, args).map_err(Report::new).unwrap(); + let style = style.anstyle(level); + if text.is_empty() { None } else { Some(format!("{style}{text}{style:#}")) } + }) + .collect() + } + + fn annotated_snippet<'a>( + &self, + annotations: Vec, + file_name: &FileName, + sm: &Arc, + ) -> Option>> { + let spans = annotations.iter().map(|a| a.span).collect::>(); + if let Some((bounding_span, source, offset_line)) = shrink_file(&spans, file_name, sm) { + let adj_lo = bounding_span.lo().to_usize(); + let filename = sm.filename_for_diagnostics(file_name).to_string_lossy().to_string(); + Some(Snippet::source(source).line_start(offset_line).path(filename).annotations( + annotations.into_iter().map(move |a| { + let lo = a.span.lo().to_usize().saturating_sub(adj_lo); + let hi = a.span.hi().to_usize().saturating_sub(adj_lo); + let ann = a.kind.span(lo..hi); + if let Some(label) = a.label { ann.label(label) } else { ann } + }), + )) + } else { + None + } + } + + fn unannotated_messages<'a>( + &self, + annotations: Vec, + file_name: &FileName, + sm: &Arc, + file_idx: usize, + report: &mut Vec>, + mut group: Group<'a>, + level: &annotate_snippets::level::Level<'static>, + ) -> Group<'a> { + let filename = sm.filename_for_diagnostics(file_name).to_string_lossy().to_string(); + let mut line_tracker = vec![]; + for (i, a) in annotations.into_iter().enumerate() { + let lo = sm.lookup_char_pos(a.span.lo()); + let hi = sm.lookup_char_pos(a.span.hi()); + if i == 0 || (a.label.is_some()) { + // Render each new file after the first in its own Group + // ╭▸ $DIR/deriving-meta-unknown-trait.rs:1:10 + // │ + // LL │ #[derive(Eqr)] + // │ ━━━ + // ╰╴ (<- It makes it so *this* will get printed) + // ╭▸ $SRC_DIR/core/src/option.rs:594:0 + // ⸬ $SRC_DIR/core/src/option.rs:602:4 + // │ + // ╰ note: not covered + if i == 0 && file_idx != 0 { + report.push(std::mem::replace(&mut group, Group::with_level(level.clone()))); + } + + if !line_tracker.contains(&lo.line) { + line_tracker.push(lo.line); + // ╭▸ $SRC_DIR/core/src/option.rs:594:0 (<- It adds *this*) + // ⸬ $SRC_DIR/core/src/option.rs:602:4 + // │ + // ╰ note: not covered + group = group.element( + Origin::path(filename.clone()) + .line(sm.doctest_offset_line(file_name, lo.line)) + .char_column(lo.col_display), + ); + } + + if hi.line > lo.line + && a.label.as_ref().is_some_and(|l| !l.is_empty()) + && !line_tracker.contains(&hi.line) + { + line_tracker.push(hi.line); + // ╭▸ $SRC_DIR/core/src/option.rs:594:0 + // ⸬ $SRC_DIR/core/src/option.rs:602:4 (<- It adds *this*) + // │ + // ╰ note: not covered + group = group.element( + Origin::path(filename.clone()) + .line(sm.doctest_offset_line(file_name, hi.line)) + .char_column(hi.col_display), + ); + } + + if let Some(label) = a.label + && !label.is_empty() + { + // ╭▸ $SRC_DIR/core/src/option.rs:594:0 + // ⸬ $SRC_DIR/core/src/option.rs:602:4 + // │ (<- It adds *this*) + // ╰ note: not covered (<- and *this*) + group = group + .element(Padding) + .element(annotate_snippets::Level::NOTE.message(label)); + } + } + } + group } } + +fn emit_to_destination( + rendered: String, + lvl: &Level, + dst: &mut Destination, + short_message: bool, +) -> io::Result<()> { + use crate::lock; + let _buffer_lock = lock::acquire_global_lock("rustc_errors"); + writeln!(dst, "{rendered}")?; + if !short_message && !lvl.is_failure_note() { + writeln!(dst)?; + } + dst.flush()?; + Ok(()) +} + +#[derive(Debug)] +struct Annotation { + kind: AnnotationKind, + span: Span, + label: Option, +} + +fn collect_annotations( + args: &FluentArgs<'_>, + msp: &MultiSpan, + sm: &Arc, + translator: &Translator, +) -> Vec<(Arc, Vec)> { + let mut output: Vec<(Arc, Vec)> = vec![]; + + for SpanLabel { span, is_primary, label } in msp.span_labels() { + // If we don't have a useful span, pick the primary span if that exists. + // Worst case we'll just print an error at the top of the main file. + let span = match (span.is_dummy(), msp.primary_span()) { + (_, None) | (false, _) => span, + (true, Some(span)) => span, + }; + let file = sm.lookup_source_file(span.lo()); + + let kind = if is_primary { AnnotationKind::Primary } else { AnnotationKind::Context }; + + let label = label.as_ref().map(|m| { + normalize_whitespace( + &translator.translate_message(m, args).map_err(Report::new).unwrap(), + ) + }); + + let ann = Annotation { kind, span, label }; + if sm.is_valid_span(ann.span).is_ok() { + // Look through each of our files for the one we're adding to. We + // use each files `stable_id` to avoid issues with file name + // collisions when multiple versions of the same crate are present + // in the dependency graph + if let Some((_, annotations)) = + output.iter_mut().find(|(f, _)| f.stable_id == file.stable_id) + { + annotations.push(ann); + } else { + output.push((file, vec![ann])); + } + } + } + output +} + +fn shrink_file( + spans: &[Span], + file_name: &FileName, + sm: &Arc, +) -> Option<(Span, String, usize)> { + let lo_byte = spans.iter().map(|s| s.lo()).min()?; + let lo_loc = sm.lookup_char_pos(lo_byte); + let lo = lo_loc.file.line_bounds(lo_loc.line.saturating_sub(1)).start; + + let hi_byte = spans.iter().map(|s| s.hi()).max()?; + let hi_loc = sm.lookup_char_pos(hi_byte); + let hi = lo_loc.file.line_bounds(hi_loc.line.saturating_sub(1)).end; + + let bounding_span = Span::with_root_ctxt(lo, hi); + let source = sm.span_to_snippet(bounding_span).unwrap_or_default(); + let offset_line = sm.doctest_offset_line(file_name, lo_loc.line); + + Some((bounding_span, source, offset_line)) +} diff --git a/compiler/rustc_errors/src/codes.rs b/compiler/rustc_errors/src/codes.rs index 947cf27ca795..787a8af99b1f 100644 --- a/compiler/rustc_errors/src/codes.rs +++ b/compiler/rustc_errors/src/codes.rs @@ -20,6 +20,8 @@ impl fmt::Display for ErrCode { } } +rustc_error_messages::into_diag_arg_using_display!(ErrCode); + macro_rules! define_error_code_constants_and_diagnostics_table { ($($name:ident: $num:literal,)*) => ( $( diff --git a/compiler/rustc_errors/src/decorate_diag.rs b/compiler/rustc_errors/src/decorate_diag.rs new file mode 100644 index 000000000000..5aef26ccf973 --- /dev/null +++ b/compiler/rustc_errors/src/decorate_diag.rs @@ -0,0 +1,85 @@ +/// This module provides types and traits for buffering lints until later in compilation. +use rustc_ast::node_id::NodeId; +use rustc_data_structures::fx::FxIndexMap; +use rustc_error_messages::MultiSpan; +use rustc_lint_defs::{BuiltinLintDiag, Lint, LintId}; + +use crate::{DynSend, LintDiagnostic, LintDiagnosticBox}; + +/// We can't implement `LintDiagnostic` for `BuiltinLintDiag`, because decorating some of its +/// variants requires types we don't have yet. So, handle that case separately. +pub enum DecorateDiagCompat { + Dynamic(Box LintDiagnosticBox<'a, ()> + DynSend + 'static>), + Builtin(BuiltinLintDiag), +} + +impl std::fmt::Debug for DecorateDiagCompat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DecorateDiagCompat").finish() + } +} + +impl !LintDiagnostic<'_, ()> for BuiltinLintDiag {} + +impl LintDiagnostic<'a, ()> + DynSend + 'static> From for DecorateDiagCompat { + #[inline] + fn from(d: D) -> Self { + Self::Dynamic(Box::new(d)) + } +} + +impl From for DecorateDiagCompat { + #[inline] + fn from(b: BuiltinLintDiag) -> Self { + Self::Builtin(b) + } +} + +/// Lints that are buffered up early on in the `Session` before the +/// `LintLevels` is calculated. +#[derive(Debug)] +pub struct BufferedEarlyLint { + /// The span of code that we are linting on. + pub span: Option, + + /// The `NodeId` of the AST node that generated the lint. + pub node_id: NodeId, + + /// A lint Id that can be passed to + /// `rustc_lint::early::EarlyContextAndPass::check_id`. + pub lint_id: LintId, + + /// Customization of the `Diag<'_>` for the lint. + pub diagnostic: DecorateDiagCompat, +} + +#[derive(Default, Debug)] +pub struct LintBuffer { + pub map: FxIndexMap>, +} + +impl LintBuffer { + pub fn add_early_lint(&mut self, early_lint: BufferedEarlyLint) { + self.map.entry(early_lint.node_id).or_default().push(early_lint); + } + + pub fn take(&mut self, id: NodeId) -> Vec { + // FIXME(#120456) - is `swap_remove` correct? + self.map.swap_remove(&id).unwrap_or_default() + } + + pub fn buffer_lint( + &mut self, + lint: &'static Lint, + node_id: NodeId, + span: impl Into, + decorate: impl Into, + ) { + self.add_early_lint(BufferedEarlyLint { + lint_id: LintId::of(lint), + node_id, + span: Some(span.into()), + diagnostic: decorate.into(), + }); + } +} diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs index 96c7ba6ed27b..96a4ed3218fb 100644 --- a/compiler/rustc_errors/src/diagnostic.rs +++ b/compiler/rustc_errors/src/diagnostic.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use std::thread::panicking; use rustc_data_structures::fx::FxIndexMap; -use rustc_error_messages::{FluentValue, fluent_value_from_str_list_sep_by_and}; +use rustc_error_messages::{DiagArgName, DiagArgValue, IntoDiagArg}; use rustc_lint_defs::{Applicability, LintExpectationId}; use rustc_macros::{Decodable, Encodable}; use rustc_span::source_map::Spanned; @@ -22,26 +22,6 @@ use crate::{ Suggestions, }; -/// Simplified version of `FluentArg` that can implement `Encodable` and `Decodable`. Collection of -/// `DiagArg` are converted to `FluentArgs` (consuming the collection) at the start of diagnostic -/// emission. -pub type DiagArg<'iter> = (&'iter DiagArgName, &'iter DiagArgValue); - -/// Name of a diagnostic argument. -pub type DiagArgName = Cow<'static, str>; - -/// Simplified version of `FluentValue` that can implement `Encodable` and `Decodable`. Converted -/// to a `FluentValue` by the emitter to be used in diagnostic translation. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)] -pub enum DiagArgValue { - Str(Cow<'static, str>), - // This gets converted to a `FluentNumber`, which is an `f64`. An `i32` - // safely fits in an `f64`. Any integers bigger than that will be converted - // to strings in `into_diag_arg` and stored using the `Str` variant. - Number(i32), - StrListSepByAnd(Vec>), -} - pub type DiagArgMap = FxIndexMap; /// Trait for types that `Diag::emit` can return as a "guarantee" (or "proof") @@ -143,36 +123,6 @@ where } } -/// Converts a value of a type into a `DiagArg` (typically a field of an `Diag` struct). -/// Implemented as a custom trait rather than `From` so that it is implemented on the type being -/// converted rather than on `DiagArgValue`, which enables types from other `rustc_*` crates to -/// implement this. -pub trait IntoDiagArg { - /// Convert `Self` into a `DiagArgValue` suitable for rendering in a diagnostic. - /// - /// It takes a `path` where "long values" could be written to, if the `DiagArgValue` is too big - /// for displaying on the terminal. This path comes from the `Diag` itself. When rendering - /// values that come from `TyCtxt`, like `Ty<'_>`, they can use `TyCtxt::short_string`. If a - /// value has no shortening logic that could be used, the argument can be safely ignored. - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue; -} - -impl IntoDiagArg for DiagArgValue { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - self - } -} - -impl From for FluentValue<'static> { - fn from(val: DiagArgValue) -> Self { - match val { - DiagArgValue::Str(s) => From::from(s), - DiagArgValue::Number(n) => From::from(n), - DiagArgValue::StrListSepByAnd(l) => fluent_value_from_str_list_sep_by_and(l), - } - } -} - /// Trait implemented by error types. This should not be implemented manually. Instead, use /// `#[derive(Subdiagnostic)]` -- see [rustc_macros::Subdiagnostic]. #[rustc_diagnostic_item = "Subdiagnostic"] @@ -188,10 +138,20 @@ where /// `#[derive(LintDiagnostic)]` -- see [rustc_macros::LintDiagnostic]. #[rustc_diagnostic_item = "LintDiagnostic"] pub trait LintDiagnostic<'a, G: EmissionGuarantee> { - /// Decorate and emit a lint. + /// Decorate a lint with the information from this type. fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, G>); } +pub trait LintDiagnosticBox<'a, G: EmissionGuarantee> { + fn decorate_lint_box<'b>(self: Box, diag: &'b mut Diag<'a, G>); +} + +impl<'a, G: EmissionGuarantee, D: LintDiagnostic<'a, G>> LintDiagnosticBox<'a, G> for D { + fn decorate_lint_box<'b>(self: Box, diag: &'b mut Diag<'a, G>) { + self.decorate_lint(diag); + } +} + #[derive(Clone, Debug, Encodable, Decodable)] pub(crate) struct DiagLocation { file: Cow<'static, str>, @@ -617,6 +577,29 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> { self.level = Level::DelayedBug; } + /// Make emitting this diagnostic fatal + /// + /// Changes the level of this diagnostic to Fatal, and importantly also changes the emission guarantee. + /// This is sound for errors that would otherwise be printed, but now simply exit the process instead. + /// This function still gives an emission guarantee, the guarantee is now just that it exits fatally. + /// For delayed bugs this is different, since those are buffered. If we upgrade one to fatal, another + /// might now be ignored. + #[rustc_lint_diagnostics] + #[track_caller] + pub fn upgrade_to_fatal(mut self) -> Diag<'a, FatalAbort> { + assert!( + matches!(self.level, Level::Error), + "upgrade_to_fatal: cannot upgrade {:?} to Fatal: not an error", + self.level + ); + self.level = Level::Fatal; + + // Take is okay since we immediately rewrap it in another diagnostic. + // i.e. we do emit it despite defusing the original diagnostic's drop bomb. + let diag = self.diag.take(); + Diag { dcx: self.dcx, diag, _marker: PhantomData } + } + with_fn! { with_span_label, /// Appends a labeled span to the diagnostic. /// @@ -847,17 +830,18 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> { self } + with_fn! { with_span_help, /// Prints the span with some help above it. /// This is like [`Diag::help()`], but it gets its own span. #[rustc_lint_diagnostics] - pub fn span_help>( + pub fn span_help( &mut self, - sp: S, + sp: impl Into, msg: impl Into, ) -> &mut Self { self.sub(Level::Help, msg, sp.into()); self - } + } } /// Disallow attaching suggestions to this diagnostic. /// Any suggestions attached e.g. with the `span_suggestion_*` methods @@ -1112,7 +1096,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> { .map(|snippet| { debug_assert!( !(sp.is_empty() && snippet.is_empty()), - "Span must not be empty and have no suggestion" + "Span `{sp:?}` must not be empty and have no suggestion" ); Substitution { parts: vec![SubstitutionPart { snippet, span: sp }] } }) @@ -1382,6 +1366,11 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> { &mut self.long_ty_path } + pub fn with_long_ty_path(mut self, long_ty_path: Option) -> Self { + self.long_ty_path = long_ty_path; + self + } + /// Most `emit_producing_guarantee` functions use this as a starting point. fn emit_producing_nothing(mut self) { let diag = self.take_diag(); diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs index eeb9ac28808b..435d16a83806 100644 --- a/compiler/rustc_errors/src/diagnostic_impls.rs +++ b/compiler/rustc_errors/src/diagnostic_impls.rs @@ -1,317 +1,22 @@ -use std::backtrace::Backtrace; use std::borrow::Cow; -use std::fmt; -use std::num::ParseIntError; -use std::path::{Path, PathBuf}; -use std::process::ExitStatus; use rustc_abi::TargetDataLayoutErrors; -use rustc_ast::util::parser::ExprPrecedence; -use rustc_ast_pretty::pprust; -use rustc_attr_data_structures::RustcVersion; +use rustc_error_messages::{DiagArgValue, IntoDiagArg}; use rustc_macros::Subdiagnostic; -use rustc_span::edition::Edition; -use rustc_span::{Ident, MacroRulesNormalizedIdent, Span, Symbol}; -use rustc_target::spec::{PanicStrategy, SplitDebuginfo, StackProtector, TargetTuple}; -use rustc_type_ir::{ClosureKind, FloatTy}; -use {rustc_ast as ast, rustc_hir as hir}; +use rustc_span::{Span, Symbol}; use crate::diagnostic::DiagLocation; use crate::{ - Diag, DiagArgValue, DiagCtxtHandle, Diagnostic, EmissionGuarantee, ErrCode, IntoDiagArg, Level, - Subdiagnostic, fluent_generated as fluent, + Diag, DiagCtxtHandle, Diagnostic, EmissionGuarantee, Level, Subdiagnostic, + fluent_generated as fluent, }; -pub struct DiagArgFromDisplay<'a>(pub &'a dyn fmt::Display); - -impl IntoDiagArg for DiagArgFromDisplay<'_> { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.0.to_string().into_diag_arg(path) - } -} - -impl<'a> From<&'a dyn fmt::Display> for DiagArgFromDisplay<'a> { - fn from(t: &'a dyn fmt::Display) -> Self { - DiagArgFromDisplay(t) - } -} - -impl<'a, T: fmt::Display> From<&'a T> for DiagArgFromDisplay<'a> { - fn from(t: &'a T) -> Self { - DiagArgFromDisplay(t) - } -} - -impl<'a, T: Clone + IntoDiagArg> IntoDiagArg for &'a T { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.clone().into_diag_arg(path) - } -} - -#[macro_export] -macro_rules! into_diag_arg_using_display { - ($( $ty:ty ),+ $(,)?) => { - $( - impl IntoDiagArg for $ty { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.to_string().into_diag_arg(path) - } - } - )+ - } -} - -macro_rules! into_diag_arg_for_number { - ($( $ty:ty ),+ $(,)?) => { - $( - impl IntoDiagArg for $ty { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - // Convert to a string if it won't fit into `Number`. - #[allow(irrefutable_let_patterns)] - if let Ok(n) = TryInto::::try_into(self) { - DiagArgValue::Number(n) - } else { - self.to_string().into_diag_arg(path) - } - } - } - )+ - } -} - -into_diag_arg_using_display!( - ast::ParamKindOrd, - std::io::Error, - Box, - std::num::NonZero, - hir::Target, - Edition, - Ident, - MacroRulesNormalizedIdent, - ParseIntError, - StackProtector, - &TargetTuple, - SplitDebuginfo, - ExitStatus, - ErrCode, - rustc_abi::ExternAbi, -); - -impl IntoDiagArg for RustcVersion { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.to_string())) - } -} - -impl IntoDiagArg for rustc_type_ir::TraitRef { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.to_string().into_diag_arg(path) - } -} - -impl IntoDiagArg for rustc_type_ir::ExistentialTraitRef { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.to_string().into_diag_arg(path) - } -} - -impl IntoDiagArg for rustc_type_ir::UnevaluatedConst { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - format!("{self:?}").into_diag_arg(path) - } -} - -impl IntoDiagArg for rustc_type_ir::FnSig { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - format!("{self:?}").into_diag_arg(path) - } -} - -impl IntoDiagArg for rustc_type_ir::Binder -where - T: IntoDiagArg, -{ - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.skip_binder().into_diag_arg(path) - } -} - -into_diag_arg_for_number!(i8, u8, i16, u16, i32, u32, i64, u64, i128, u128, isize, usize); - -impl IntoDiagArg for bool { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - if self { - DiagArgValue::Str(Cow::Borrowed("true")) - } else { - DiagArgValue::Str(Cow::Borrowed("false")) - } - } -} - -impl IntoDiagArg for char { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(format!("{self:?}"))) - } -} - -impl IntoDiagArg for Vec { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::StrListSepByAnd( - self.into_iter().map(|c| Cow::Owned(format!("{c:?}"))).collect(), - ) - } -} - -impl IntoDiagArg for Symbol { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.to_ident_string().into_diag_arg(path) - } -} - -impl<'a> IntoDiagArg for &'a str { - fn into_diag_arg(self, path: &mut Option) -> DiagArgValue { - self.to_string().into_diag_arg(path) - } -} - -impl IntoDiagArg for String { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self)) - } -} - -impl<'a> IntoDiagArg for Cow<'a, str> { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.into_owned())) - } -} - -impl<'a> IntoDiagArg for &'a Path { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.display().to_string())) - } -} - -impl IntoDiagArg for PathBuf { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.display().to_string())) - } -} - -impl IntoDiagArg for PanicStrategy { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.desc().to_string())) - } -} - -impl IntoDiagArg for hir::ConstContext { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Borrowed(match self { - hir::ConstContext::ConstFn => "const_fn", - hir::ConstContext::Static(_) => "static", - hir::ConstContext::Const { .. } => "const", - })) - } -} - -impl IntoDiagArg for ast::Expr { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(pprust::expr_to_string(&self))) - } -} - -impl IntoDiagArg for ast::Path { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(pprust::path_to_string(&self))) - } -} - -impl IntoDiagArg for ast::token::Token { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(pprust::token_to_string(&self)) - } -} - -impl IntoDiagArg for ast::token::TokenKind { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(pprust::token_kind_to_string(&self)) - } -} - -impl IntoDiagArg for FloatTy { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Borrowed(self.name_str())) - } -} - -impl IntoDiagArg for std::ffi::CString { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned())) - } -} - -impl IntoDiagArg for rustc_data_structures::small_c_str::SmallCStr { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Owned(self.to_string_lossy().into_owned())) - } -} - -impl IntoDiagArg for ast::Visibility { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - let s = pprust::vis_to_string(&self); - let s = s.trim_end().to_string(); - DiagArgValue::Str(Cow::Owned(s)) - } -} - -impl IntoDiagArg for rustc_lint_defs::Level { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Borrowed(self.to_cmd_flag())) - } -} - -impl IntoDiagArg for hir::def::Res { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Borrowed(self.descr())) - } -} - impl IntoDiagArg for DiagLocation { fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { DiagArgValue::Str(Cow::from(self.to_string())) } } -impl IntoDiagArg for Backtrace { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::from(self.to_string())) - } -} - -impl IntoDiagArg for Level { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::from(self.to_string())) - } -} - -impl IntoDiagArg for ClosureKind { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(self.as_str().into()) - } -} - -impl IntoDiagArg for hir::def::Namespace { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Str(Cow::Borrowed(self.descr())) - } -} - -impl IntoDiagArg for ExprPrecedence { - fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { - DiagArgValue::Number(self as i32) - } -} - #[derive(Clone)] pub struct DiagSymbolList(Vec); diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs index 46a4a1868247..25b6d992ab36 100644 --- a/compiler/rustc_errors/src/emitter.rs +++ b/compiler/rustc_errors/src/emitter.rs @@ -16,8 +16,10 @@ use std::iter; use std::path::Path; use std::sync::Arc; +use anstream::{AutoStream, ColorChoice}; +use anstyle::{AnsiColor, Effects}; use derive_setters::Setters; -use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet}; +use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::sync::{DynSend, IntoDynSyncSend}; use rustc_error_messages::{FluentArgs, SpanLabel}; use rustc_lexer; @@ -25,7 +27,6 @@ use rustc_lint_defs::pluralize; use rustc_span::hygiene::{ExpnKind, MacroKind}; use rustc_span::source_map::SourceMap; use rustc_span::{FileLines, FileName, SourceFile, Span, char_width, str_width}; -use termcolor::{Buffer, BufferWriter, Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; use tracing::{debug, instrument, trace, warn}; use crate::registry::Registry; @@ -262,19 +263,11 @@ pub trait Emitter { format!("help: {msg}") } else { // Show the default suggestion text with the substitution - format!( - "help: {}{}: `{}`", - msg, - if self - .source_map() - .is_some_and(|sm| is_case_difference(sm, snippet, part.span,)) - { - " (notice the capitalization)" - } else { - "" - }, - snippet, - ) + let confusion_type = self + .source_map() + .map(|sm| detect_confusion_type(sm, snippet, part.span)) + .unwrap_or(ConfusionType::None); + format!("help: {}{}: `{}`", msg, confusion_type.label_text(), snippet,) }; primary_span.push_span_label(part.span, msg); @@ -417,7 +410,7 @@ pub trait Emitter { if !redundant_span || always_backtrace { let msg: Cow<'static, _> = match trace.kind { ExpnKind::Macro(MacroKind::Attr, _) => { - "this procedural macro expansion".into() + "this attribute macro expansion".into() } ExpnKind::Macro(MacroKind::Derive, _) => { "this derive macro expansion".into() @@ -533,10 +526,6 @@ impl Emitter for HumanEmitter { !self.short_message } - fn supports_color(&self) -> bool { - self.dst.supports_color() - } - fn translator(&self) -> &Translator { &self.translator } @@ -1470,7 +1459,7 @@ impl HumanEmitter { max_line_num_len: usize, is_secondary: bool, is_cont: bool, - ) -> io::Result<()> { + ) -> io::Result { let mut buffer = StyledBuffer::new(); if !msp.has_primary_spans() && !msp.has_span_labels() && is_secondary && !self.short_message @@ -1534,16 +1523,17 @@ impl HumanEmitter { label_width += 2; } let mut line = 0; + let mut pad = false; for (text, style) in msgs.iter() { let text = self.translator.translate_message(text, args).map_err(Report::new).unwrap(); // Account for newlines to align output to its label. - for text in normalize_whitespace(&text).lines() { + for text in normalize_whitespace(&text).split('\n') { buffer.append( line, &format!( "{}{}", - if line == 0 { String::new() } else { " ".repeat(label_width) }, + if pad { " ".repeat(label_width) } else { String::new() }, text ), match style { @@ -1552,7 +1542,9 @@ impl HumanEmitter { }, ); line += 1; + pad = true; } + pad = false; // We add lines above, but if the last line has no explicit newline (which would // yield an empty line), then we revert one line up to continue with the next // styled text chunk on the same line as the last one from the prior one. Otherwise @@ -1583,12 +1575,14 @@ impl HumanEmitter { } let mut annotated_files = FileWithAnnotatedLines::collect_annotations(self, args, msp); trace!("{annotated_files:#?}"); + let mut code_window_status = CodeWindowStatus::Open; // Make sure our primary file comes first let primary_span = msp.primary_span().unwrap_or_default(); let (Some(sm), false) = (self.sm.as_ref(), primary_span.is_dummy()) else { // If we don't have span information, emit and exit - return emit_to_destination(&buffer.render(), level, &mut self.dst, self.short_message); + return emit_to_destination(&buffer.render(), level, &mut self.dst, self.short_message) + .map(|_| code_window_status); }; let primary_lo = sm.lookup_char_pos(primary_span.lo()); if let Ok(pos) = @@ -1597,8 +1591,12 @@ impl HumanEmitter { annotated_files.swap(0, pos); } + // An end column separator should be emitted when a file with with a + // source, is followed by one without a source + let mut col_sep_before_no_show_source = false; + let annotated_files_len = annotated_files.len(); // Print out the annotate source lines that correspond with the error - for annotated_file in annotated_files { + for (file_idx, annotated_file) in annotated_files.into_iter().enumerate() { // we can't annotate anything if the source is unavailable. if !should_show_source_code( &self.ignored_directories_in_source_blocks, @@ -1606,6 +1604,26 @@ impl HumanEmitter { &annotated_file.file, ) { if !self.short_message { + // Add an end column separator when a file without a source + // comes after one with a source + // ╭▸ $DIR/deriving-meta-unknown-trait.rs:1:10 + // │ + // LL │ #[derive(Eqr)] + // │ ━━━ + // ╰╴ (<- It prints *this* line) + // ╭▸ $SRC_DIR/core/src/cmp.rs:356:0 + // │ + // ╰╴note: similarly named derive macro `Eq` defined here + if col_sep_before_no_show_source { + let buffer_msg_line_offset = buffer.num_lines(); + self.draw_col_separator_end( + &mut buffer, + buffer_msg_line_offset, + max_line_num_len + 1, + ); + } + col_sep_before_no_show_source = false; + // We'll just print an unannotated message. for (annotation_id, line) in annotated_file.lines.iter().enumerate() { let mut annotations = line.annotations.clone(); @@ -1646,31 +1664,43 @@ impl HumanEmitter { } line_idx += 1; } - for (label, is_primary) in labels.into_iter() { + if is_cont + && file_idx == annotated_files_len - 1 + && annotation_id == annotated_file.lines.len() - 1 + && !labels.is_empty() + { + code_window_status = CodeWindowStatus::Closed; + } + let labels_len = labels.len(); + for (label_idx, (label, is_primary)) in labels.into_iter().enumerate() { let style = if is_primary { Style::LabelPrimary } else { Style::LabelSecondary }; - let pipe = self.col_separator(); - buffer.prepend(line_idx, &format!(" {pipe}"), Style::LineNumber); - for _ in 0..max_line_num_len { - buffer.prepend(line_idx, " ", Style::NoStyle); - } + self.draw_col_separator_no_space( + &mut buffer, + line_idx, + max_line_num_len + 1, + ); line_idx += 1; - let chr = self.note_separator(); - buffer.append(line_idx, &format!(" {chr} note: "), style); - for _ in 0..max_line_num_len { - buffer.prepend(line_idx, " ", Style::NoStyle); - } + self.draw_note_separator( + &mut buffer, + line_idx, + max_line_num_len + 1, + label_idx != labels_len - 1, + ); + buffer.append(line_idx, "note", Style::MainHeaderMsg); + buffer.append(line_idx, ": ", Style::NoStyle); buffer.append(line_idx, label, style); line_idx += 1; } } } continue; + } else { + col_sep_before_no_show_source = true; } - // print out the span location and spacer before we print the annotated source // to do this, we need to know if this span will be primary let is_primary = primary_lo.file.name == annotated_file.file.name; @@ -1855,10 +1885,12 @@ impl HumanEmitter { width_offset, code_offset, margin, - !is_cont && line_idx + 1 == annotated_file.lines.len(), + !is_cont + && file_idx + 1 == annotated_files_len + && line_idx + 1 == annotated_file.lines.len(), ); - let mut to_add = FxHashMap::default(); + let mut to_add = FxIndexMap::default(); for (depth, style) in depths { // FIXME(#120456) - is `swap_remove` correct? @@ -1981,14 +2013,14 @@ impl HumanEmitter { // final step: take our styled buffer, render it, then output it emit_to_destination(&buffer.render(), level, &mut self.dst, self.short_message)?; - Ok(()) + Ok(code_window_status) } fn column_width(&self, code_offset: usize) -> usize { if let Some(width) = self.diagnostic_width { width.saturating_sub(code_offset) } else if self.ui_testing || cfg!(miri) { - DEFAULT_COLUMN_WIDTH + DEFAULT_COLUMN_WIDTH.saturating_sub(code_offset) } else { termize::dimensions() .map(|(w, _)| w.saturating_sub(code_offset)) @@ -2028,12 +2060,12 @@ impl HumanEmitter { buffer.append(0, ": ", Style::HeaderMsg); let mut msg = vec![(suggestion.msg.to_owned(), Style::NoStyle)]; - if suggestions - .iter() - .take(MAX_SUGGESTIONS) - .any(|(_, _, _, only_capitalization)| *only_capitalization) + if let Some(confusion_type) = + suggestions.iter().take(MAX_SUGGESTIONS).find_map(|(_, _, _, confusion_type)| { + if confusion_type.has_confusion() { Some(*confusion_type) } else { None } + }) { - msg.push((" (notice the capitalization difference)".into(), Style::NoStyle)); + msg.push((confusion_type.label_text().into(), Style::NoStyle)); } self.msgs_to_buffer( &mut buffer, @@ -2121,11 +2153,11 @@ impl HumanEmitter { assert!(!file_lines.lines.is_empty() || parts[0].span.is_dummy()); - let line_start = sm.lookup_char_pos(parts[0].span.lo()).line; + let line_start = sm.lookup_char_pos(parts[0].original_span.lo()).line; let mut lines = complete.lines(); if lines.clone().next().is_none() { // Account for a suggestion to completely remove a line(s) with whitespace (#94192). - let line_end = sm.lookup_char_pos(parts[0].span.hi()).line; + let line_end = sm.lookup_char_pos(parts[0].original_span.hi()).line; for line in line_start..=line_end { self.draw_line_num( &mut buffer, @@ -2380,7 +2412,7 @@ impl HumanEmitter { // too bad to begin with, so we side-step that issue here. for (i, line) in snippet.lines().enumerate() { let line = normalize_whitespace(line); - let row = row_num - 2 - (newlines - i - 1); + let row = (row_num - 2 - (newlines - i - 1)).max(2); // On the first line, we highlight between the start of the part // span, and the end of that line. // On the last line, we highlight between the start of the line, and @@ -2496,7 +2528,7 @@ impl HumanEmitter { !children.is_empty() || suggestions.iter().any(|s| s.style != SuggestionStyle::CompletelyHidden), ) { - Ok(()) => { + Ok(code_window_status) => { if !children.is_empty() || suggestions.iter().any(|s| s.style != SuggestionStyle::CompletelyHidden) { @@ -2507,7 +2539,7 @@ impl HumanEmitter { { // We'll continue the vertical bar to point into the next note. self.draw_col_separator_no_space(&mut buffer, 0, max_line_num_len + 1); - } else { + } else if matches!(code_window_status, CodeWindowStatus::Open) { // We'll close the vertical bar to visually end the code window. self.draw_col_separator_end(&mut buffer, 0, max_line_num_len + 1); } @@ -2670,8 +2702,7 @@ impl HumanEmitter { [SubstitutionHighlight { start: 0, end }] if *end == line_to_add.len() => { buffer.puts(*row_num, max_line_num_len + 1, "+ ", Style::Addition); } - [] => { - // FIXME: needed? Doesn't get exercised in any test. + [] | [SubstitutionHighlight { start: 0, end: 0 }] => { self.draw_col_separator_no_space(buffer, *row_num, max_line_num_len + 1); } _ => { @@ -2834,10 +2865,11 @@ impl HumanEmitter { } } - fn note_separator(&self) -> char { + fn note_separator(&self, is_cont: bool) -> &'static str { match self.theme { - OutputTheme::Ascii => '=', - OutputTheme::Unicode => '╰', + OutputTheme::Ascii => "= ", + OutputTheme::Unicode if is_cont => "├ ", + OutputTheme::Unicode => "╰ ", } } @@ -2950,11 +2982,7 @@ impl HumanEmitter { col: usize, is_cont: bool, ) { - let chr = match self.theme { - OutputTheme::Ascii => "= ", - OutputTheme::Unicode if is_cont => "├ ", - OutputTheme::Unicode => "╰ ", - }; + let chr = self.note_separator(is_cont); buffer.puts(line, col, chr, Style::LineNumber); } @@ -2985,7 +3013,7 @@ impl HumanEmitter { fn secondary_file_start(&self) -> &'static str { match self.theme { OutputTheme::Ascii => "::: ", - OutputTheme::Unicode => " ⸬ ", + OutputTheme::Unicode => " ⸬ ", } } @@ -3055,6 +3083,12 @@ enum DisplaySuggestion { Add, } +#[derive(Clone, Copy, Debug)] +enum CodeWindowStatus { + Closed, + Open, +} + impl FileWithAnnotatedLines { /// Preprocess all the annotations so that they are grouped by file and by line number /// This helps us quickly iterate over the whole message (including secondary file spans) @@ -3092,7 +3126,6 @@ impl FileWithAnnotatedLines { multiline_depth: 0, }); } - let mut output = vec![]; let mut multiline_annotations = vec![]; @@ -3326,7 +3359,7 @@ const OUTPUT_REPLACEMENTS: &[(char, &str)] = &[ ('\u{2069}', "�"), ]; -fn normalize_whitespace(s: &str) -> String { +pub(crate) fn normalize_whitespace(s: &str) -> String { const { let mut i = 1; while i < OUTPUT_REPLACEMENTS.len() { @@ -3371,7 +3404,7 @@ fn overlaps(a1: &Annotation, a2: &Annotation, padding: usize) -> bool { ) } -fn emit_to_destination( +pub(crate) fn emit_to_destination( rendered_buffer: &[Vec], lvl: &Level, dst: &mut Destination, @@ -3394,10 +3427,8 @@ fn emit_to_destination( let _buffer_lock = lock::acquire_global_lock("rustc_errors"); for (pos, line) in rendered_buffer.iter().enumerate() { for part in line { - let style = part.style.color_spec(*lvl); - dst.set_color(&style)?; - write!(dst, "{}", part.text)?; - dst.reset()?; + let style = part.style.anstyle(*lvl); + write!(dst, "{style}{}{style:#}", part.text)?; } if !short_message && (!lvl.is_failure_note() || pos != rendered_buffer.len() - 1) { writeln!(dst)?; @@ -3407,11 +3438,11 @@ fn emit_to_destination( Ok(()) } -pub type Destination = Box; +pub type Destination = AutoStream>; struct Buffy { - buffer_writer: BufferWriter, - buffer: Buffer, + buffer_writer: std::io::Stderr, + buffer: Vec, } impl Write for Buffy { @@ -3420,7 +3451,7 @@ impl Write for Buffy { } fn flush(&mut self) -> io::Result<()> { - self.buffer_writer.print(&self.buffer)?; + self.buffer_writer.write_all(&self.buffer)?; self.buffer.clear(); Ok(()) } @@ -3435,22 +3466,16 @@ impl Drop for Buffy { } } -impl WriteColor for Buffy { - fn supports_color(&self) -> bool { - self.buffer.supports_color() - } - - fn set_color(&mut self, spec: &ColorSpec) -> io::Result<()> { - self.buffer.set_color(spec) - } - - fn reset(&mut self) -> io::Result<()> { - self.buffer.reset() - } -} - pub fn stderr_destination(color: ColorConfig) -> Destination { + let buffer_writer = std::io::stderr(); let choice = color.to_color_choice(); + // We need to resolve `ColorChoice::Auto` before `Box`ing since + // `ColorChoice::Auto` on `dyn Write` will always resolve to `Never` + let choice = if matches!(choice, ColorChoice::Auto) { + AutoStream::choice(&buffer_writer) + } else { + choice + }; // On Windows we'll be performing global synchronization on the entire // system for emitting rustc errors, so there's no need to buffer // anything. @@ -3458,60 +3483,42 @@ pub fn stderr_destination(color: ColorConfig) -> Destination { // On non-Windows we rely on the atomicity of `write` to ensure errors // don't get all jumbled up. if cfg!(windows) { - Box::new(StandardStream::stderr(choice)) + AutoStream::new(Box::new(buffer_writer), choice) } else { - let buffer_writer = BufferWriter::stderr(choice); - let buffer = buffer_writer.buffer(); - Box::new(Buffy { buffer_writer, buffer }) + let buffer = Vec::new(); + AutoStream::new(Box::new(Buffy { buffer_writer, buffer }), choice) } } /// On Windows, BRIGHT_BLUE is hard to read on black. Use cyan instead. /// /// See #36178. -const BRIGHT_BLUE: Color = if cfg!(windows) { Color::Cyan } else { Color::Blue }; +const BRIGHT_BLUE: anstyle::Style = if cfg!(windows) { + AnsiColor::BrightCyan.on_default() +} else { + AnsiColor::BrightBlue.on_default() +}; impl Style { - fn color_spec(&self, lvl: Level) -> ColorSpec { - let mut spec = ColorSpec::new(); + pub(crate) fn anstyle(&self, lvl: Level) -> anstyle::Style { match self { - Style::Addition => { - spec.set_fg(Some(Color::Green)).set_intense(true); - } - Style::Removal => { - spec.set_fg(Some(Color::Red)).set_intense(true); - } - Style::LineAndColumn => {} - Style::LineNumber => { - spec.set_bold(true); - spec.set_intense(true); - spec.set_fg(Some(BRIGHT_BLUE)); - } - Style::Quotation => {} - Style::MainHeaderMsg => { - spec.set_bold(true); - if cfg!(windows) { - spec.set_intense(true).set_fg(Some(Color::White)); - } - } - Style::UnderlinePrimary | Style::LabelPrimary => { - spec = lvl.color(); - spec.set_bold(true); - } - Style::UnderlineSecondary | Style::LabelSecondary => { - spec.set_bold(true).set_intense(true); - spec.set_fg(Some(BRIGHT_BLUE)); - } - Style::HeaderMsg | Style::NoStyle => {} - Style::Level(lvl) => { - spec = lvl.color(); - spec.set_bold(true); - } - Style::Highlight => { - spec.set_bold(true).set_fg(Some(Color::Magenta)); + Style::Addition => AnsiColor::BrightGreen.on_default(), + Style::Removal => AnsiColor::BrightRed.on_default(), + Style::LineAndColumn => anstyle::Style::new(), + Style::LineNumber => BRIGHT_BLUE.effects(Effects::BOLD), + Style::Quotation => anstyle::Style::new(), + Style::MainHeaderMsg => if cfg!(windows) { + AnsiColor::BrightWhite.on_default() + } else { + anstyle::Style::new() } + .effects(Effects::BOLD), + Style::UnderlinePrimary | Style::LabelPrimary => lvl.color().effects(Effects::BOLD), + Style::UnderlineSecondary | Style::LabelSecondary => BRIGHT_BLUE.effects(Effects::BOLD), + Style::HeaderMsg | Style::NoStyle => anstyle::Style::new(), + Style::Level(lvl) => lvl.color().effects(Effects::BOLD), + Style::Highlight => AnsiColor::Magenta.on_default().effects(Effects::BOLD), } - spec } } @@ -3528,24 +3535,107 @@ pub fn is_different(sm: &SourceMap, suggested: &str, sp: Span) -> bool { } /// Whether the original and suggested code are visually similar enough to warrant extra wording. -pub fn is_case_difference(sm: &SourceMap, suggested: &str, sp: Span) -> bool { - // FIXME: this should probably be extended to also account for `FO0` → `FOO` and unicode. +pub fn detect_confusion_type(sm: &SourceMap, suggested: &str, sp: Span) -> ConfusionType { let found = match sm.span_to_snippet(sp) { Ok(snippet) => snippet, Err(e) => { warn!(error = ?e, "Invalid span {:?}", sp); - return false; + return ConfusionType::None; } }; - let ascii_confusables = &['c', 'f', 'i', 'k', 'o', 's', 'u', 'v', 'w', 'x', 'y', 'z']; - // All the chars that differ in capitalization are confusable (above): - let confusable = iter::zip(found.chars(), suggested.chars()) - .filter(|(f, s)| f != s) - .all(|(f, s)| ascii_confusables.contains(&f) || ascii_confusables.contains(&s)); - confusable && found.to_lowercase() == suggested.to_lowercase() - // FIXME: We sometimes suggest the same thing we already have, which is a - // bug, but be defensive against that here. - && found != suggested + + let mut has_case_confusion = false; + let mut has_digit_letter_confusion = false; + + if found.len() == suggested.len() { + let mut has_case_diff = false; + let mut has_digit_letter_confusable = false; + let mut has_other_diff = false; + + let ascii_confusables = &['c', 'f', 'i', 'k', 'o', 's', 'u', 'v', 'w', 'x', 'y', 'z']; + + let digit_letter_confusables = [('0', 'O'), ('1', 'l'), ('5', 'S'), ('8', 'B'), ('9', 'g')]; + + for (f, s) in iter::zip(found.chars(), suggested.chars()) { + if f != s { + if f.eq_ignore_ascii_case(&s) { + // Check for case differences (any character that differs only in case) + if ascii_confusables.contains(&f) || ascii_confusables.contains(&s) { + has_case_diff = true; + } else { + has_other_diff = true; + } + } else if digit_letter_confusables.contains(&(f, s)) + || digit_letter_confusables.contains(&(s, f)) + { + // Check for digit-letter confusables (like 0 vs O, 1 vs l, etc.) + has_digit_letter_confusable = true; + } else { + has_other_diff = true; + } + } + } + + // If we have case differences and no other differences + if has_case_diff && !has_other_diff && found != suggested { + has_case_confusion = true; + } + if has_digit_letter_confusable && !has_other_diff && found != suggested { + has_digit_letter_confusion = true; + } + } + + match (has_case_confusion, has_digit_letter_confusion) { + (true, true) => ConfusionType::Both, + (true, false) => ConfusionType::Case, + (false, true) => ConfusionType::DigitLetter, + (false, false) => ConfusionType::None, + } +} + +/// Represents the type of confusion detected between original and suggested code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConfusionType { + /// No confusion detected + None, + /// Only case differences (e.g., "hello" vs "Hello") + Case, + /// Only digit-letter confusion (e.g., "0" vs "O", "1" vs "l") + DigitLetter, + /// Both case and digit-letter confusion + Both, +} + +impl ConfusionType { + /// Returns the appropriate label text for this confusion type. + pub fn label_text(&self) -> &'static str { + match self { + ConfusionType::None => "", + ConfusionType::Case => " (notice the capitalization)", + ConfusionType::DigitLetter => " (notice the digit/letter confusion)", + ConfusionType::Both => " (notice the capitalization and digit/letter confusion)", + } + } + + /// Combines two confusion types. If either is `Both`, the result is `Both`. + /// If one is `Case` and the other is `DigitLetter`, the result is `Both`. + /// Otherwise, returns the non-`None` type, or `None` if both are `None`. + pub fn combine(self, other: ConfusionType) -> ConfusionType { + match (self, other) { + (ConfusionType::None, other) => other, + (this, ConfusionType::None) => this, + (ConfusionType::Both, _) | (_, ConfusionType::Both) => ConfusionType::Both, + (ConfusionType::Case, ConfusionType::DigitLetter) + | (ConfusionType::DigitLetter, ConfusionType::Case) => ConfusionType::Both, + (ConfusionType::Case, ConfusionType::Case) => ConfusionType::Case, + (ConfusionType::DigitLetter, ConfusionType::DigitLetter) => ConfusionType::DigitLetter, + } + } + + /// Returns true if this confusion type represents any kind of confusion. + pub fn has_confusion(&self) -> bool { + *self != ConfusionType::None + } } pub(crate) fn should_show_source_code( diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs index 719d4ca625ae..03ce1d82ef3c 100644 --- a/compiler/rustc_errors/src/json.rs +++ b/compiler/rustc_errors/src/json.rs @@ -15,6 +15,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use std::vec; +use anstream::{AutoStream, ColorChoice}; use derive_setters::Setters; use rustc_data_structures::sync::IntoDynSyncSend; use rustc_error_messages::FluentArgs; @@ -23,7 +24,6 @@ use rustc_span::Span; use rustc_span::hygiene::ExpnData; use rustc_span::source_map::{FilePathMapping, SourceMap}; use serde::Serialize; -use termcolor::{ColorSpec, WriteColor}; use crate::diagnostic::IsLint; use crate::emitter::{ @@ -333,7 +333,7 @@ impl Diagnostic { // generate regular command line output and store it in the json // A threadsafe buffer for writing. - #[derive(Default, Clone)] + #[derive(Clone)] struct BufWriter(Arc>>); impl Write for BufWriter { @@ -344,19 +344,6 @@ impl Diagnostic { self.0.lock().unwrap().flush() } } - impl WriteColor for BufWriter { - fn supports_color(&self) -> bool { - false - } - - fn set_color(&mut self, _spec: &ColorSpec) -> io::Result<()> { - Ok(()) - } - - fn reset(&mut self) -> io::Result<()> { - Ok(()) - } - } let translated_message = je.translator.translate_messages(&diag.messages, &args); @@ -382,13 +369,15 @@ impl Diagnostic { children .insert(0, Diagnostic::from_sub_diagnostic(&diag.emitted_at_sub_diag(), &args, je)); } - let buf = BufWriter::default(); - let mut dst: Destination = Box::new(buf.clone()); + let buf = BufWriter(Arc::new(Mutex::new(Vec::new()))); let short = je.json_rendered.short(); - match je.color_config { - ColorConfig::Always | ColorConfig::Auto => dst = Box::new(termcolor::Ansi::new(dst)), - ColorConfig::Never => {} - } + let dst: Destination = AutoStream::new( + Box::new(buf.clone()), + match je.color_config.to_color_choice() { + ColorChoice::Auto => ColorChoice::Always, + choice => choice, + }, + ); HumanEmitter::new(dst, je.translator.clone()) .short_message(short) .sm(je.sm.clone()) diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs index 381d780077d1..17cd466f96b8 100644 --- a/compiler/rustc_errors/src/lib.rs +++ b/compiler/rustc_errors/src/lib.rs @@ -39,28 +39,34 @@ use std::path::{Path, PathBuf}; use std::{fmt, panic}; use Level::*; +// Used by external projects such as `rust-gpu`. +// See https://github.com/rust-lang/rust/pull/115393. +pub use anstream::{AutoStream, ColorChoice}; +pub use anstyle::{ + Ansi256Color, AnsiColor, Color, EffectIter, Effects, Reset, RgbColor, Style as Anstyle, +}; pub use codes::*; +pub use decorate_diag::{BufferedEarlyLint, DecorateDiagCompat, LintBuffer}; pub use diagnostic::{ - BugAbort, Diag, DiagArg, DiagArgMap, DiagArgName, DiagArgValue, DiagInner, DiagStyledString, - Diagnostic, EmissionGuarantee, FatalAbort, IntoDiagArg, LintDiagnostic, StringPart, Subdiag, - Subdiagnostic, + BugAbort, Diag, DiagArgMap, DiagInner, DiagStyledString, Diagnostic, EmissionGuarantee, + FatalAbort, LintDiagnostic, LintDiagnosticBox, StringPart, Subdiag, Subdiagnostic, }; pub use diagnostic_impls::{ - DiagArgFromDisplay, DiagSymbolList, ElidedLifetimeInPathSubdiag, ExpectedLifetimeParameter, + DiagSymbolList, ElidedLifetimeInPathSubdiag, ExpectedLifetimeParameter, IndicateAnonymousLifetime, SingleLabelManySpans, }; pub use emitter::ColorConfig; -use emitter::{DynEmitter, Emitter, is_case_difference, is_different}; +use emitter::{ConfusionType, DynEmitter, Emitter, detect_confusion_type, is_different}; use rustc_data_structures::AtomicRef; use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_data_structures::stable_hasher::StableHasher; use rustc_data_structures::sync::{DynSend, Lock}; pub use rustc_error_messages::{ - DiagMessage, FluentBundle, LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, - SubdiagMessage, fallback_fluent_bundle, fluent_bundle, + DiagArg, DiagArgFromDisplay, DiagArgName, DiagArgValue, DiagMessage, FluentBundle, IntoDiagArg, + LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, SubdiagMessage, + fallback_fluent_bundle, fluent_bundle, into_diag_arg_using_display, }; use rustc_hashes::Hash128; -use rustc_hir::HirId; pub use rustc_lint_defs::{Applicability, listify, pluralize}; use rustc_lint_defs::{Lint, LintExpectationId}; use rustc_macros::{Decodable, Encodable}; @@ -69,9 +75,6 @@ pub use rustc_span::fatal_error::{FatalError, FatalErrorMarker}; use rustc_span::source_map::SourceMap; use rustc_span::{BytePos, DUMMY_SP, Loc, Span}; pub use snippet::Style; -// Used by external projects such as `rust-gpu`. -// See https://github.com/rust-lang/rust/pull/115393. -pub use termcolor::{Color, ColorSpec, WriteColor}; use tracing::debug; use crate::emitter::TimingEvent; @@ -80,6 +83,7 @@ use crate::timings::TimingRecord; pub mod annotate_snippet_emitter_writer; pub mod codes; +mod decorate_diag; mod diagnostic; mod diagnostic_impls; pub mod emitter; @@ -108,13 +112,14 @@ rustc_data_structures::static_assert_size!(PResult<'_, bool>, 24); /// Used to avoid depending on `rustc_middle` in `rustc_attr_parsing`. /// Always the `TyCtxt`. pub trait LintEmitter: Copy { + type Id: Copy; #[track_caller] fn emit_node_span_lint( self, lint: &'static Lint, - hir_id: HirId, + hir_id: Self::Id, span: impl Into, - decorator: impl for<'a> LintDiagnostic<'a, ()>, + decorator: impl for<'a> LintDiagnostic<'a, ()> + DynSend + 'static, ); } @@ -222,6 +227,13 @@ pub struct SubstitutionPart { pub snippet: String, } +#[derive(Clone, Debug, PartialEq, Hash, Encodable, Decodable)] +pub struct TrimmedSubstitutionPart { + pub original_span: Span, + pub span: Span, + pub snippet: String, +} + /// Used to translate between `Span`s and byte positions within a single output line in highlighted /// code of structured suggestions. #[derive(Debug, Clone, Copy)] @@ -231,6 +243,35 @@ pub(crate) struct SubstitutionHighlight { } impl SubstitutionPart { + /// Try to turn a replacement into an addition when the span that is being + /// overwritten matches either the prefix or suffix of the replacement. + fn trim_trivial_replacements(self, sm: &SourceMap) -> TrimmedSubstitutionPart { + let mut trimmed_part = TrimmedSubstitutionPart { + original_span: self.span, + span: self.span, + snippet: self.snippet, + }; + if trimmed_part.snippet.is_empty() { + return trimmed_part; + } + let Ok(snippet) = sm.span_to_snippet(trimmed_part.span) else { + return trimmed_part; + }; + + if let Some((prefix, substr, suffix)) = as_substr(&snippet, &trimmed_part.snippet) { + trimmed_part.span = Span::new( + trimmed_part.span.lo() + BytePos(prefix as u32), + trimmed_part.span.hi() - BytePos(suffix as u32), + trimmed_part.span.ctxt(), + trimmed_part.span.parent(), + ); + trimmed_part.snippet = substr.to_string(); + } + trimmed_part + } +} + +impl TrimmedSubstitutionPart { pub fn is_addition(&self, sm: &SourceMap) -> bool { !self.snippet.is_empty() && !self.replaces_meaningful_content(sm) } @@ -258,27 +299,6 @@ impl SubstitutionPart { sm.span_to_snippet(self.span) .map_or(!self.span.is_empty(), |snippet| !snippet.trim().is_empty()) } - - /// Try to turn a replacement into an addition when the span that is being - /// overwritten matches either the prefix or suffix of the replacement. - fn trim_trivial_replacements(&mut self, sm: &SourceMap) { - if self.snippet.is_empty() { - return; - } - let Ok(snippet) = sm.span_to_snippet(self.span) else { - return; - }; - - if let Some((prefix, substr, suffix)) = as_substr(&snippet, &self.snippet) { - self.span = Span::new( - self.span.lo() + BytePos(prefix as u32), - self.span.hi() - BytePos(suffix as u32), - self.span.ctxt(), - self.span.parent(), - ); - self.snippet = substr.to_string(); - } - } } /// Given an original string like `AACC`, and a suggestion like `AABBCC`, try to detect @@ -308,7 +328,8 @@ impl CodeSuggestion { pub(crate) fn splice_lines( &self, sm: &SourceMap, - ) -> Vec<(String, Vec, Vec>, bool)> { + ) -> Vec<(String, Vec, Vec>, ConfusionType)> + { // For the `Vec>` value, the first level of the vector // corresponds to the output snippet's lines, while the second level corresponds to the // substrings within that line that should be highlighted. @@ -414,14 +435,20 @@ impl CodeSuggestion { // We need to keep track of the difference between the existing code and the added // or deleted code in order to point at the correct column *after* substitution. let mut acc = 0; - let mut only_capitalization = false; - for part in &mut substitution.parts { + let mut confusion_type = ConfusionType::None; + + let trimmed_parts = substitution + .parts + .into_iter() // If this is a replacement of, e.g. `"a"` into `"ab"`, adjust the // suggestion and snippet to look as if we just suggested to add // `"b"`, which is typically much easier for the user to understand. - part.trim_trivial_replacements(sm); + .map(|part| part.trim_trivial_replacements(sm)) + .collect::>(); - only_capitalization |= is_case_difference(sm, &part.snippet, part.span); + for part in &trimmed_parts { + let part_confusion = detect_confusion_type(sm, &part.snippet, part.span); + confusion_type = confusion_type.combine(part_confusion); let cur_lo = sm.lookup_char_pos(part.span.lo()); if prev_hi.line == cur_lo.line { let mut count = @@ -511,7 +538,7 @@ impl CodeSuggestion { if highlights.iter().all(|parts| parts.is_empty()) { None } else { - Some((buf, substitution.parts, highlights, only_capitalization)) + Some((buf, trimmed_parts, highlights, confusion_type)) } }) .collect() @@ -1157,7 +1184,7 @@ impl<'a> DiagCtxtHandle<'a> { // - It's only produce with JSON output. // - It's not emitted the usual way, via `emit_diagnostic`. // - The `$message_type` field is "unused_externs" rather than the usual - // "diagnosic". + // "diagnostic". // // We count it as a lint error because it has a lint level. The value // of `loud` (which comes from "unused-externs" or @@ -1951,25 +1978,21 @@ impl fmt::Display for Level { } impl Level { - fn color(self) -> ColorSpec { - let mut spec = ColorSpec::new(); + fn color(self) -> anstyle::Style { match self { - Bug | Fatal | Error | DelayedBug => { - spec.set_fg(Some(Color::Red)).set_intense(true); - } + Bug | Fatal | Error | DelayedBug => AnsiColor::BrightRed.on_default(), ForceWarning | Warning => { - spec.set_fg(Some(Color::Yellow)).set_intense(cfg!(windows)); + if cfg!(windows) { + AnsiColor::BrightYellow.on_default() + } else { + AnsiColor::Yellow.on_default() + } } - Note | OnceNote => { - spec.set_fg(Some(Color::Green)).set_intense(true); - } - Help | OnceHelp => { - spec.set_fg(Some(Color::Cyan)).set_intense(true); - } - FailureNote => {} + Note | OnceNote => AnsiColor::BrightGreen.on_default(), + Help | OnceHelp => AnsiColor::BrightCyan.on_default(), + FailureNote => anstyle::Style::new(), Allow | Expect => unreachable!(), } - spec } pub fn to_str(self) -> &'static str { @@ -1998,6 +2021,12 @@ impl Level { } } +impl IntoDiagArg for Level { + fn into_diag_arg(self, _: &mut Option) -> DiagArgValue { + DiagArgValue::Str(Cow::from(self.to_string())) + } +} + // FIXME(eddyb) this doesn't belong here AFAICT, should be moved to callsite. pub fn elided_lifetime_in_path_suggestion( source_map: &SourceMap, diff --git a/compiler/rustc_errors/src/markdown/mod.rs b/compiler/rustc_errors/src/markdown/mod.rs index 64576cdc8cac..4f5e2328234d 100644 --- a/compiler/rustc_errors/src/markdown/mod.rs +++ b/compiler/rustc_errors/src/markdown/mod.rs @@ -4,7 +4,6 @@ use std::io; -use termcolor::{Buffer, BufferWriter, ColorChoice}; mod parse; mod term; @@ -19,15 +18,15 @@ impl<'a> MdStream<'a> { parse::entrypoint(s) } - /// Write formatted output to a termcolor buffer - pub fn write_termcolor_buf(&self, buf: &mut Buffer) -> io::Result<()> { + /// Write formatted output to an anstream buffer + pub fn write_anstream_buf(&self, buf: &mut Vec) -> io::Result<()> { term::entrypoint(self, buf) } } -/// Create a termcolor buffer with the `Always` color choice -pub fn create_stdout_bufwtr() -> BufferWriter { - BufferWriter::stdout(ColorChoice::Always) +/// Create an anstream buffer with the `Always` color choice +pub fn create_stdout_bufwtr() -> anstream::Stdout { + anstream::Stdout::always(std::io::stdout()) } /// A single tokentree within a Markdown document diff --git a/compiler/rustc_errors/src/markdown/term.rs b/compiler/rustc_errors/src/markdown/term.rs index fe1d80bdbe8e..b0ce01548f00 100644 --- a/compiler/rustc_errors/src/markdown/term.rs +++ b/compiler/rustc_errors/src/markdown/term.rs @@ -1,7 +1,7 @@ use std::cell::Cell; use std::io::{self, Write}; -use termcolor::{Buffer, Color, ColorSpec, WriteColor}; +use anstyle::{AnsiColor, Effects, Style}; use crate::markdown::{MdStream, MdTree}; @@ -15,7 +15,7 @@ thread_local! { } /// Print to terminal output to a buffer -pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Buffer) -> io::Result<()> { +pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Vec) -> io::Result<()> { #[cfg(not(test))] if let Some((w, _)) = termize::dimensions() { WIDTH.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH)); @@ -23,57 +23,65 @@ pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Buffer) -> io::Result< write_stream(stream, buf, None, 0)?; buf.write_all(b"\n") } - /// Write the buffer, reset to the default style after each fn write_stream( MdStream(stream): &MdStream<'_>, - buf: &mut Buffer, - default: Option<&ColorSpec>, + buf: &mut Vec, + default: Option +

// This is a regular comment
+/// This is a doc comment
+fn main() {
+    // Another comment
+    println!("Hello, world!");
+}
\ No newline at end of file diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html index 00925bd81ed8..d058191aef72 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_general.html @@ -127,9 +127,9 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd let y = &mut x; let z = &y; - let Foo { x: z, y } = Foo { x: z, y }; + let Foo { x: z, y } = Foo { x: z, y }; - y; + y; let mut foo = Foo { x, y: x }; let foo2 = Foo { x, y: x }; @@ -142,13 +142,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd copy.qux(); copy.baz(copy); - let a = |x| x; + let a = |x| x; let bar = Foo::baz; let baz = (-42,); let baz = -baz.0; - let _ = !true; + let _ = !true; 'foo: loop { break 'foo; @@ -172,13 +172,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd } async fn learn_and_sing() { - let song = learn_song().await; - sing_song(song).await; + let song = learn_song().await; + sing_song(song).await; } async fn async_main() { let f1 = learn_and_sing(); - let f2 = dance(); + let f2 = dance(); futures::join!(f1, f2); } diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html index 3b468ab6dba6..579c6ceadcb8 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html @@ -43,18 +43,19 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd 
fn fixture(#[rust_analyzer::rust_fixture] ra_fixture: &str) {}
 
 fn main() {
-    fixture(r#"
-trait Foo {
-    fn foo() {
-        println!("2 + 2 = {}", 4);
-    }
+    fixture(r#"
+@@- minicore: sized
+trait Foo: Sized {
+    fn foo() {
+        println!("2 + 2 = {}", 4);
+    }
 }"#
     );
-    fixture(r"
-fn foo() {
-    foo($0{
-        92
-    }$0)
+    fixture(r"
+fn foo() {
+    foo($0{
+        92
+    }$0)
 }"
     );
 }
\ No newline at end of file diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html new file mode 100644 index 000000000000..fc2d9a387016 --- /dev/null +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_injection_2.html @@ -0,0 +1,61 @@ + + +
fn fixture(#[rust_analyzer::rust_fixture] ra_fixture: &str) {}
+
+fn main() {
+    fixture(r#"
+@@- /main.rs crate:main deps:other_crate
+fn test() {
+    let x = other_crate::foo::S::thing();
+    x;
+} //^ i128
+
+@@- /lib.rs crate:other_crate
+pub mod foo {
+    pub struct S;
+    impl S {
+        pub fn thing() -> i128 { 0 }
+    }
+}
+    "#);
+}
\ No newline at end of file diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html index 9c42401ed077..cceb159c9dd4 100644 --- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html +++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html @@ -41,7 +41,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd .unresolved_reference { color: #FC5555; text-decoration: wavy underline; } 
fn main() {
-    1 + 1 - 1 * 1 / 1 % 1 | 1 & 1 ! 1 ^ 1 >> 1 << 1;
+    1 + 1 - 1 * 1 / 1 % 1 | 1 & 1 ! 1 ^ 1 >> 1 << 1;
     let mut a = 0;
     a += 1;
     a -= 1;
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
index f7d798208037..47ee2ad1c0d7 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
@@ -78,8 +78,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 }
 
 use foo::bar as baz;
-trait Bar = Baz;
-trait Foo = Bar;
+trait Bar = Baz;
+trait Foo = Bar;
 
 fn main() {
     let a = '\n';
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
index 828b8f762c58..8339daf32462 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
@@ -96,7 +96,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 
         u.field;
         &u.field;
-        &raw const u.field;
+        &raw const u.field;
         // this should be safe!
         let Union { field: _ };
         // but not these
diff --git a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs
index dd359326c61d..4e84127c29f8 100644
--- a/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/syntax_highlighting/tests.rs
@@ -1,14 +1,15 @@
 use std::time::Instant;
 
 use expect_test::{ExpectFile, expect_file};
-use ide_db::SymbolKind;
+use ide_db::{MiniCore, SymbolKind};
 use span::Edition;
 use test_utils::{AssertLinear, bench, bench_fixture, skip_slow_tests};
 
 use crate::{FileRange, HighlightConfig, HlTag, TextRange, fixture};
 
-const HL_CONFIG: HighlightConfig = HighlightConfig {
+const HL_CONFIG: HighlightConfig<'_> = HighlightConfig {
     strings: true,
+    comments: true,
     punctuation: true,
     specialize_punctuation: true,
     specialize_operator: true,
@@ -16,6 +17,7 @@ const HL_CONFIG: HighlightConfig = HighlightConfig {
     inject_doc_comment: true,
     macro_bang: true,
     syntactic_name_ref_highlighting: false,
+    minicore: MiniCore::default(),
 };
 
 #[test]
@@ -1015,6 +1017,35 @@ impl t for foo {
     )
 }
 
+#[test]
+fn test_injection_2() {
+    check_highlighting(
+        r##"
+fn fixture(#[rust_analyzer::rust_fixture] ra_fixture: &str) {}
+
+fn main() {
+    fixture(r#"
+@@- /main.rs crate:main deps:other_crate
+fn test() {
+    let x = other_crate::foo::S::thing();
+    x;
+} //^ i128
+
+@@- /lib.rs crate:other_crate
+pub mod foo {
+    pub struct S;
+    impl S {
+        pub fn thing() -> i128 { 0 }
+    }
+}
+    "#);
+}
+"##,
+        expect_file!["./test_data/highlight_injection_2.html"],
+        false,
+    );
+}
+
 #[test]
 fn test_injection() {
     check_highlighting(
@@ -1023,7 +1054,8 @@ fn fixture(#[rust_analyzer::rust_fixture] ra_fixture: &str) {}
 
 fn main() {
     fixture(r#"
-trait Foo {
+@@- minicore: sized
+trait Foo: Sized {
     fn foo() {
         println!("2 + 2 = {}", 4);
     }
@@ -1220,14 +1252,23 @@ fn foo(x: &fn(&dyn Trait)) {}
 /// Highlights the code given by the `ra_fixture` argument, renders the
 /// result as HTML, and compares it with the HTML file given as `snapshot`.
 /// Note that the `snapshot` file is overwritten by the rendered HTML.
+fn check_highlighting_with_config(
+    #[rust_analyzer::rust_fixture] ra_fixture: &str,
+    config: HighlightConfig<'_>,
+    expect: ExpectFile,
+    rainbow: bool,
+) {
+    let (analysis, file_id) = fixture::file(ra_fixture.trim());
+    let actual_html = &analysis.highlight_as_html_with_config(config, file_id, rainbow).unwrap();
+    expect.assert_eq(actual_html)
+}
+
 fn check_highlighting(
     #[rust_analyzer::rust_fixture] ra_fixture: &str,
     expect: ExpectFile,
     rainbow: bool,
 ) {
-    let (analysis, file_id) = fixture::file(ra_fixture.trim());
-    let actual_html = &analysis.highlight_as_html(file_id, rainbow).unwrap();
-    expect.assert_eq(actual_html)
+    check_highlighting_with_config(ra_fixture, HL_CONFIG, expect, rainbow)
 }
 
 #[test]
@@ -1435,3 +1476,24 @@ fn main() {
         false,
     );
 }
+
+#[test]
+fn test_comment_highlighting_disabled() {
+    // Test that comments are not highlighted when disabled
+    check_highlighting_with_config(
+        r#"
+// This is a regular comment
+/// This is a doc comment
+fn main() {
+    // Another comment
+    println!("Hello, world!");
+}
+"#,
+        HighlightConfig {
+            comments: false, // Disable comment highlighting
+            ..HL_CONFIG
+        },
+        expect_file!["./test_data/highlight_comments_disabled.html"],
+        false,
+    );
+}
diff --git a/src/tools/rust-analyzer/crates/ide/src/test_explorer.rs b/src/tools/rust-analyzer/crates/ide/src/test_explorer.rs
index 06cbd50e946a..bd60ffe55912 100644
--- a/src/tools/rust-analyzer/crates/ide/src/test_explorer.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/test_explorer.rs
@@ -94,7 +94,7 @@ fn discover_tests_in_module(
         if !f.is_test(db) {
             continue;
         }
-        let nav = f.try_to_nav(db).map(|r| r.call_site);
+        let nav = f.try_to_nav(&sema).map(|r| r.call_site);
         let fn_name = f.name(db).as_str().to_owned();
         r.push(TestItem {
             id: format!("{prefix_id}::{fn_name}"),
diff --git a/src/tools/rust-analyzer/crates/ide/src/view_memory_layout.rs b/src/tools/rust-analyzer/crates/ide/src/view_memory_layout.rs
index 63701a4d15e9..47ca616f3199 100644
--- a/src/tools/rust-analyzer/crates/ide/src/view_memory_layout.rs
+++ b/src/tools/rust-analyzer/crates/ide/src/view_memory_layout.rs
@@ -98,7 +98,7 @@ pub(crate) fn view_memory_layout(
         Definition::BuiltinType(it) => it.ty(db),
         Definition::SelfType(it) => it.self_ty(db),
         Definition::Local(it) => it.ty(db),
-        Definition::Field(it) => it.ty(db),
+        Definition::Field(it) => it.ty(db).to_type(db),
         Definition::Const(it) => it.ty(db),
         Definition::Static(it) => it.ty(db),
         _ => return None,
@@ -141,7 +141,7 @@ pub(crate) fn view_memory_layout(
             if let Ok(child_layout) = child_ty.layout(db) {
                 nodes.push(MemoryLayoutNode {
                     item_name: field.name(db),
-                    typename: child_ty.display(db, display_target).to_string(),
+                    typename: { child_ty.display(db, display_target).to_string() },
                     size: child_layout.size(),
                     alignment: child_layout.align(),
                     offset: match *field {
@@ -219,7 +219,7 @@ mod tests {
     ) -> Option {
         let (analysis, position, _) = fixture::annotations(ra_fixture);
 
-        view_memory_layout(&analysis.db, position)
+        hir::attach_db(&analysis.db, || view_memory_layout(&analysis.db, position))
     }
 
     #[test]
diff --git a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
index 4780743c4d92..756377fe56f7 100644
--- a/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
+++ b/src/tools/rust-analyzer/crates/intern/src/symbol/symbols.rs
@@ -178,6 +178,8 @@ define_symbols! {
     core,
     coroutine_state,
     coroutine,
+    coroutine_return,
+    coroutine_yield,
     count,
     crate_type,
     CStr,
@@ -226,6 +228,8 @@ define_symbols! {
     async_fn_once_output,
     async_fn_mut,
     async_fn,
+    call_ref_future,
+    call_once_future,
     fn_ptr_addr,
     fn_ptr_trait,
     format_alignment,
@@ -416,6 +420,7 @@ define_symbols! {
     rustc_allow_incoherent_impl,
     rustc_builtin_macro,
     rustc_coherence_is_core,
+    rustc_coinductive,
     rustc_const_panic_str,
     rustc_deallocator,
     rustc_deprecated_safe_2024,
@@ -432,6 +437,7 @@ define_symbols! {
     rustc_safe_intrinsic,
     rustc_skip_array_during_method_dispatch,
     rustc_skip_during_method_dispatch,
+    rustc_force_inline,
     semitransparent,
     shl_assign,
     shl,
@@ -510,4 +516,7 @@ define_symbols! {
     flags,
     precision,
     width,
+    never_type_fallback,
+    specialization,
+    min_specialization,
 }
diff --git a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
index 26ee698af081..ad838a6550ec 100644
--- a/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/load-cargo/src/lib.rs
@@ -475,10 +475,10 @@ fn load_crate_graph_into_db(
     }
     let changes = vfs.take_changes();
     for (_, file) in changes {
-        if let vfs::Change::Create(v, _) | vfs::Change::Modify(v, _) = file.change {
-            if let Ok(text) = String::from_utf8(v) {
-                analysis_change.change_file(file.file_id, Some(text))
-            }
+        if let vfs::Change::Create(v, _) | vfs::Change::Modify(v, _) = file.change
+            && let Ok(text) = String::from_utf8(v)
+        {
+            analysis_change.change_file(file.file_id, Some(text))
         }
     }
     let source_roots = source_root_config.partition(vfs);
@@ -533,7 +533,7 @@ impl ProcMacroExpander for Expander {
             current_dir,
         ) {
             Ok(Ok(subtree)) => Ok(subtree),
-            Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)),
+            Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err)),
             Err(err) => Err(ProcMacroExpansionError::System(err.to_string())),
         }
     }
diff --git a/src/tools/rust-analyzer/crates/macros/Cargo.toml b/src/tools/rust-analyzer/crates/macros/Cargo.toml
new file mode 100644
index 000000000000..8184cc6d1c03
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/macros/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "macros"
+version = "0.0.0"
+repository.workspace = true
+description = "Proc macros for rust-analyzer."
+
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+
+[lib]
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+quote = "1"
+syn = "2.0.9"
+synstructure = "0.13.0"
diff --git a/src/tools/rust-analyzer/crates/macros/src/lib.rs b/src/tools/rust-analyzer/crates/macros/src/lib.rs
new file mode 100644
index 000000000000..3f90ecc8f902
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/macros/src/lib.rs
@@ -0,0 +1,203 @@
+//! Proc macros for rust-analyzer.
+
+use quote::{ToTokens, quote};
+use syn::parse_quote;
+use synstructure::decl_derive;
+
+decl_derive!(
+    [TypeFoldable, attributes(type_foldable)] =>
+    /// Derives `TypeFoldable` for the annotated `struct` or `enum` (`union` is not supported).
+    ///
+    /// The fold will produce a value of the same struct or enum variant as the input, with
+    /// each field respectively folded using the `TypeFoldable` implementation for its type.
+    /// However, if a field of a struct or an enum variant is annotated with
+    /// `#[type_foldable(identity)]` then that field will retain its incumbent value (and its
+    /// type is not required to implement `TypeFoldable`).
+    type_foldable_derive
+);
+decl_derive!(
+    [TypeVisitable, attributes(type_visitable)] =>
+    /// Derives `TypeVisitable` for the annotated `struct` or `enum` (`union` is not supported).
+    ///
+    /// Each field of the struct or enum variant will be visited in definition order, using the
+    /// `TypeVisitable` implementation for its type. However, if a field of a struct or an enum
+    /// variant is annotated with `#[type_visitable(ignore)]` then that field will not be
+    /// visited (and its type is not required to implement `TypeVisitable`).
+    type_visitable_derive
+);
+
+fn type_visitable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    if let syn::Data::Union(_) = s.ast().data {
+        panic!("cannot derive on union")
+    }
+
+    // ignore fields with #[type_visitable(ignore)]
+    s.filter(|bi| {
+        let mut ignored = false;
+
+        bi.ast().attrs.iter().for_each(|attr| {
+            if !attr.path().is_ident("type_visitable") {
+                return;
+            }
+            let _ = attr.parse_nested_meta(|nested| {
+                if nested.path.is_ident("ignore") {
+                    ignored = true;
+                }
+                Ok(())
+            });
+        });
+
+        !ignored
+    });
+
+    if !s.ast().generics.lifetimes().any(|lt| lt.lifetime.ident == "db") {
+        s.add_impl_generic(parse_quote! { 'db });
+    }
+
+    s.add_bounds(synstructure::AddBounds::Generics);
+    let body_visit = s.each(|bind| {
+        quote! {
+            match ::rustc_type_ir::VisitorResult::branch(
+                ::rustc_type_ir::TypeVisitable::visit_with(#bind, __visitor)
+            ) {
+                ::core::ops::ControlFlow::Continue(()) => {},
+                ::core::ops::ControlFlow::Break(r) => {
+                    return ::rustc_type_ir::VisitorResult::from_residual(r);
+                },
+            }
+        }
+    });
+    s.bind_with(|_| synstructure::BindStyle::Move);
+
+    s.bound_impl(
+        quote!(::rustc_type_ir::TypeVisitable<::hir_ty::next_solver::DbInterner<'db>>),
+        quote! {
+            fn visit_with<__V: ::rustc_type_ir::TypeVisitor<::hir_ty::next_solver::DbInterner<'db>>>(
+                &self,
+                __visitor: &mut __V
+            ) -> __V::Result {
+                match *self { #body_visit }
+                <__V::Result as ::rustc_type_ir::VisitorResult>::output()
+            }
+        },
+    )
+}
+
+fn type_foldable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    if let syn::Data::Union(_) = s.ast().data {
+        panic!("cannot derive on union")
+    }
+
+    if !s.ast().generics.lifetimes().any(|lt| lt.lifetime.ident == "db") {
+        s.add_impl_generic(parse_quote! { 'db });
+    }
+
+    s.add_bounds(synstructure::AddBounds::Generics);
+    s.bind_with(|_| synstructure::BindStyle::Move);
+    let try_body_fold = s.each_variant(|vi| {
+        let bindings = vi.bindings();
+        vi.construct(|_, index| {
+            let bind = &bindings[index];
+
+            // retain value of fields with #[type_foldable(identity)]
+            if has_ignore_attr(&bind.ast().attrs, "type_foldable", "identity") {
+                bind.to_token_stream()
+            } else {
+                quote! {
+                    ::rustc_type_ir::TypeFoldable::try_fold_with(#bind, __folder)?
+                }
+            }
+        })
+    });
+
+    let body_fold = s.each_variant(|vi| {
+        let bindings = vi.bindings();
+        vi.construct(|_, index| {
+            let bind = &bindings[index];
+
+            // retain value of fields with #[type_foldable(identity)]
+            if has_ignore_attr(&bind.ast().attrs, "type_foldable", "identity") {
+                bind.to_token_stream()
+            } else {
+                quote! {
+                    ::rustc_type_ir::TypeFoldable::fold_with(#bind, __folder)
+                }
+            }
+        })
+    });
+
+    s.bound_impl(
+        quote!(::rustc_type_ir::TypeFoldable<::hir_ty::next_solver::DbInterner<'db>>),
+        quote! {
+            fn try_fold_with<__F: ::rustc_type_ir::FallibleTypeFolder<::hir_ty::next_solver::DbInterner<'db>>>(
+                self,
+                __folder: &mut __F
+            ) -> Result {
+                Ok(match self { #try_body_fold })
+            }
+
+            fn fold_with<__F: ::rustc_type_ir::TypeFolder<::hir_ty::next_solver::DbInterner<'db>>>(
+                self,
+                __folder: &mut __F
+            ) -> Self {
+                match self { #body_fold }
+            }
+        },
+    )
+}
+
+fn has_ignore_attr(attrs: &[syn::Attribute], name: &'static str, meta: &'static str) -> bool {
+    let mut ignored = false;
+    attrs.iter().for_each(|attr| {
+        if !attr.path().is_ident(name) {
+            return;
+        }
+        let _ = attr.parse_nested_meta(|nested| {
+            if nested.path.is_ident(meta) {
+                ignored = true;
+            }
+            Ok(())
+        });
+    });
+
+    ignored
+}
+
+decl_derive!(
+    [UpmapFromRaFixture] => upmap_from_ra_fixture
+);
+
+fn upmap_from_ra_fixture(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    if let syn::Data::Union(_) = s.ast().data {
+        panic!("cannot derive on union")
+    }
+
+    s.add_bounds(synstructure::AddBounds::Generics);
+    s.bind_with(|_| synstructure::BindStyle::Move);
+    let body = s.each_variant(|vi| {
+        let bindings = vi.bindings();
+        vi.construct(|_, index| {
+            let bind = &bindings[index];
+
+            quote! {
+                ::ide_db::ra_fixture::UpmapFromRaFixture::upmap_from_ra_fixture(
+                    #bind, __analysis, __virtual_file_id, __real_file_id,
+                )?
+            }
+        })
+    });
+
+    s.bound_impl(
+        quote!(::ide_db::ra_fixture::UpmapFromRaFixture),
+        quote! {
+            fn upmap_from_ra_fixture(
+                self,
+                __analysis: &::ide_db::ra_fixture::RaFixtureAnalysis,
+                __virtual_file_id: ::ide_db::ra_fixture::FileId,
+                __real_file_id: ::ide_db::ra_fixture::FileId,
+            ) -> Result {
+                Ok(match self { #body })
+            }
+        },
+    )
+}
diff --git a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs
index 04ac85ad43dd..b185556b5c7b 100644
--- a/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs
+++ b/src/tools/rust-analyzer/crates/mbe/src/benchmark.rs
@@ -185,24 +185,22 @@ fn invocation_fixtures(
                     for it in tokens.iter() {
                         collect_from_op(it, builder, seed);
                     }
-                    if i + 1 != cnt {
-                        if let Some(sep) = separator {
-                            match &**sep {
-                                Separator::Literal(it) => {
-                                    builder.push(tt::Leaf::Literal(it.clone()))
+                    if i + 1 != cnt
+                        && let Some(sep) = separator
+                    {
+                        match &**sep {
+                            Separator::Literal(it) => builder.push(tt::Leaf::Literal(it.clone())),
+                            Separator::Ident(it) => builder.push(tt::Leaf::Ident(it.clone())),
+                            Separator::Puncts(puncts) => {
+                                for it in puncts {
+                                    builder.push(tt::Leaf::Punct(*it))
                                 }
-                                Separator::Ident(it) => builder.push(tt::Leaf::Ident(it.clone())),
-                                Separator::Puncts(puncts) => {
-                                    for it in puncts {
-                                        builder.push(tt::Leaf::Punct(*it))
-                                    }
-                                }
-                                Separator::Lifetime(punct, ident) => {
-                                    builder.push(tt::Leaf::Punct(*punct));
-                                    builder.push(tt::Leaf::Ident(ident.clone()));
-                                }
-                            };
-                        }
+                            }
+                            Separator::Lifetime(punct, ident) => {
+                                builder.push(tt::Leaf::Punct(*punct));
+                                builder.push(tt::Leaf::Ident(ident.clone()));
+                            }
+                        };
                     }
                 }
             }
diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs
index a8d5965d480c..189efcd15c2f 100644
--- a/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs
+++ b/src/tools/rust-analyzer/crates/mbe/src/expander/matcher.rs
@@ -475,12 +475,12 @@ fn match_loop_inner<'t>(
                 })
             }
             OpDelimited::Op(Op::Subtree { tokens, delimiter }) => {
-                if let Ok((subtree, _)) = src.clone().expect_subtree() {
-                    if subtree.delimiter.kind == delimiter.kind {
-                        item.stack.push(item.dot);
-                        item.dot = tokens.iter_delimited_with(*delimiter);
-                        cur_items.push(item);
-                    }
+                if let Ok((subtree, _)) = src.clone().expect_subtree()
+                    && subtree.delimiter.kind == delimiter.kind
+                {
+                    item.stack.push(item.dot);
+                    item.dot = tokens.iter_delimited_with(*delimiter);
+                    cur_items.push(item);
                 }
             }
             OpDelimited::Op(Op::Var { kind, name, .. }) => {
diff --git a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs
index 2c046df10f5e..3e4ab8bdc1d8 100644
--- a/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs
+++ b/src/tools/rust-analyzer/crates/mbe/src/expander/transcriber.rs
@@ -401,7 +401,19 @@ fn expand_var(
                     let sub = sub.strip_invisible();
                     let mut span = id;
                     marker(&mut span);
-                    let wrap_in_parens = !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)])
+
+                    // Check if this is a simple negative literal (MINUS + LITERAL)
+                    // that should not be wrapped in parentheses
+                    let is_negative_literal = matches!(
+                        sub.flat_tokens(),
+                        [
+                            tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })),
+                            tt::TokenTree::Leaf(tt::Leaf::Literal(_))
+                        ]
+                    );
+
+                    let wrap_in_parens = !is_negative_literal
+                        && !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)])
                         && sub.try_into_subtree().is_none_or(|it| {
                             it.top_subtree().delimiter.kind == tt::DelimiterKind::Invisible
                         });
diff --git a/src/tools/rust-analyzer/crates/parser/Cargo.toml b/src/tools/rust-analyzer/crates/parser/Cargo.toml
index c7da654de6d9..8384d5bec21a 100644
--- a/src/tools/rust-analyzer/crates/parser/Cargo.toml
+++ b/src/tools/rust-analyzer/crates/parser/Cargo.toml
@@ -19,6 +19,7 @@ rustc-literal-escaper.workspace = true
 tracing = { workspace = true, optional = true }
 
 edition.workspace = true
+winnow = { version = "0.7.13", default-features = false }
 
 [dev-dependencies]
 expect-test = "1.5.1"
diff --git a/src/tools/rust-analyzer/crates/parser/src/frontmatter.rs b/src/tools/rust-analyzer/crates/parser/src/frontmatter.rs
new file mode 100644
index 000000000000..2747db4327c5
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/src/frontmatter.rs
@@ -0,0 +1,348 @@
+// Copied from https://github.com/rust-lang/cargo/blob/367fd9f213750cd40317803dd0a5a3ce3f0c676d/src/cargo/util/frontmatter.rs
+#![expect(dead_code)] // avoid editing
+#![expect(unreachable_pub)] // avoid editing
+#![expect(clippy::useless_format)] // avoid editing
+
+type Span = std::ops::Range;
+
+#[derive(Debug)]
+pub struct ScriptSource<'s> {
+    /// The full file
+    raw: &'s str,
+    /// The `#!/usr/bin/env cargo` line, if present
+    shebang: Option,
+    /// The code fence opener (`---`)
+    open: Option,
+    /// Trailing text after `ScriptSource::open` that identifies the meaning of
+    /// `ScriptSource::frontmatter`
+    info: Option,
+    /// The lines between `ScriptSource::open` and `ScriptSource::close`
+    frontmatter: Option,
+    /// The code fence closer (`---`)
+    close: Option,
+    /// All content after the frontmatter and shebang
+    content: Span,
+}
+
+impl<'s> ScriptSource<'s> {
+    pub fn parse(raw: &'s str) -> Result {
+        use winnow::stream::FindSlice as _;
+        use winnow::stream::Location as _;
+        use winnow::stream::Offset as _;
+        use winnow::stream::Stream as _;
+
+        let content_end = raw.len();
+        let mut source = Self {
+            raw,
+            shebang: None,
+            open: None,
+            info: None,
+            frontmatter: None,
+            close: None,
+            content: 0..content_end,
+        };
+
+        let mut input = winnow::stream::LocatingSlice::new(raw);
+
+        if let Some(shebang_end) = strip_shebang(input.as_ref()) {
+            let shebang_start = input.current_token_start();
+            let _ = input.next_slice(shebang_end);
+            let shebang_end = input.current_token_start();
+            source.shebang = Some(shebang_start..shebang_end);
+            source.content = shebang_end..content_end;
+        }
+
+        // Whitespace may precede a frontmatter but must end with a newline
+        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
+            let _ = input.next_slice(nl_end);
+        }
+
+        // Opens with a line that starts with 3 or more `-` followed by an optional identifier
+        const FENCE_CHAR: char = '-';
+        let fence_length = input
+            .as_ref()
+            .char_indices()
+            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
+            .unwrap_or_else(|| input.eof_offset());
+        let open_start = input.current_token_start();
+        let fence_pattern = input.next_slice(fence_length);
+        let open_end = input.current_token_start();
+        match fence_length {
+            0 => {
+                return Ok(source);
+            }
+            1 | 2 => {
+                // either not a frontmatter or invalid frontmatter opening
+                return Err(FrontmatterError::new(
+                    format!(
+                        "found {fence_length} `{FENCE_CHAR}` in rust frontmatter, expected at least 3"
+                    ),
+                    raw.len()..raw.len(),
+                ).push_visible_span(open_start..open_end));
+            }
+            _ => {}
+        }
+        source.open = Some(open_start..open_end);
+        let Some(info_nl) = input.find_slice("\n") else {
+            return Err(FrontmatterError::new(
+                format!("unclosed frontmatter; expected `{fence_pattern}`"),
+                raw.len()..raw.len(),
+            )
+            .push_visible_span(open_start..open_end));
+        };
+        let info = input.next_slice(info_nl.start);
+        let info = info.strip_suffix('\r').unwrap_or(info); // already excludes `\n`
+        let info = info.trim_matches(is_horizontal_whitespace);
+        if !info.is_empty() {
+            let info_start = info.offset_from(&raw);
+            let info_end = info_start + info.len();
+            source.info = Some(info_start..info_end);
+        }
+
+        // Ends with a line that starts with a matching number of `-` only followed by whitespace
+        let nl_fence_pattern = format!("\n{fence_pattern}");
+        let Some(frontmatter_nl) = input.find_slice(nl_fence_pattern.as_str()) else {
+            for len in (2..(nl_fence_pattern.len() - 1)).rev() {
+                let Some(frontmatter_nl) = input.find_slice(&nl_fence_pattern[0..len]) else {
+                    continue;
+                };
+                let _ = input.next_slice(frontmatter_nl.start + 1);
+                let close_start = input.current_token_start();
+                let _ = input.next_slice(len);
+                let close_end = input.current_token_start();
+                let fewer_dashes = fence_length - len;
+                return Err(FrontmatterError::new(
+                    format!(
+                        "closing code fence has {fewer_dashes} less `-` than the opening fence"
+                    ),
+                    close_start..close_end,
+                )
+                .push_visible_span(open_start..open_end));
+            }
+            return Err(FrontmatterError::new(
+                format!("unclosed frontmatter; expected `{fence_pattern}`"),
+                raw.len()..raw.len(),
+            )
+            .push_visible_span(open_start..open_end));
+        };
+        let frontmatter_start = input.current_token_start() + 1; // skip nl from infostring
+        let _ = input.next_slice(frontmatter_nl.start + 1);
+        let frontmatter_end = input.current_token_start();
+        source.frontmatter = Some(frontmatter_start..frontmatter_end);
+        let close_start = input.current_token_start();
+        let _ = input.next_slice(fence_length);
+        let close_end = input.current_token_start();
+        source.close = Some(close_start..close_end);
+
+        let nl = input.find_slice("\n");
+        let after_closing_fence =
+            input.next_slice(nl.map(|span| span.end).unwrap_or_else(|| input.eof_offset()));
+        let content_start = input.current_token_start();
+        let extra_dashes = after_closing_fence.chars().take_while(|b| *b == FENCE_CHAR).count();
+        if 0 < extra_dashes {
+            let extra_start = close_end;
+            let extra_end = extra_start + extra_dashes;
+            return Err(FrontmatterError::new(
+                format!("closing code fence has {extra_dashes} more `-` than the opening fence"),
+                extra_start..extra_end,
+            )
+            .push_visible_span(open_start..open_end));
+        } else {
+            let after_closing_fence = strip_newline(after_closing_fence);
+            let after_closing_fence = after_closing_fence.trim_matches(is_horizontal_whitespace);
+            if !after_closing_fence.is_empty() {
+                // extra characters beyond the original fence pattern
+                let after_start = after_closing_fence.offset_from(&raw);
+                let after_end = after_start + after_closing_fence.len();
+                return Err(FrontmatterError::new(
+                    format!("unexpected characters after frontmatter close"),
+                    after_start..after_end,
+                )
+                .push_visible_span(open_start..open_end));
+            }
+        }
+
+        source.content = content_start..content_end;
+
+        if let Some(nl_end) = strip_ws_lines(input.as_ref()) {
+            let _ = input.next_slice(nl_end);
+        }
+        let fence_length = input
+            .as_ref()
+            .char_indices()
+            .find_map(|(i, c)| (c != FENCE_CHAR).then_some(i))
+            .unwrap_or_else(|| input.eof_offset());
+        if 0 < fence_length {
+            let fence_start = input.current_token_start();
+            let fence_end = fence_start + fence_length;
+            return Err(FrontmatterError::new(
+                format!("only one frontmatter is supported"),
+                fence_start..fence_end,
+            )
+            .push_visible_span(open_start..open_end)
+            .push_visible_span(close_start..close_end));
+        }
+
+        Ok(source)
+    }
+
+    pub fn shebang(&self) -> Option<&'s str> {
+        self.shebang.clone().map(|span| &self.raw[span])
+    }
+
+    pub fn shebang_span(&self) -> Option {
+        self.shebang.clone()
+    }
+
+    pub fn open_span(&self) -> Option {
+        self.open.clone()
+    }
+
+    pub fn info(&self) -> Option<&'s str> {
+        self.info.clone().map(|span| &self.raw[span])
+    }
+
+    pub fn info_span(&self) -> Option {
+        self.info.clone()
+    }
+
+    pub fn frontmatter(&self) -> Option<&'s str> {
+        self.frontmatter.clone().map(|span| &self.raw[span])
+    }
+
+    pub fn frontmatter_span(&self) -> Option {
+        self.frontmatter.clone()
+    }
+
+    pub fn close_span(&self) -> Option {
+        self.close.clone()
+    }
+
+    pub fn content(&self) -> &'s str {
+        &self.raw[self.content.clone()]
+    }
+
+    pub fn content_span(&self) -> Span {
+        self.content.clone()
+    }
+}
+
+/// Returns the index after the shebang line, if present
+pub fn strip_shebang(input: &str) -> Option {
+    // See rust-lang/rust's compiler/rustc_lexer/src/lib.rs's `strip_shebang`
+    // Shebang must start with `#!` literally, without any preceding whitespace.
+    // For simplicity we consider any line starting with `#!` a shebang,
+    // regardless of restrictions put on shebangs by specific platforms.
+    if let Some(rest) = input.strip_prefix("#!") {
+        // Ok, this is a shebang but if the next non-whitespace token is `[`,
+        // then it may be valid Rust code, so consider it Rust code.
+        //
+        // NOTE: rustc considers line and block comments to be whitespace but to avoid
+        // any more awareness of Rust grammar, we are excluding it.
+        if !rest.trim_start().starts_with('[') {
+            // No other choice than to consider this a shebang.
+            let newline_end = input.find('\n').map(|pos| pos + 1).unwrap_or(input.len());
+            return Some(newline_end);
+        }
+    }
+    None
+}
+
+/// Returns the index after any lines with only whitespace, if present
+pub fn strip_ws_lines(input: &str) -> Option {
+    let ws_end = input.find(|c| !is_whitespace(c)).unwrap_or(input.len());
+    if ws_end == 0 {
+        return None;
+    }
+
+    let nl_start = input[0..ws_end].rfind('\n')?;
+    let nl_end = nl_start + 1;
+    Some(nl_end)
+}
+
+/// True if `c` is considered a whitespace according to Rust language definition.
+/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
+/// for definitions of these classes.
+fn is_whitespace(c: char) -> bool {
+    // This is Pattern_White_Space.
+    //
+    // Note that this set is stable (ie, it doesn't change with different
+    // Unicode versions), so it's ok to just hard-code the values.
+
+    matches!(
+        c,
+        // End-of-line characters
+        | '\u{000A}' // line feed (\n)
+        | '\u{000B}' // vertical tab
+        | '\u{000C}' // form feed
+        | '\u{000D}' // carriage return (\r)
+        | '\u{0085}' // next line (from latin1)
+        | '\u{2028}' // LINE SEPARATOR
+        | '\u{2029}' // PARAGRAPH SEPARATOR
+
+        // `Default_Ignorable_Code_Point` characters
+        | '\u{200E}' // LEFT-TO-RIGHT MARK
+        | '\u{200F}' // RIGHT-TO-LEFT MARK
+
+        // Horizontal space characters
+        | '\u{0009}'   // tab (\t)
+        | '\u{0020}' // space
+    )
+}
+
+/// True if `c` is considered horizontal whitespace according to Rust language definition.
+fn is_horizontal_whitespace(c: char) -> bool {
+    // This is Pattern_White_Space.
+    //
+    // Note that this set is stable (ie, it doesn't change with different
+    // Unicode versions), so it's ok to just hard-code the values.
+
+    matches!(
+        c,
+        // Horizontal space characters
+        '\u{0009}'   // tab (\t)
+        | '\u{0020}' // space
+    )
+}
+
+fn strip_newline(text: &str) -> &str {
+    text.strip_suffix("\r\n").or_else(|| text.strip_suffix('\n')).unwrap_or(text)
+}
+
+#[derive(Debug)]
+pub struct FrontmatterError {
+    message: String,
+    primary_span: Span,
+    visible_spans: Vec,
+}
+
+impl FrontmatterError {
+    pub fn new(message: impl Into, span: Span) -> Self {
+        Self { message: message.into(), primary_span: span, visible_spans: Vec::new() }
+    }
+
+    pub fn push_visible_span(mut self, span: Span) -> Self {
+        self.visible_spans.push(span);
+        self
+    }
+
+    pub fn message(&self) -> &str {
+        self.message.as_str()
+    }
+
+    pub fn primary_span(&self) -> Span {
+        self.primary_span.clone()
+    }
+
+    pub fn visible_spans(&self) -> &[Span] {
+        &self.visible_spans
+    }
+}
+
+impl std::fmt::Display for FrontmatterError {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.message.fmt(fmt)
+    }
+}
+
+impl std::error::Error for FrontmatterError {}
diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar/expressions.rs b/src/tools/rust-analyzer/crates/parser/src/grammar/expressions.rs
index 2b4151e3b752..76d26c1ecdfc 100644
--- a/src/tools/rust-analyzer/crates/parser/src/grammar/expressions.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/grammar/expressions.rs
@@ -77,38 +77,38 @@ pub(super) fn stmt(p: &mut Parser<'_>, semicolon: Semicolon) {
         return;
     }
 
-    if let Some((cm, blocklike)) = expr_stmt(p, Some(m)) {
-        if !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF))) {
-            // test no_semi_after_block
-            // fn foo() {
-            //     if true {}
-            //     loop {}
-            //     match () {}
-            //     while true {}
-            //     for _ in () {}
-            //     {}
-            //     {}
-            //     macro_rules! test {
-            //          () => {}
-            //     }
-            //     test!{}
-            // }
-            let m = cm.precede(p);
-            match semicolon {
-                Semicolon::Required => {
-                    if blocklike.is_block() {
-                        p.eat(T![;]);
-                    } else {
-                        p.expect(T![;]);
-                    }
-                }
-                Semicolon::Optional => {
+    if let Some((cm, blocklike)) = expr_stmt(p, Some(m))
+        && !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF)))
+    {
+        // test no_semi_after_block
+        // fn foo() {
+        //     if true {}
+        //     loop {}
+        //     match () {}
+        //     while true {}
+        //     for _ in () {}
+        //     {}
+        //     {}
+        //     macro_rules! test {
+        //          () => {}
+        //     }
+        //     test!{}
+        // }
+        let m = cm.precede(p);
+        match semicolon {
+            Semicolon::Required => {
+                if blocklike.is_block() {
                     p.eat(T![;]);
+                } else {
+                    p.expect(T![;]);
                 }
-                Semicolon::Forbidden => (),
             }
-            m.complete(p, EXPR_STMT);
+            Semicolon::Optional => {
+                p.eat(T![;]);
+            }
+            Semicolon::Forbidden => (),
         }
+        m.complete(p, EXPR_STMT);
     }
 }
 
@@ -134,14 +134,11 @@ pub(super) fn let_stmt(p: &mut Parser<'_>, with_semi: Semicolon) {
     if p.at(T![else]) {
         // test_err let_else_right_curly_brace
         // fn func() { let Some(_) = {Some(1)} else { panic!("h") };}
-        if let Some(expr) = expr_after_eq {
-            if let Some(token) = expr.last_token(p) {
-                if token == T!['}'] {
-                    p.error(
-                        "right curly brace `}` before `else` in a `let...else` statement not allowed"
-                    )
-                }
-            }
+        if let Some(expr) = expr_after_eq
+            && let Some(token) = expr.last_token(p)
+            && token == T!['}']
+        {
+            p.error("right curly brace `}` before `else` in a `let...else` statement not allowed")
         }
 
         // test let_else
@@ -433,6 +430,11 @@ fn postfix_expr(
             // }
             T!['('] if allow_calls => call_expr(p, lhs),
             T!['['] if allow_calls => index_expr(p, lhs),
+            // test_err postfix_dot_expr_ambiguity
+            // fn foo() {
+            //     x.
+            //     ()
+            // }
             T![.] => match postfix_dot_expr::(p, lhs) {
                 Ok(it) => it,
                 Err(it) => {
@@ -461,6 +463,7 @@ fn postfix_dot_expr(
 
     if PATH_NAME_REF_KINDS.contains(p.nth(nth1))
         && (p.nth(nth2) == T!['('] || p.nth_at(nth2, T![::]))
+        || p.nth(nth1) == T!['(']
     {
         return Ok(method_call_expr::(p, lhs));
     }
@@ -529,19 +532,26 @@ fn method_call_expr(
     lhs: CompletedMarker,
 ) -> CompletedMarker {
     if FLOAT_RECOVERY {
-        assert!(p.at_ts(PATH_NAME_REF_KINDS) && (p.nth(1) == T!['('] || p.nth_at(1, T![::])));
-    } else {
         assert!(
-            p.at(T![.])
-                && PATH_NAME_REF_KINDS.contains(p.nth(1))
-                && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))
+            p.at_ts(PATH_NAME_REF_KINDS) && (p.nth(1) == T!['('] || p.nth_at(1, T![::]))
+                || p.current() == T!['(']
+        );
+    } else {
+        assert!(p.at(T![.]));
+        assert!(
+            PATH_NAME_REF_KINDS.contains(p.nth(1)) && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))
+                || p.nth(1) == T!['(']
         );
     }
     let m = lhs.precede(p);
     if !FLOAT_RECOVERY {
         p.bump(T![.]);
     }
-    name_ref_mod_path(p);
+    if p.at_ts(PATH_NAME_REF_KINDS) {
+        name_ref_mod_path(p);
+    } else {
+        p.error("expected method name, field name or number");
+    }
     generic_args::opt_generic_arg_list_expr(p);
     if p.at(T!['(']) {
         arg_list(p);
diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar/generic_params.rs b/src/tools/rust-analyzer/crates/parser/src/grammar/generic_params.rs
index cb1b59f64977..d419817e5cd7 100644
--- a/src/tools/rust-analyzer/crates/parser/src/grammar/generic_params.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/grammar/generic_params.rs
@@ -182,12 +182,6 @@ fn type_bound(p: &mut Parser<'_>) -> bool {
             );
             m.complete(p, USE_BOUND_GENERIC_ARGS);
         }
-        T![?] if p.nth_at(1, T![for]) => {
-            // test question_for_type_trait_bound
-            // fn f() where T: ?for<> Sized {}
-            p.bump_any();
-            types::for_type(p, false)
-        }
         _ => {
             if path_type_bound(p).is_err() {
                 m.abandon(p);
@@ -219,8 +213,13 @@ fn path_type_bound(p: &mut Parser<'_>) -> Result<(), ()> {
     // test async_trait_bound
     // fn async_foo(_: impl async Fn(&i32)) {}
     p.eat(T![async]);
+    // test question_for_type_trait_bound
+    // fn f() where T: for<> ?Sized {}
     p.eat(T![?]);
 
+    // test_err invalid_question_for_type_trait_bound
+    // fn f() where T: ?for<> Sized {}
+
     if paths::is_use_path_start(p) {
         types::path_type_bounds(p, false);
         // test_err type_bounds_macro_call_recovery
diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar/items/consts.rs b/src/tools/rust-analyzer/crates/parser/src/grammar/items/consts.rs
index 8e255985a205..e6a8aca5861a 100644
--- a/src/tools/rust-analyzer/crates/parser/src/grammar/items/consts.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/grammar/items/consts.rs
@@ -25,21 +25,28 @@ fn const_or_static(p: &mut Parser<'_>, m: Marker, is_const: bool) {
     }
 
     // FIXME: Recover on statics with generic params/where clause.
-    if is_const {
-        // test generic_const
-        // const C: u32 = 0;
-        // impl Foo {
-        //     const C<'a>: &'a () = &();
-        // }
-        generic_params::opt_generic_param_list(p);
+    if !is_const && p.at(T![<]) {
+        // test_err generic_static
+        // static C: u32 = 0;
+        p.error("`static` may not have generic parameters");
     }
-    // test_err generic_static
-    // static C: u32 = 0;
+    // test generic_const
+    // const C: u32 = 0;
+    // impl Foo {
+    //     const C<'a>: &'a () = &();
+    // }
+    generic_params::opt_generic_param_list(p);
 
     if p.at(T![:]) {
         types::ascription(p);
+    } else if is_const {
+        // test_err missing_const_type
+        // const C = 0;
+        p.error("missing type for `const`");
     } else {
-        p.error("missing type for `const` or `static`");
+        // test_err missing_static_type
+        // static C = 0;
+        p.error("missing type for `static`");
     }
     if p.eat(T![=]) {
         expressions::expr(p);
diff --git a/src/tools/rust-analyzer/crates/parser/src/grammar/items/traits.rs b/src/tools/rust-analyzer/crates/parser/src/grammar/items/traits.rs
index 47f86ce8c6cc..c1b1a3fc8a94 100644
--- a/src/tools/rust-analyzer/crates/parser/src/grammar/items/traits.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/grammar/items/traits.rs
@@ -20,7 +20,7 @@ pub(super) fn trait_(p: &mut Parser<'_>, m: Marker) {
         // trait Z = where Self: T;
         generic_params::opt_where_clause(p);
         p.expect(T![;]);
-        m.complete(p, TRAIT_ALIAS);
+        m.complete(p, TRAIT);
         return;
     }
 
diff --git a/src/tools/rust-analyzer/crates/parser/src/input.rs b/src/tools/rust-analyzer/crates/parser/src/input.rs
index 4490956f9704..331bc58dd052 100644
--- a/src/tools/rust-analyzer/crates/parser/src/input.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/input.rs
@@ -61,7 +61,7 @@ impl Input {
     #[inline]
     fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
         let idx = self.len();
-        if idx % (bits::BITS as usize) == 0 {
+        if idx.is_multiple_of(bits::BITS as usize) {
             self.joint.push(0);
         }
         self.kind.push(kind);
diff --git a/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs b/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
index 8fff1c3db748..7c78ba8faf5f 100644
--- a/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/lexed_str.rs
@@ -37,10 +37,17 @@ impl<'a> LexedStr<'a> {
     pub fn new(edition: Edition, text: &'a str) -> LexedStr<'a> {
         let _p = tracing::info_span!("LexedStr::new").entered();
         let mut conv = Converter::new(edition, text);
-        if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
-            conv.res.push(SHEBANG, conv.offset);
-            conv.offset = shebang_len;
-        };
+        if let Ok(script) = crate::frontmatter::ScriptSource::parse(text) {
+            if let Some(shebang) = script.shebang_span() {
+                conv.push(SHEBANG, shebang.end - shebang.start, Vec::new());
+            }
+            if script.frontmatter().is_some() {
+                conv.push(FRONTMATTER, script.content_span().start - conv.offset, Vec::new());
+            }
+        } else if let Some(shebang_len) = rustc_lexer::strip_shebang(text) {
+            // Leave error reporting to `rustc_lexer`
+            conv.push(SHEBANG, shebang_len, Vec::new());
+        }
 
         // Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer
         // but we want to split it to two in edition <2024.
@@ -149,6 +156,24 @@ impl<'a> Converter<'a> {
         }
     }
 
+    /// Check for likely unterminated string by analyzing STRING token content
+    fn has_likely_unterminated_string(&self) -> bool {
+        let Some(last_idx) = self.res.kind.len().checked_sub(1) else { return false };
+
+        for i in (0..=last_idx).rev().take(5) {
+            if self.res.kind[i] == STRING {
+                let start = self.res.start[i] as usize;
+                let end = self.res.start.get(i + 1).map(|&s| s as usize).unwrap_or(self.offset);
+                let content = &self.res.text[start..end];
+
+                if content.contains('(') && (content.contains("//") || content.contains(";\n")) {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
     fn finalize_with_eof(mut self) -> LexedStr<'a> {
         self.res.push(EOF, self.offset);
         self.res
@@ -267,7 +292,16 @@ impl<'a> Converter<'a> {
                 rustc_lexer::TokenKind::Unknown => ERROR,
                 rustc_lexer::TokenKind::UnknownPrefix if token_text == "builtin" => IDENT,
                 rustc_lexer::TokenKind::UnknownPrefix => {
-                    errors.push("unknown literal prefix".into());
+                    let has_unterminated = self.has_likely_unterminated_string();
+
+                    let error_msg = if has_unterminated {
+                        format!(
+                            "unknown literal prefix `{token_text}` (note: check for unterminated string literal)"
+                        )
+                    } else {
+                        "unknown literal prefix".to_owned()
+                    };
+                    errors.push(error_msg);
                     IDENT
                 }
                 rustc_lexer::TokenKind::Eof => EOF,
diff --git a/src/tools/rust-analyzer/crates/parser/src/lib.rs b/src/tools/rust-analyzer/crates/parser/src/lib.rs
index 7963f00bb25c..53444ef52cff 100644
--- a/src/tools/rust-analyzer/crates/parser/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/lib.rs
@@ -26,6 +26,7 @@ extern crate ra_ap_rustc_lexer as rustc_lexer;
 extern crate rustc_lexer;
 
 mod event;
+mod frontmatter;
 mod grammar;
 mod input;
 mod lexed_str;
diff --git a/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs b/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs
index e2baec890c3a..d5e513933f7a 100644
--- a/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/shortcuts.rs
@@ -252,10 +252,10 @@ fn n_attached_trivias<'a>(
                     WHITESPACE if text.contains("\n\n") => {
                         // we check whether the next token is a doc-comment
                         // and skip the whitespace in this case
-                        if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
-                            if is_outer(peek_text) {
-                                continue;
-                            }
+                        if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair)
+                            && is_outer(peek_text)
+                        {
+                            continue;
                         }
                         break;
                     }
diff --git a/src/tools/rust-analyzer/crates/parser/src/syntax_kind/generated.rs b/src/tools/rust-analyzer/crates/parser/src/syntax_kind/generated.rs
index 3a8041d2df9e..93e02a92abda 100644
--- a/src/tools/rust-analyzer/crates/parser/src/syntax_kind/generated.rs
+++ b/src/tools/rust-analyzer/crates/parser/src/syntax_kind/generated.rs
@@ -284,7 +284,6 @@ pub enum SyntaxKind {
     STRUCT,
     TOKEN_TREE,
     TRAIT,
-    TRAIT_ALIAS,
     TRY_EXPR,
     TUPLE_EXPR,
     TUPLE_FIELD,
@@ -457,7 +456,6 @@ impl SyntaxKind {
             | STRUCT
             | TOKEN_TREE
             | TRAIT
-            | TRAIT_ALIAS
             | TRY_EXPR
             | TUPLE_EXPR
             | TUPLE_FIELD
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/generated/runner.rs b/src/tools/rust-analyzer/crates/parser/test_data/generated/runner.rs
index c642e1a3354f..9bdbe5633033 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/generated/runner.rs
+++ b/src/tools/rust-analyzer/crates/parser/test_data/generated/runner.rs
@@ -796,6 +796,12 @@ mod err {
     #[test]
     fn impl_type() { run_and_expect_errors("test_data/parser/inline/err/impl_type.rs"); }
     #[test]
+    fn invalid_question_for_type_trait_bound() {
+        run_and_expect_errors(
+            "test_data/parser/inline/err/invalid_question_for_type_trait_bound.rs",
+        );
+    }
+    #[test]
     fn let_else_right_curly_brace() {
         run_and_expect_errors("test_data/parser/inline/err/let_else_right_curly_brace.rs");
     }
@@ -818,10 +824,18 @@ mod err {
         run_and_expect_errors("test_data/parser/inline/err/misplaced_label_err.rs");
     }
     #[test]
+    fn missing_const_type() {
+        run_and_expect_errors("test_data/parser/inline/err/missing_const_type.rs");
+    }
+    #[test]
     fn missing_fn_param_type() {
         run_and_expect_errors("test_data/parser/inline/err/missing_fn_param_type.rs");
     }
     #[test]
+    fn missing_static_type() {
+        run_and_expect_errors("test_data/parser/inline/err/missing_static_type.rs");
+    }
+    #[test]
     fn path_item_without_excl() {
         run_and_expect_errors("test_data/parser/inline/err/path_item_without_excl.rs");
     }
@@ -830,6 +844,10 @@ mod err {
         run_and_expect_errors("test_data/parser/inline/err/pointer_type_no_mutability.rs");
     }
     #[test]
+    fn postfix_dot_expr_ambiguity() {
+        run_and_expect_errors("test_data/parser/inline/err/postfix_dot_expr_ambiguity.rs");
+    }
+    #[test]
     fn precise_capturing_invalid() {
         run_and_expect_errors("test_data/parser/inline/err/precise_capturing_invalid.rs");
     }
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rast b/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rast
new file mode 100644
index 000000000000..f7f24ca3f810
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rast
@@ -0,0 +1,15 @@
+FN_KW "fn"
+WHITESPACE " "
+IDENT "main"
+L_PAREN "("
+R_PAREN ")"
+WHITESPACE " "
+L_CURLY "{"
+WHITESPACE "\n    "
+IDENT "hello"
+L_PAREN "("
+STRING "\"world);\n    // a bunch of code was here\n    env(\"FLAGS"
+STRING "\", \""
+MINUS "-"
+IDENT "help" error: unknown literal prefix `help` (note: check for unterminated string literal)
+STRING "\")\n}" error: Missing trailing `"` symbol to terminate the string literal
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rs b/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rs
new file mode 100644
index 000000000000..338b9582605b
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/err/unterminated_string_unknown_prefix.rs
@@ -0,0 +1,5 @@
+fn main() {
+    hello("world);
+    // a bunch of code was here
+    env("FLAGS", "-help")
+}
\ No newline at end of file
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rast b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rast
new file mode 100644
index 000000000000..2c7d3cdb1227
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rast
@@ -0,0 +1,12 @@
+FRONTMATTER "\n---\n[dependencies]\nclap = \"4\"\n---\n"
+WHITESPACE "\n"
+FN_KW "fn"
+WHITESPACE " "
+IDENT "main"
+L_PAREN "("
+R_PAREN ")"
+WHITESPACE " "
+L_CURLY "{"
+WHITESPACE "\n"
+R_CURLY "}"
+WHITESPACE "\n"
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rs b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rs
new file mode 100644
index 000000000000..be7bf74fdba2
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/frontmatter.rs
@@ -0,0 +1,8 @@
+
+---
+[dependencies]
+clap = "4"
+---
+
+fn main() {
+}
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rast b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rast
new file mode 100644
index 000000000000..fb4787f4001f
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rast
@@ -0,0 +1,13 @@
+SHEBANG "#!/usr/bin/env cargo\n"
+FRONTMATTER "\n---\n[dependencies]\nclap = \"4\"\n---\n"
+WHITESPACE "\n"
+FN_KW "fn"
+WHITESPACE " "
+IDENT "main"
+L_PAREN "("
+R_PAREN ")"
+WHITESPACE " "
+L_CURLY "{"
+WHITESPACE "\n"
+R_CURLY "}"
+WHITESPACE "\n"
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rs b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rs
new file mode 100644
index 000000000000..090b7713feb3
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/shebang_frontmatter.rs
@@ -0,0 +1,9 @@
+#!/usr/bin/env cargo
+
+---
+[dependencies]
+clap = "4"
+---
+
+fn main() {
+}
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/single_line_comments.rast b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/single_line_comments.rast
index a7681e9f5086..c4e531b449f7 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/single_line_comments.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/lexer/ok/single_line_comments.rast
@@ -1,5 +1,4 @@
-SHEBANG "#!/usr/bin/env bash"
-WHITESPACE "\n"
+SHEBANG "#!/usr/bin/env bash\n"
 COMMENT "// hello"
 WHITESPACE "\n"
 COMMENT "//! World"
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0002_duplicate_shebang.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0002_duplicate_shebang.rast
index 3159a15a3b1c..7ee1ecfbb159 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0002_duplicate_shebang.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0002_duplicate_shebang.rast
@@ -1,6 +1,5 @@
 SOURCE_FILE
-  SHEBANG "#!/use/bin/env rusti"
-  WHITESPACE "\n"
+  SHEBANG "#!/use/bin/env rusti\n"
   ATTR
     POUND "#"
     BANG "!"
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0044_item_modifiers.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0044_item_modifiers.rast
index d6e3219c3957..7a3ca66476e3 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0044_item_modifiers.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/err/0044_item_modifiers.rast
@@ -42,7 +42,7 @@ SOURCE_FILE
   WHITESPACE "\n"
 error 6: expected fn, trait or impl
 error 38: expected a name
-error 40: missing type for `const` or `static`
+error 40: missing type for `const`
 error 40: expected SEMICOLON
 error 44: expected an item
 error 44: expected an item
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/generic_static.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/generic_static.rast
index 485ad11f233a..08017cb0fcd9 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/generic_static.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/generic_static.rast
@@ -4,39 +4,24 @@ SOURCE_FILE
     WHITESPACE " "
     NAME
       IDENT "C"
-  ERROR
-    L_ANGLE "<"
-  ERROR
-    PATH
-      PATH_SEGMENT
-        NAME_REF
+    GENERIC_PARAM_LIST
+      L_ANGLE "<"
+      TYPE_PARAM
+        NAME
           IDENT "i32"
-  ERROR
-    R_ANGLE ">"
-  ERROR
+      R_ANGLE ">"
     COLON ":"
-  WHITESPACE " "
-  ERROR
-    PATH
-      PATH_SEGMENT
-        NAME_REF
-          IDENT "u32"
-  WHITESPACE " "
-  ERROR
+    WHITESPACE " "
+    PATH_TYPE
+      PATH
+        PATH_SEGMENT
+          NAME_REF
+            IDENT "u32"
+    WHITESPACE " "
     EQ "="
-  WHITESPACE " "
-  ERROR
-    INT_NUMBER "0"
-  ERROR
+    WHITESPACE " "
+    LITERAL
+      INT_NUMBER "0"
     SEMICOLON ";"
   WHITESPACE "\n"
-error 8: missing type for `const` or `static`
-error 8: expected SEMICOLON
-error 8: expected an item
-error 12: expected an item
-error 12: expected an item
-error 13: expected an item
-error 18: expected an item
-error 19: expected an item
-error 21: expected an item
-error 22: expected an item
+error 8: `static` may not have generic parameters
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rast
new file mode 100644
index 000000000000..b060ee81d617
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rast
@@ -0,0 +1,49 @@
+SOURCE_FILE
+  FN
+    FN_KW "fn"
+    WHITESPACE " "
+    NAME
+      IDENT "f"
+    GENERIC_PARAM_LIST
+      L_ANGLE "<"
+      TYPE_PARAM
+        NAME
+          IDENT "T"
+      R_ANGLE ">"
+    PARAM_LIST
+      L_PAREN "("
+      R_PAREN ")"
+    WHITESPACE " "
+    WHERE_CLAUSE
+      WHERE_KW "where"
+      WHITESPACE " "
+      WHERE_PRED
+        PATH_TYPE
+          PATH
+            PATH_SEGMENT
+              NAME_REF
+                IDENT "T"
+        COLON ":"
+        WHITESPACE " "
+        TYPE_BOUND_LIST
+          QUESTION "?"
+      WHERE_PRED
+        FOR_BINDER
+          FOR_KW "for"
+          GENERIC_PARAM_LIST
+            L_ANGLE "<"
+            R_ANGLE ">"
+        WHITESPACE " "
+        PATH_TYPE
+          PATH
+            PATH_SEGMENT
+              NAME_REF
+                IDENT "Sized"
+    WHITESPACE " "
+    BLOCK_EXPR
+      STMT_LIST
+        L_CURLY "{"
+        R_CURLY "}"
+  WHITESPACE "\n"
+error 20: expected comma
+error 31: expected colon
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rs b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rs
new file mode 100644
index 000000000000..f80dd90d446c
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/invalid_question_for_type_trait_bound.rs
@@ -0,0 +1 @@
+fn f() where T: ?for<> Sized {}
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rast
new file mode 100644
index 000000000000..11097232a683
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rast
@@ -0,0 +1,14 @@
+SOURCE_FILE
+  CONST
+    CONST_KW "const"
+    WHITESPACE " "
+    NAME
+      IDENT "C"
+    WHITESPACE " "
+    EQ "="
+    WHITESPACE " "
+    LITERAL
+      INT_NUMBER "0"
+    SEMICOLON ";"
+  WHITESPACE "\n"
+error 7: missing type for `const`
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rs b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rs
new file mode 100644
index 000000000000..e3ce44dadfbd
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_const_type.rs
@@ -0,0 +1 @@
+const C = 0;
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rast
new file mode 100644
index 000000000000..a9595401d6d6
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rast
@@ -0,0 +1,14 @@
+SOURCE_FILE
+  STATIC
+    STATIC_KW "static"
+    WHITESPACE " "
+    NAME
+      IDENT "C"
+    WHITESPACE " "
+    EQ "="
+    WHITESPACE " "
+    LITERAL
+      INT_NUMBER "0"
+    SEMICOLON ";"
+  WHITESPACE "\n"
+error 8: missing type for `static`
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rs b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rs
new file mode 100644
index 000000000000..33ecedf80772
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/missing_static_type.rs
@@ -0,0 +1 @@
+static C = 0;
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rast
new file mode 100644
index 000000000000..4ee318de2515
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rast
@@ -0,0 +1,29 @@
+SOURCE_FILE
+  FN
+    FN_KW "fn"
+    WHITESPACE " "
+    NAME
+      IDENT "foo"
+    PARAM_LIST
+      L_PAREN "("
+      R_PAREN ")"
+    WHITESPACE " "
+    BLOCK_EXPR
+      STMT_LIST
+        L_CURLY "{"
+        WHITESPACE "\n    "
+        METHOD_CALL_EXPR
+          PATH_EXPR
+            PATH
+              PATH_SEGMENT
+                NAME_REF
+                  IDENT "x"
+          DOT "."
+          WHITESPACE "\n    "
+          ARG_LIST
+            L_PAREN "("
+            R_PAREN ")"
+        WHITESPACE "\n"
+        R_CURLY "}"
+  WHITESPACE "\n"
+error 17: expected method name, field name or number
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rs b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rs
new file mode 100644
index 000000000000..c1aed3034288
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/err/postfix_dot_expr_ambiguity.rs
@@ -0,0 +1,4 @@
+fn foo() {
+    x.
+    ()
+}
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rast
index cb296153c8f1..69db1aee2c5a 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rast
@@ -27,19 +27,18 @@ SOURCE_FILE
         WHITESPACE " "
         TYPE_BOUND_LIST
           TYPE_BOUND
+            FOR_BINDER
+              FOR_KW "for"
+              GENERIC_PARAM_LIST
+                L_ANGLE "<"
+                R_ANGLE ">"
+            WHITESPACE " "
             QUESTION "?"
-            FOR_TYPE
-              FOR_BINDER
-                FOR_KW "for"
-                GENERIC_PARAM_LIST
-                  L_ANGLE "<"
-                  R_ANGLE ">"
-              WHITESPACE " "
-              PATH_TYPE
-                PATH
-                  PATH_SEGMENT
-                    NAME_REF
-                      IDENT "Sized"
+            PATH_TYPE
+              PATH
+                PATH_SEGMENT
+                  NAME_REF
+                    IDENT "Sized"
     WHITESPACE " "
     BLOCK_EXPR
       STMT_LIST
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rs b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rs
index f80dd90d446c..96353df3b9b3 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rs
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/question_for_type_trait_bound.rs
@@ -1 +1 @@
-fn f() where T: ?for<> Sized {}
+fn f() where T: for<> ?Sized {}
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias.rast
index c45f87089800..2ef66484ae48 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias.rast
@@ -1,5 +1,5 @@
 SOURCE_FILE
-  TRAIT_ALIAS
+  TRAIT
     TRAIT_KW "trait"
     WHITESPACE " "
     NAME
diff --git a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias_where_clause.rast b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias_where_clause.rast
index 8f678247731d..4443d9d14263 100644
--- a/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias_where_clause.rast
+++ b/src/tools/rust-analyzer/crates/parser/test_data/parser/inline/ok/trait_alias_where_clause.rast
@@ -1,5 +1,5 @@
 SOURCE_FILE
-  TRAIT_ALIAS
+  TRAIT
     TRAIT_KW "trait"
     WHITESPACE " "
     NAME
@@ -50,7 +50,7 @@ SOURCE_FILE
                     IDENT "Copy"
     SEMICOLON ";"
   WHITESPACE "\n"
-  TRAIT_ALIAS
+  TRAIT
     TRAIT_KW "trait"
     WHITESPACE " "
     NAME
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
new file mode 100644
index 000000000000..ee96b899fe57
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol.rs
@@ -0,0 +1,172 @@
+//! The initial proc-macro-srv protocol, soon to be deprecated.
+
+pub mod json;
+pub mod msg;
+
+use std::{
+    io::{BufRead, Write},
+    sync::Arc,
+};
+
+use paths::AbsPath;
+use span::Span;
+
+use crate::{
+    ProcMacro, ProcMacroKind, ServerError,
+    legacy_protocol::{
+        json::{read_json, write_json},
+        msg::{
+            ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response,
+            ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map,
+            flat::serialize_span_data_index_map,
+        },
+    },
+    process::ProcMacroServerProcess,
+    version,
+};
+
+pub(crate) use crate::legacy_protocol::msg::SpanMode;
+
+/// Legacy span type, only defined here as it is still used by the proc-macro server.
+/// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for
+/// proc-macro expansion.
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SpanId(pub u32);
+
+impl std::fmt::Debug for SpanId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+pub(crate) fn version_check(srv: &ProcMacroServerProcess) -> Result {
+    let request = Request::ApiVersionCheck {};
+    let response = send_task(srv, request)?;
+
+    match response {
+        Response::ApiVersionCheck(version) => Ok(version),
+        _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+    }
+}
+
+/// Enable support for rust-analyzer span mode if the server supports it.
+pub(crate) fn enable_rust_analyzer_spans(
+    srv: &ProcMacroServerProcess,
+) -> Result {
+    let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer });
+    let response = send_task(srv, request)?;
+
+    match response {
+        Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode),
+        _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+    }
+}
+
+/// Finds proc-macros in a given dynamic library.
+pub(crate) fn find_proc_macros(
+    srv: &ProcMacroServerProcess,
+    dylib_path: &AbsPath,
+) -> Result, String>, ServerError> {
+    let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() };
+
+    let response = send_task(srv, request)?;
+
+    match response {
+        Response::ListMacros(it) => Ok(it),
+        _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+    }
+}
+
+pub(crate) fn expand(
+    proc_macro: &ProcMacro,
+    subtree: tt::SubtreeView<'_, Span>,
+    attr: Option>,
+    env: Vec<(String, String)>,
+    def_site: Span,
+    call_site: Span,
+    mixed_site: Span,
+    current_dir: String,
+) -> Result>, String>, crate::ServerError>
+{
+    let version = proc_macro.process.version();
+    let mut span_data_table = SpanDataIndexMap::default();
+    let def_site = span_data_table.insert_full(def_site).0;
+    let call_site = span_data_table.insert_full(call_site).0;
+    let mixed_site = span_data_table.insert_full(mixed_site).0;
+    let task = ExpandMacro {
+        data: ExpandMacroData {
+            macro_body: FlatTree::new(subtree, version, &mut span_data_table),
+            macro_name: proc_macro.name.to_string(),
+            attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)),
+            has_global_spans: ExpnGlobals {
+                serialize: version >= version::HAS_GLOBAL_SPANS,
+                def_site,
+                call_site,
+                mixed_site,
+            },
+            span_data_table: if proc_macro.process.rust_analyzer_spans() {
+                serialize_span_data_index_map(&span_data_table)
+            } else {
+                Vec::new()
+            },
+        },
+        lib: proc_macro.dylib_path.to_path_buf().into(),
+        env,
+        current_dir: Some(current_dir),
+    };
+
+    let response = send_task(&proc_macro.process, Request::ExpandMacro(Box::new(task)))?;
+
+    match response {
+        Response::ExpandMacro(it) => Ok(it
+            .map(|tree| {
+                let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
+                if proc_macro.needs_fixup_change() {
+                    proc_macro.change_fixup_to_match_old_server(&mut expanded);
+                }
+                expanded
+            })
+            .map_err(|msg| msg.0)),
+        Response::ExpandMacroExtended(it) => Ok(it
+            .map(|resp| {
+                let mut expanded = FlatTree::to_subtree_resolved(
+                    resp.tree,
+                    version,
+                    &deserialize_span_data_index_map(&resp.span_data_table),
+                );
+                if proc_macro.needs_fixup_change() {
+                    proc_macro.change_fixup_to_match_old_server(&mut expanded);
+                }
+                expanded
+            })
+            .map_err(|msg| msg.0)),
+        _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+    }
+}
+
+/// Sends a request to the proc-macro server and waits for a response.
+fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result {
+    if let Some(server_error) = srv.exited() {
+        return Err(server_error.clone());
+    }
+
+    srv.send_task(send_request, req)
+}
+
+/// Sends a request to the server and reads the response.
+fn send_request(
+    mut writer: &mut dyn Write,
+    mut reader: &mut dyn BufRead,
+    req: Request,
+    buf: &mut String,
+) -> Result, ServerError> {
+    req.write(write_json, &mut writer).map_err(|err| ServerError {
+        message: "failed to write request".into(),
+        io: Some(Arc::new(err)),
+    })?;
+    let res = Response::read(read_json, &mut reader, buf).map_err(|err| ServerError {
+        message: "failed to read response".into(),
+        io: Some(Arc::new(err)),
+    })?;
+    Ok(res)
+}
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs
index 165936269d35..b795c4558956 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg.rs
@@ -1,5 +1,6 @@
 //! Defines messages for cross-process message passing based on `ndjson` wire protocol
 pub(crate) mod flat;
+pub use self::flat::*;
 
 use std::io::{self, BufRead, Write};
 
@@ -9,24 +10,6 @@ use serde_derive::{Deserialize, Serialize};
 
 use crate::ProcMacroKind;
 
-pub use self::flat::{
-    FlatTree, SpanDataIndexMap, deserialize_span_data_index_map, serialize_span_data_index_map,
-};
-pub use span::TokenId;
-
-// The versions of the server protocol
-pub const NO_VERSION_CHECK_VERSION: u32 = 0;
-pub const VERSION_CHECK_VERSION: u32 = 1;
-pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2;
-pub const HAS_GLOBAL_SPANS: u32 = 3;
-pub const RUST_ANALYZER_SPAN_SUPPORT: u32 = 4;
-/// Whether literals encode their kind as an additional u32 field and idents their rawness as a u32 field.
-pub const EXTENDED_LEAF_DATA: u32 = 5;
-pub const HASHED_AST_ID: u32 = 6;
-
-/// Current API version of the proc-macro protocol.
-pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID;
-
 /// Represents requests sent from the client to the proc-macro-srv.
 #[derive(Debug, Serialize, Deserialize)]
 pub enum Request {
@@ -48,7 +31,7 @@ pub enum Request {
 }
 
 /// Defines the mode used for handling span data.
-#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize)]
+#[derive(Copy, Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub enum SpanMode {
     /// Default mode, where spans are identified by an ID.
     #[default]
@@ -210,6 +193,8 @@ mod tests {
         TopSubtreeBuilder,
     };
 
+    use crate::version;
+
     use super::*;
 
     fn fixture_token_tree() -> TopSubtree {
@@ -308,7 +293,7 @@ mod tests {
     #[test]
     fn test_proc_macro_rpc_works() {
         let tt = fixture_token_tree();
-        for v in RUST_ANALYZER_SPAN_SUPPORT..=CURRENT_API_VERSION {
+        for v in version::RUST_ANALYZER_SPAN_SUPPORT..=version::CURRENT_API_VERSION {
             let mut span_data_table = Default::default();
             let task = ExpandMacro {
                 data: ExpandMacroData {
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
index 597ffa05d203..fb3542d24f46 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
@@ -40,9 +40,12 @@ use std::collections::VecDeque;
 use intern::Symbol;
 use rustc_hash::FxHashMap;
 use serde_derive::{Deserialize, Serialize};
-use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange, TokenId};
+use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange};
 
-use crate::legacy_protocol::msg::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA};
+use crate::{
+    legacy_protocol::SpanId,
+    version::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA},
+};
 
 pub type SpanDataIndexMap =
     indexmap::IndexSet>;
@@ -62,7 +65,7 @@ pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec {
 }
 
 pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap {
-    debug_assert!(map.len() % 5 == 0);
+    debug_assert!(map.len().is_multiple_of(5));
     map.chunks_exact(5)
         .map(|span| {
             let &[file_id, ast_id, start, end, e] = span else { unreachable!() };
@@ -91,27 +94,27 @@ pub struct FlatTree {
 }
 
 struct SubtreeRepr {
-    open: TokenId,
-    close: TokenId,
+    open: SpanId,
+    close: SpanId,
     kind: tt::DelimiterKind,
     tt: [u32; 2],
 }
 
 struct LiteralRepr {
-    id: TokenId,
+    id: SpanId,
     text: u32,
     suffix: u32,
     kind: u16,
 }
 
 struct PunctRepr {
-    id: TokenId,
+    id: SpanId,
     char: char,
     spacing: tt::Spacing,
 }
 
 struct IdentRepr {
-    id: TokenId,
+    id: SpanId,
     text: u32,
     is_raw: bool,
 }
@@ -122,7 +125,7 @@ impl FlatTree {
         version: u32,
         span_data_table: &mut SpanDataIndexMap,
     ) -> FlatTree {
-        let mut w = Writer {
+        let mut w = Writer:: {
             string_table: FxHashMap::default(),
             work: VecDeque::new(),
             span_data_table,
@@ -159,8 +162,11 @@ impl FlatTree {
         }
     }
 
-    pub fn new_raw(subtree: tt::SubtreeView<'_, TokenId>, version: u32) -> FlatTree {
-        let mut w = Writer {
+    pub fn new_raw>(
+        subtree: tt::SubtreeView<'_, T::Span>,
+        version: u32,
+    ) -> FlatTree {
+        let mut w = Writer:: {
             string_table: FxHashMap::default(),
             work: VecDeque::new(),
             span_data_table: &mut (),
@@ -202,7 +208,7 @@ impl FlatTree {
         version: u32,
         span_data_table: &SpanDataIndexMap,
     ) -> tt::TopSubtree {
-        Reader {
+        Reader:: {
             subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
                 read_vec(self.subtree, SubtreeRepr::read_with_close_span)
             } else {
@@ -227,8 +233,11 @@ impl FlatTree {
         .read()
     }
 
-    pub fn to_subtree_unresolved(self, version: u32) -> tt::TopSubtree {
-        Reader {
+    pub fn to_subtree_unresolved>(
+        self,
+        version: u32,
+    ) -> tt::TopSubtree {
+        Reader:: {
             subtree: if version >= ENCODE_CLOSE_SPAN_VERSION {
                 read_vec(self.subtree, SubtreeRepr::read_with_close_span)
             } else {
@@ -283,7 +292,7 @@ impl SubtreeRepr {
             3 => tt::DelimiterKind::Bracket,
             other => panic!("bad kind {other}"),
         };
-        SubtreeRepr { open: TokenId(open), close: TokenId(!0), kind, tt: [lo, len] }
+        SubtreeRepr { open: SpanId(open), close: SpanId(!0), kind, tt: [lo, len] }
     }
     fn write_with_close_span(self) -> [u32; 5] {
         let kind = match self.kind {
@@ -302,7 +311,7 @@ impl SubtreeRepr {
             3 => tt::DelimiterKind::Bracket,
             other => panic!("bad kind {other}"),
         };
-        SubtreeRepr { open: TokenId(open), close: TokenId(close), kind, tt: [lo, len] }
+        SubtreeRepr { open: SpanId(open), close: SpanId(close), kind, tt: [lo, len] }
     }
 }
 
@@ -311,13 +320,13 @@ impl LiteralRepr {
         [self.id.0, self.text]
     }
     fn read([id, text]: [u32; 2]) -> LiteralRepr {
-        LiteralRepr { id: TokenId(id), text, kind: 0, suffix: !0 }
+        LiteralRepr { id: SpanId(id), text, kind: 0, suffix: !0 }
     }
     fn write_with_kind(self) -> [u32; 4] {
         [self.id.0, self.text, self.kind as u32, self.suffix]
     }
     fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr {
-        LiteralRepr { id: TokenId(id), text, kind: kind as u16, suffix }
+        LiteralRepr { id: SpanId(id), text, kind: kind as u16, suffix }
     }
 }
 
@@ -335,7 +344,7 @@ impl PunctRepr {
             1 => tt::Spacing::Joint,
             other => panic!("bad spacing {other}"),
         };
-        PunctRepr { id: TokenId(id), char: char.try_into().unwrap(), spacing }
+        PunctRepr { id: SpanId(id), char: char.try_into().unwrap(), spacing }
     }
 }
 
@@ -344,44 +353,46 @@ impl IdentRepr {
         [self.id.0, self.text]
     }
     fn read(data: [u32; 2]) -> IdentRepr {
-        IdentRepr { id: TokenId(data[0]), text: data[1], is_raw: false }
+        IdentRepr { id: SpanId(data[0]), text: data[1], is_raw: false }
     }
     fn write_with_rawness(self) -> [u32; 3] {
         [self.id.0, self.text, self.is_raw as u32]
     }
     fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr {
-        IdentRepr { id: TokenId(id), text, is_raw: is_raw == 1 }
+        IdentRepr { id: SpanId(id), text, is_raw: is_raw == 1 }
     }
 }
 
-trait InternableSpan: Copy {
+pub trait SpanTransformer {
     type Table;
-    fn token_id_of(table: &mut Self::Table, s: Self) -> TokenId;
-    fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self;
+    type Span: Copy;
+    fn token_id_of(table: &mut Self::Table, s: Self::Span) -> SpanId;
+    fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span;
 }
-
-impl InternableSpan for TokenId {
+impl SpanTransformer for SpanId {
     type Table = ();
-    fn token_id_of((): &mut Self::Table, token_id: Self) -> TokenId {
+    type Span = Self;
+    fn token_id_of((): &mut Self::Table, token_id: Self::Span) -> SpanId {
         token_id
     }
 
-    fn span_for_token_id((): &Self::Table, id: TokenId) -> Self {
+    fn span_for_token_id((): &Self::Table, id: SpanId) -> Self::Span {
         id
     }
 }
-impl InternableSpan for Span {
+impl SpanTransformer for Span {
     type Table = SpanDataIndexMap;
-    fn token_id_of(table: &mut Self::Table, span: Self) -> TokenId {
-        TokenId(table.insert_full(span).0 as u32)
+    type Span = Self;
+    fn token_id_of(table: &mut Self::Table, span: Self::Span) -> SpanId {
+        SpanId(table.insert_full(span).0 as u32)
     }
-    fn span_for_token_id(table: &Self::Table, id: TokenId) -> Self {
+    fn span_for_token_id(table: &Self::Table, id: SpanId) -> Self::Span {
         *table.get_index(id.0 as usize).unwrap_or_else(|| &table[0])
     }
 }
 
-struct Writer<'a, 'span, S: InternableSpan> {
-    work: VecDeque<(usize, tt::iter::TtIter<'a, S>)>,
+struct Writer<'a, 'span, S: SpanTransformer> {
+    work: VecDeque<(usize, tt::iter::TtIter<'a, S::Span>)>,
     string_table: FxHashMap, u32>,
     span_data_table: &'span mut S::Table,
     version: u32,
@@ -394,8 +405,8 @@ struct Writer<'a, 'span, S: InternableSpan> {
     text: Vec,
 }
 
-impl<'a, S: InternableSpan> Writer<'a, '_, S> {
-    fn write(&mut self, root: tt::SubtreeView<'a, S>) {
+impl<'a, T: SpanTransformer> Writer<'a, '_, T> {
+    fn write(&mut self, root: tt::SubtreeView<'a, T::Span>) {
         let subtree = root.top_subtree();
         self.enqueue(subtree, root.iter());
         while let Some((idx, subtree)) = self.work.pop_front() {
@@ -403,11 +414,11 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
         }
     }
 
-    fn token_id_of(&mut self, span: S) -> TokenId {
-        S::token_id_of(self.span_data_table, span)
+    fn token_id_of(&mut self, span: T::Span) -> SpanId {
+        T::token_id_of(self.span_data_table, span)
     }
 
-    fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, S>) {
+    fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, T::Span>) {
         let mut first_tt = self.token_tree.len();
         let n_tt = subtree.clone().count(); // FIXME: `count()` walks over the entire iterator.
         self.token_tree.resize(first_tt + n_tt, !0);
@@ -478,7 +489,11 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
         }
     }
 
-    fn enqueue(&mut self, subtree: &'a tt::Subtree, contents: tt::iter::TtIter<'a, S>) -> u32 {
+    fn enqueue(
+        &mut self,
+        subtree: &'a tt::Subtree,
+        contents: tt::iter::TtIter<'a, T::Span>,
+    ) -> u32 {
         let idx = self.subtree.len();
         let open = self.token_id_of(subtree.delimiter.open);
         let close = self.token_id_of(subtree.delimiter.close);
@@ -507,7 +522,7 @@ impl<'a, S: InternableSpan> Writer<'a, '_, S> {
     }
 }
 
-struct Reader<'span, S: InternableSpan> {
+struct Reader<'span, S: SpanTransformer> {
     version: u32,
     subtree: Vec,
     literal: Vec,
@@ -518,11 +533,11 @@ struct Reader<'span, S: InternableSpan> {
     span_data_table: &'span S::Table,
 }
 
-impl Reader<'_, S> {
-    pub(crate) fn read(self) -> tt::TopSubtree {
-        let mut res: Vec, Vec>)>> =
+impl Reader<'_, T> {
+    pub(crate) fn read(self) -> tt::TopSubtree {
+        let mut res: Vec, Vec>)>> =
             vec![None; self.subtree.len()];
-        let read_span = |id| S::span_for_token_id(self.span_data_table, id);
+        let read_span = |id| T::span_for_token_id(self.span_data_table, id);
         for i in (0..self.subtree.len()).rev() {
             let repr = &self.subtree[i];
             let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize];
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
index 516c7418bde8..97919b85b513 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/lib.rs
@@ -5,24 +5,29 @@
 //! is used to provide basic infrastructure for communication between two
 //! processes: Client (RA itself), Server (the external program)
 
-pub mod legacy_protocol {
-    pub mod json;
-    pub mod msg;
-}
+pub mod legacy_protocol;
 mod process;
 
 use paths::{AbsPath, AbsPathBuf};
 use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span};
 use std::{fmt, io, sync::Arc, time::SystemTime};
 
-use crate::{
-    legacy_protocol::msg::{
-        ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, HAS_GLOBAL_SPANS, HASHED_AST_ID,
-        PanicMessage, RUST_ANALYZER_SPAN_SUPPORT, Request, Response, SpanDataIndexMap,
-        deserialize_span_data_index_map, flat::serialize_span_data_index_map,
-    },
-    process::ProcMacroServerProcess,
-};
+use crate::process::ProcMacroServerProcess;
+
+/// The versions of the server protocol
+pub mod version {
+    pub const NO_VERSION_CHECK_VERSION: u32 = 0;
+    pub const VERSION_CHECK_VERSION: u32 = 1;
+    pub const ENCODE_CLOSE_SPAN_VERSION: u32 = 2;
+    pub const HAS_GLOBAL_SPANS: u32 = 3;
+    pub const RUST_ANALYZER_SPAN_SUPPORT: u32 = 4;
+    /// Whether literals encode their kind as an additional u32 field and idents their rawness as a u32 field.
+    pub const EXTENDED_LEAF_DATA: u32 = 5;
+    pub const HASHED_AST_ID: u32 = 6;
+
+    /// Current API version of the proc-macro protocol.
+    pub const CURRENT_API_VERSION: u32 = HASHED_AST_ID;
+}
 
 /// Represents different kinds of procedural macros that can be expanded by the external server.
 #[derive(Copy, Clone, Eq, PartialEq, Debug, serde_derive::Serialize, serde_derive::Deserialize)]
@@ -163,7 +168,7 @@ impl ProcMacro {
 
     fn needs_fixup_change(&self) -> bool {
         let version = self.process.version();
-        (RUST_ANALYZER_SPAN_SUPPORT..HASHED_AST_ID).contains(&version)
+        (version::RUST_ANALYZER_SPAN_SUPPORT..version::HASHED_AST_ID).contains(&version)
     }
 
     /// On some server versions, the fixup ast id is different than ours. So change it to match.
@@ -204,7 +209,7 @@ impl ProcMacro {
         call_site: Span,
         mixed_site: Span,
         current_dir: String,
-    ) -> Result, PanicMessage>, ServerError> {
+    ) -> Result, String>, ServerError> {
         let (mut subtree, mut attr) = (subtree, attr);
         let (mut subtree_changed, mut attr_changed);
         if self.needs_fixup_change() {
@@ -219,57 +224,15 @@ impl ProcMacro {
             }
         }
 
-        let version = self.process.version();
-
-        let mut span_data_table = SpanDataIndexMap::default();
-        let def_site = span_data_table.insert_full(def_site).0;
-        let call_site = span_data_table.insert_full(call_site).0;
-        let mixed_site = span_data_table.insert_full(mixed_site).0;
-        let task = ExpandMacro {
-            data: ExpandMacroData {
-                macro_body: FlatTree::new(subtree, version, &mut span_data_table),
-                macro_name: self.name.to_string(),
-                attributes: attr
-                    .map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)),
-                has_global_spans: ExpnGlobals {
-                    serialize: version >= HAS_GLOBAL_SPANS,
-                    def_site,
-                    call_site,
-                    mixed_site,
-                },
-                span_data_table: if version >= RUST_ANALYZER_SPAN_SUPPORT {
-                    serialize_span_data_index_map(&span_data_table)
-                } else {
-                    Vec::new()
-                },
-            },
-            lib: self.dylib_path.to_path_buf().into(),
+        legacy_protocol::expand(
+            self,
+            subtree,
+            attr,
             env,
-            current_dir: Some(current_dir),
-        };
-
-        let response = self.process.send_task(Request::ExpandMacro(Box::new(task)))?;
-
-        match response {
-            Response::ExpandMacro(it) => Ok(it.map(|tree| {
-                let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table);
-                if self.needs_fixup_change() {
-                    self.change_fixup_to_match_old_server(&mut expanded);
-                }
-                expanded
-            })),
-            Response::ExpandMacroExtended(it) => Ok(it.map(|resp| {
-                let mut expanded = FlatTree::to_subtree_resolved(
-                    resp.tree,
-                    version,
-                    &deserialize_span_data_index_map(&resp.span_data_table),
-                );
-                if self.needs_fixup_change() {
-                    self.change_fixup_to_match_old_server(&mut expanded);
-                }
-                expanded
-            })),
-            _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
-        }
+            def_site,
+            call_site,
+            mixed_site,
+            current_dir,
+        )
     }
 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
index fcea75ef672a..fe274a027a80 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-api/src/process.rs
@@ -12,13 +12,8 @@ use stdx::JodChild;
 
 use crate::{
     ProcMacroKind, ServerError,
-    legacy_protocol::{
-        json::{read_json, write_json},
-        msg::{
-            CURRENT_API_VERSION, Message, RUST_ANALYZER_SPAN_SUPPORT, Request, Response,
-            ServerConfig, SpanMode,
-        },
-    },
+    legacy_protocol::{self, SpanMode},
+    version,
 };
 
 /// Represents a process handling proc-macro communication.
@@ -28,11 +23,16 @@ pub(crate) struct ProcMacroServerProcess {
     /// hence the lock on the state.
     state: Mutex,
     version: u32,
-    mode: SpanMode,
+    protocol: Protocol,
     /// Populated when the server exits.
     exited: OnceLock>,
 }
 
+#[derive(Debug)]
+enum Protocol {
+    LegacyJson { mode: SpanMode },
+}
+
 /// Maintains the state of the proc-macro server process.
 #[derive(Debug)]
 struct ProcessSrvState {
@@ -56,34 +56,40 @@ impl ProcMacroServerProcess {
             io::Result::Ok(ProcMacroServerProcess {
                 state: Mutex::new(ProcessSrvState { process, stdin, stdout }),
                 version: 0,
-                mode: SpanMode::Id,
+                protocol: Protocol::LegacyJson { mode: SpanMode::Id },
                 exited: OnceLock::new(),
             })
         };
         let mut srv = create_srv()?;
         tracing::info!("sending proc-macro server version check");
         match srv.version_check() {
-            Ok(v) if v > CURRENT_API_VERSION => Err(io::Error::other(
-                format!( "The version of the proc-macro server ({v}) in your Rust toolchain is newer than the version supported by your rust-analyzer ({CURRENT_API_VERSION}).
-            This will prevent proc-macro expansion from working. Please consider updating your rust-analyzer to ensure compatibility with your current toolchain."
-                ),
-            )),
+            Ok(v) if v > version::CURRENT_API_VERSION => {
+                #[allow(clippy::disallowed_methods)]
+                let process_version = Command::new(process_path)
+                    .arg("--version")
+                    .output()
+                    .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned())
+                    .unwrap_or_else(|_| "unknown version".to_owned());
+                Err(io::Error::other(format!(
+                    "Your installed proc-macro server is too new for your rust-analyzer. API version: {}, server version: {process_version}. \
+                        This will prevent proc-macro expansion from working. Please consider updating your rust-analyzer to ensure compatibility with your current toolchain.",
+                    version::CURRENT_API_VERSION
+                )))
+            }
             Ok(v) => {
                 tracing::info!("Proc-macro server version: {v}");
                 srv.version = v;
-                if srv.version >= RUST_ANALYZER_SPAN_SUPPORT {
-                    if let Ok(mode) = srv.enable_rust_analyzer_spans() {
-                        srv.mode = mode;
-                    }
+                if srv.version >= version::RUST_ANALYZER_SPAN_SUPPORT
+                    && let Ok(mode) = srv.enable_rust_analyzer_spans()
+                {
+                    srv.protocol = Protocol::LegacyJson { mode };
                 }
-                tracing::info!("Proc-macro server span mode: {:?}", srv.mode);
+                tracing::info!("Proc-macro server protocol: {:?}", srv.protocol);
                 Ok(srv)
             }
             Err(e) => {
                 tracing::info!(%e, "proc-macro version check failed");
-                Err(
-                    io::Error::other(format!("proc-macro server version check failed: {e}")),
-                )
+                Err(io::Error::other(format!("proc-macro server version check failed: {e}")))
             }
         }
     }
@@ -98,25 +104,24 @@ impl ProcMacroServerProcess {
         self.version
     }
 
+    /// Enable support for rust-analyzer span mode if the server supports it.
+    pub(crate) fn rust_analyzer_spans(&self) -> bool {
+        match self.protocol {
+            Protocol::LegacyJson { mode } => mode == SpanMode::RustAnalyzer,
+        }
+    }
+
     /// Checks the API version of the running proc-macro server.
     fn version_check(&self) -> Result {
-        let request = Request::ApiVersionCheck {};
-        let response = self.send_task(request)?;
-
-        match response {
-            Response::ApiVersionCheck(version) => Ok(version),
-            _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+        match self.protocol {
+            Protocol::LegacyJson { .. } => legacy_protocol::version_check(self),
         }
     }
 
     /// Enable support for rust-analyzer span mode if the server supports it.
     fn enable_rust_analyzer_spans(&self) -> Result {
-        let request = Request::SetConfig(ServerConfig { span_mode: SpanMode::RustAnalyzer });
-        let response = self.send_task(request)?;
-
-        match response {
-            Response::SetConfig(ServerConfig { span_mode }) => Ok(span_mode),
-            _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+        match self.protocol {
+            Protocol::LegacyJson { .. } => legacy_protocol::enable_rust_analyzer_spans(self),
         }
     }
 
@@ -125,25 +130,24 @@ impl ProcMacroServerProcess {
         &self,
         dylib_path: &AbsPath,
     ) -> Result, String>, ServerError> {
-        let request = Request::ListMacros { dylib_path: dylib_path.to_path_buf().into() };
-
-        let response = self.send_task(request)?;
-
-        match response {
-            Response::ListMacros(it) => Ok(it),
-            _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }),
+        match self.protocol {
+            Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path),
         }
     }
 
-    /// Sends a request to the proc-macro server and waits for a response.
-    pub(crate) fn send_task(&self, req: Request) -> Result {
-        if let Some(server_error) = self.exited.get() {
-            return Err(server_error.0.clone());
-        }
-
+    pub(crate) fn send_task(
+        &self,
+        serialize_req: impl FnOnce(
+            &mut dyn Write,
+            &mut dyn BufRead,
+            Request,
+            &mut String,
+        ) -> Result, ServerError>,
+        req: Request,
+    ) -> Result {
         let state = &mut *self.state.lock().unwrap();
         let mut buf = String::new();
-        send_request(&mut state.stdin, &mut state.stdout, req, &mut buf)
+        serialize_req(&mut state.stdin, &mut state.stdout, req, &mut buf)
             .and_then(|res| {
                 res.ok_or_else(|| {
                     let message = "proc-macro server did not respond with data".to_owned();
@@ -162,10 +166,10 @@ impl ProcMacroServerProcess {
                         Ok(None) | Err(_) => e,
                         Ok(Some(status)) => {
                             let mut msg = String::new();
-                            if !status.success() {
-                                if let Some(stderr) = state.process.child.stderr.as_mut() {
-                                    _ = stderr.read_to_string(&mut msg);
-                                }
+                            if !status.success()
+                                && let Some(stderr) = state.process.child.stderr.as_mut()
+                            {
+                                _ = stderr.read_to_string(&mut msg);
                             }
                             let server_error = ServerError {
                                 message: format!(
@@ -242,21 +246,3 @@ fn mk_child<'a>(
     }
     cmd.spawn()
 }
-
-/// Sends a request to the server and reads the response.
-fn send_request(
-    mut writer: &mut impl Write,
-    mut reader: &mut impl BufRead,
-    req: Request,
-    buf: &mut String,
-) -> Result, ServerError> {
-    req.write(write_json, &mut writer).map_err(|err| ServerError {
-        message: "failed to write request".into(),
-        io: Some(Arc::new(err)),
-    })?;
-    let res = Response::read(read_json, &mut reader, buf).map_err(|err| ServerError {
-        message: "failed to read response".into(),
-        io: Some(Arc::new(err)),
-    })?;
-    Ok(res)
-}
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml
index ab421021b8bf..91e9e62b084b 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/Cargo.toml
@@ -14,10 +14,14 @@ publish = false
 proc-macro-srv.workspace = true
 proc-macro-api.workspace = true
 tt.workspace = true
+clap = {version = "4.5.42", default-features = false, features = ["std"]}
+postcard = { version = "1.1.3", optional = true }
 
 [features]
+default = ["postcard"]
 sysroot-abi = ["proc-macro-srv/sysroot-abi"]
 in-rust-tree = ["proc-macro-srv/in-rust-tree", "sysroot-abi"]
+postcard = ["dep:postcard"]
 
 
 [[bin]]
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/build.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/build.rs
index 07f914fece0e..12e7c8b05bac 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/build.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/build.rs
@@ -1,5 +1,49 @@
-//! This teaches cargo about our cfg(rust_analyzer)
+//! Construct version in the `commit-hash date channel` format
+
+use std::{env, path::PathBuf, process::Command};
 
 fn main() {
-    println!("cargo:rustc-check-cfg=cfg(rust_analyzer)");
+    set_rerun();
+    set_commit_info();
+    println!("cargo::rustc-check-cfg=cfg(rust_analyzer)");
+}
+
+fn set_rerun() {
+    println!("cargo:rerun-if-env-changed=CFG_RELEASE");
+
+    let mut manifest_dir = PathBuf::from(
+        env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
+    );
+
+    while manifest_dir.parent().is_some() {
+        let head_ref = manifest_dir.join(".git/HEAD");
+        if head_ref.exists() {
+            println!("cargo:rerun-if-changed={}", head_ref.display());
+            return;
+        }
+
+        manifest_dir.pop();
+    }
+
+    println!("cargo:warning=Could not find `.git/HEAD` from manifest dir!");
+}
+
+fn set_commit_info() {
+    #[allow(clippy::disallowed_methods)]
+    let output = match Command::new("git")
+        .arg("log")
+        .arg("-1")
+        .arg("--date=short")
+        .arg("--format=%H %h %cd")
+        .output()
+    {
+        Ok(output) if output.status.success() => output,
+        _ => return,
+    };
+    let stdout = String::from_utf8(output.stdout).unwrap();
+    let mut parts = stdout.split_whitespace();
+    let mut next = || parts.next().unwrap();
+    println!("cargo:rustc-env=RA_COMMIT_HASH={}", next());
+    println!("cargo:rustc-env=RA_COMMIT_SHORT_HASH={}", next());
+    println!("cargo:rustc-env=RA_COMMIT_DATE={}", next())
 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
index c47ed053254b..662d34865eff 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main.rs
@@ -2,13 +2,16 @@
 //! Driver for proc macro server
 #![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
 #![cfg_attr(not(feature = "sysroot-abi"), allow(unused_crate_dependencies))]
-#![allow(clippy::print_stderr)]
+#![allow(clippy::print_stdout, clippy::print_stderr)]
 
 #[cfg(feature = "in-rust-tree")]
 extern crate rustc_driver as _;
 
+mod version;
+
 #[cfg(any(feature = "sysroot-abi", rust_analyzer))]
 mod main_loop;
+use clap::{Command, ValueEnum};
 #[cfg(any(feature = "sysroot-abi", rust_analyzer))]
 use main_loop::run;
 
@@ -23,12 +26,59 @@ fn main() -> std::io::Result<()> {
         );
         std::process::exit(122);
     }
+    let matches = Command::new("proc-macro-srv")
+        .args(&[
+            clap::Arg::new("format")
+                .long("format")
+                .action(clap::ArgAction::Set)
+                .default_value("json")
+                .value_parser(clap::builder::EnumValueParser::::new()),
+            clap::Arg::new("version")
+                .long("version")
+                .action(clap::ArgAction::SetTrue)
+                .help("Prints the version of the proc-macro-srv"),
+        ])
+        .get_matches();
+    if matches.get_flag("version") {
+        println!("rust-analyzer-proc-macro-srv {}", version::version());
+        return Ok(());
+    }
+    let &format =
+        matches.get_one::("format").expect("format value should always be present");
+    run(format)
+}
 
-    run()
+#[derive(Copy, Clone)]
+enum ProtocolFormat {
+    Json,
+    #[cfg(feature = "postcard")]
+    Postcard,
+}
+
+impl ValueEnum for ProtocolFormat {
+    fn value_variants<'a>() -> &'a [Self] {
+        &[ProtocolFormat::Json]
+    }
+
+    fn to_possible_value(&self) -> Option {
+        match self {
+            ProtocolFormat::Json => Some(clap::builder::PossibleValue::new("json")),
+            #[cfg(feature = "postcard")]
+            ProtocolFormat::Postcard => Some(clap::builder::PossibleValue::new("postcard")),
+        }
+    }
+    fn from_str(input: &str, _ignore_case: bool) -> Result {
+        match input {
+            "json" => Ok(ProtocolFormat::Json),
+            #[cfg(feature = "postcard")]
+            "postcard" => Ok(ProtocolFormat::Postcard),
+            _ => Err(format!("unknown protocol format: {input}")),
+        }
+    }
 }
 
 #[cfg(not(any(feature = "sysroot-abi", rust_analyzer)))]
-fn run() -> std::io::Result<()> {
+fn run(_: ProtocolFormat) -> std::io::Result<()> {
     Err(std::io::Error::new(
         std::io::ErrorKind::Unsupported,
         "proc-macro-srv-cli needs to be compiled with the `sysroot-abi` feature to function"
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
index f54dff1f2d82..703bc965db25 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/main_loop.rs
@@ -1,16 +1,48 @@
 //! The main loop of the proc-macro server.
 use std::io;
 
-use proc_macro_api::legacy_protocol::{
-    json::{read_json, write_json},
-    msg::{
-        self, CURRENT_API_VERSION, ExpandMacroData, ExpnGlobals, Message, SpanMode, TokenId,
-        deserialize_span_data_index_map, serialize_span_data_index_map,
+use proc_macro_api::{
+    legacy_protocol::{
+        json::{read_json, write_json},
+        msg::{
+            self, ExpandMacroData, ExpnGlobals, Message, SpanMode, SpanTransformer,
+            deserialize_span_data_index_map, serialize_span_data_index_map,
+        },
     },
+    version::CURRENT_API_VERSION,
 };
-use proc_macro_srv::EnvSnapshot;
+use proc_macro_srv::{EnvSnapshot, SpanId};
 
-pub(crate) fn run() -> io::Result<()> {
+use crate::ProtocolFormat;
+
+struct SpanTrans;
+
+impl SpanTransformer for SpanTrans {
+    type Table = ();
+    type Span = SpanId;
+    fn token_id_of(
+        _: &mut Self::Table,
+        span: Self::Span,
+    ) -> proc_macro_api::legacy_protocol::SpanId {
+        proc_macro_api::legacy_protocol::SpanId(span.0)
+    }
+    fn span_for_token_id(
+        _: &Self::Table,
+        id: proc_macro_api::legacy_protocol::SpanId,
+    ) -> Self::Span {
+        SpanId(id.0)
+    }
+}
+
+pub(crate) fn run(format: ProtocolFormat) -> io::Result<()> {
+    match format {
+        ProtocolFormat::Json => run_json(),
+        #[cfg(feature = "postcard")]
+        ProtocolFormat::Postcard => unimplemented!(),
+    }
+}
+
+fn run_json() -> io::Result<()> {
     fn macro_kind_to_api(kind: proc_macro_srv::ProcMacroKind) -> proc_macro_api::ProcMacroKind {
         match kind {
             proc_macro_srv::ProcMacroKind::CustomDerive => {
@@ -54,19 +86,20 @@ pub(crate) fn run() -> io::Result<()> {
                 } = *task;
                 match span_mode {
                     SpanMode::Id => msg::Response::ExpandMacro({
-                        let def_site = TokenId(def_site as u32);
-                        let call_site = TokenId(call_site as u32);
-                        let mixed_site = TokenId(mixed_site as u32);
+                        let def_site = SpanId(def_site as u32);
+                        let call_site = SpanId(call_site as u32);
+                        let mixed_site = SpanId(mixed_site as u32);
 
-                        let macro_body = macro_body.to_subtree_unresolved(CURRENT_API_VERSION);
-                        let attributes =
-                            attributes.map(|it| it.to_subtree_unresolved(CURRENT_API_VERSION));
+                        let macro_body =
+                            macro_body.to_subtree_unresolved::(CURRENT_API_VERSION);
+                        let attributes = attributes
+                            .map(|it| it.to_subtree_unresolved::(CURRENT_API_VERSION));
 
                         srv.expand(
                             lib,
-                            env,
+                            &env,
                             current_dir,
-                            macro_name,
+                            ¯o_name,
                             macro_body,
                             attributes,
                             def_site,
@@ -74,8 +107,12 @@ pub(crate) fn run() -> io::Result<()> {
                             mixed_site,
                         )
                         .map(|it| {
-                            msg::FlatTree::new_raw(tt::SubtreeView::new(&it), CURRENT_API_VERSION)
+                            msg::FlatTree::new_raw::(
+                                tt::SubtreeView::new(&it),
+                                CURRENT_API_VERSION,
+                            )
                         })
+                        .map_err(|e| e.into_string().unwrap_or_default())
                         .map_err(msg::PanicMessage)
                     }),
                     SpanMode::RustAnalyzer => msg::Response::ExpandMacroExtended({
@@ -92,9 +129,9 @@ pub(crate) fn run() -> io::Result<()> {
                         });
                         srv.expand(
                             lib,
-                            env,
+                            &env,
                             current_dir,
-                            macro_name,
+                            ¯o_name,
                             macro_body,
                             attributes,
                             def_site,
@@ -115,6 +152,7 @@ pub(crate) fn run() -> io::Result<()> {
                             tree,
                             span_data_table,
                         })
+                        .map_err(|e| e.into_string().unwrap_or_default())
                         .map_err(msg::PanicMessage)
                     }),
                 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/version.rs b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/version.rs
new file mode 100644
index 000000000000..32499d055d1e
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv-cli/src/version.rs
@@ -0,0 +1,58 @@
+//! Code for representing rust-analyzer's release version number.
+#![expect(dead_code)]
+
+use std::fmt;
+
+/// Information about the git repository where rust-analyzer was built from.
+pub(crate) struct CommitInfo {
+    pub(crate) short_commit_hash: &'static str,
+    pub(crate) commit_hash: &'static str,
+    pub(crate) commit_date: &'static str,
+}
+
+/// Cargo's version.
+pub(crate) struct VersionInfo {
+    /// rust-analyzer's version, such as "1.57.0", "1.58.0-beta.1", "1.59.0-nightly", etc.
+    pub(crate) version: &'static str,
+    /// The release channel we were built for (stable/beta/nightly/dev).
+    ///
+    /// `None` if not built via bootstrap.
+    pub(crate) release_channel: Option<&'static str>,
+    /// Information about the Git repository we may have been built from.
+    ///
+    /// `None` if not built from a git repo.
+    pub(crate) commit_info: Option,
+}
+
+impl fmt::Display for VersionInfo {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.version)?;
+
+        if let Some(ci) = &self.commit_info {
+            write!(f, " ({} {})", ci.short_commit_hash, ci.commit_date)?;
+        };
+        Ok(())
+    }
+}
+
+/// Returns information about cargo's version.
+pub(crate) const fn version() -> VersionInfo {
+    let version = match option_env!("CFG_RELEASE") {
+        Some(x) => x,
+        None => "0.0.0",
+    };
+
+    let release_channel = option_env!("CFG_RELEASE_CHANNEL");
+    let commit_info = match (
+        option_env!("RA_COMMIT_SHORT_HASH"),
+        option_env!("RA_COMMIT_HASH"),
+        option_env!("RA_COMMIT_DATE"),
+    ) {
+        (Some(short_commit_hash), Some(commit_hash), Some(commit_date)) => {
+            Some(CommitInfo { short_commit_hash, commit_hash, commit_date })
+        }
+        _ => None,
+    };
+
+    VersionInfo { version, release_channel, commit_info }
+}
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml
index 4034f244393b..d037e715e703 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/Cargo.toml
@@ -16,6 +16,7 @@ doctest = false
 object.workspace = true
 libloading.workspace = true
 memmap2.workspace = true
+temp-dir.workspace = true
 
 tt.workspace = true
 syntax-bridge.workspace = true
@@ -26,6 +27,7 @@ intern.workspace = true
 
 ra-ap-rustc_lexer.workspace = true
 
+
 [target.'cfg(unix)'.dependencies]
 libc.workspace = true
 
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs
index c49159df9916..c8513a10675d 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib.rs
@@ -1,65 +1,66 @@
 //! Handles dynamic library loading for proc macro
 
+mod proc_macros;
 mod version;
 
 use proc_macro::bridge;
 use std::{fmt, fs, io, time::SystemTime};
+use temp_dir::TempDir;
 
 use libloading::Library;
 use object::Object;
 use paths::{Utf8Path, Utf8PathBuf};
 
-use crate::{ProcMacroKind, ProcMacroSrvSpan, proc_macros::ProcMacros, server_impl::TopSubtree};
+use crate::{
+    PanicMessage, ProcMacroKind, ProcMacroSrvSpan, dylib::proc_macros::ProcMacros,
+    server_impl::TopSubtree,
+};
 
-/// Loads dynamic library in platform dependent manner.
-///
-/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
-/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
-/// and [here](https://github.com/rust-lang/rust/issues/60593).
-///
-/// Usage of RTLD_DEEPBIND
-/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
-///
-/// It seems that on Windows that behaviour is default, so we do nothing in that case.
-///
-/// # Safety
-///
-/// The caller is responsible for ensuring that the path is valid proc-macro library
-#[cfg(windows)]
-unsafe fn load_library(file: &Utf8Path) -> Result {
-    // SAFETY: The caller is responsible for ensuring that the path is valid proc-macro library
-    unsafe { Library::new(file) }
+pub(crate) struct Expander {
+    inner: ProcMacroLibrary,
+    modified_time: SystemTime,
 }
 
-/// Loads dynamic library in platform dependent manner.
-///
-/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
-/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
-/// and [here](https://github.com/rust-lang/rust/issues/60593).
-///
-/// Usage of RTLD_DEEPBIND
-/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
-///
-/// It seems that on Windows that behaviour is default, so we do nothing in that case.
-///
-/// # Safety
-///
-/// The caller is responsible for ensuring that the path is valid proc-macro library
-#[cfg(unix)]
-unsafe fn load_library(file: &Utf8Path) -> Result {
-    // not defined by POSIX, different values on mips vs other targets
-    #[cfg(target_env = "gnu")]
-    use libc::RTLD_DEEPBIND;
-    use libloading::os::unix::Library as UnixLibrary;
-    // defined by POSIX
-    use libloading::os::unix::RTLD_NOW;
+impl Expander {
+    pub(crate) fn new(
+        temp_dir: &TempDir,
+        lib: &Utf8Path,
+    ) -> Result {
+        // Some libraries for dynamic loading require canonicalized path even when it is
+        // already absolute
+        let lib = lib.canonicalize_utf8()?;
+        let modified_time = fs::metadata(&lib).and_then(|it| it.modified())?;
 
-    // MUSL and bionic don't have it..
-    #[cfg(not(target_env = "gnu"))]
-    const RTLD_DEEPBIND: std::os::raw::c_int = 0x0;
+        let path = ensure_file_with_lock_free_access(temp_dir, &lib)?;
+        let library = ProcMacroLibrary::open(path.as_ref())?;
 
-    // SAFETY: The caller is responsible for ensuring that the path is valid proc-macro library
-    unsafe { UnixLibrary::open(Some(file), RTLD_NOW | RTLD_DEEPBIND).map(|lib| lib.into()) }
+        Ok(Expander { inner: library, modified_time })
+    }
+
+    pub(crate) fn expand(
+        &self,
+        macro_name: &str,
+        macro_body: TopSubtree,
+        attributes: Option>,
+        def_site: S,
+        call_site: S,
+        mixed_site: S,
+    ) -> Result, PanicMessage>
+    where
+        ::TokenStream: Default,
+    {
+        self.inner
+            .proc_macros
+            .expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site)
+    }
+
+    pub(crate) fn list_macros(&self) -> impl Iterator {
+        self.inner.proc_macros.list_macros()
+    }
+
+    pub(crate) fn modified_time(&self) -> SystemTime {
+        self.modified_time
+    }
 }
 
 #[derive(Debug)]
@@ -133,54 +134,6 @@ impl ProcMacroLibrary {
     }
 }
 
-// Drop order matters as we can't remove the dylib before the library is unloaded
-pub(crate) struct Expander {
-    inner: ProcMacroLibrary,
-    _remove_on_drop: RemoveFileOnDrop,
-    modified_time: SystemTime,
-}
-
-impl Expander {
-    pub(crate) fn new(lib: &Utf8Path) -> Result {
-        // Some libraries for dynamic loading require canonicalized path even when it is
-        // already absolute
-        let lib = lib.canonicalize_utf8()?;
-        let modified_time = fs::metadata(&lib).and_then(|it| it.modified())?;
-
-        let path = ensure_file_with_lock_free_access(&lib)?;
-        let library = ProcMacroLibrary::open(path.as_ref())?;
-
-        Ok(Expander { inner: library, _remove_on_drop: RemoveFileOnDrop(path), modified_time })
-    }
-
-    pub(crate) fn expand(
-        &self,
-        macro_name: &str,
-        macro_body: TopSubtree,
-        attributes: Option>,
-        def_site: S,
-        call_site: S,
-        mixed_site: S,
-    ) -> Result, String>
-    where
-        ::TokenStream: Default,
-    {
-        let result = self
-            .inner
-            .proc_macros
-            .expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site);
-        result.map_err(|e| e.into_string().unwrap_or_default())
-    }
-
-    pub(crate) fn list_macros(&self) -> Vec<(String, ProcMacroKind)> {
-        self.inner.proc_macros.list_macros()
-    }
-
-    pub(crate) fn modified_time(&self) -> SystemTime {
-        self.modified_time
-    }
-}
-
 fn invalid_data_err(e: impl Into>) -> io::Error {
     io::Error::new(io::ErrorKind::InvalidData, e)
 }
@@ -210,18 +163,12 @@ fn find_registrar_symbol(obj: &object::File<'_>) -> object::Result io::Result {
+fn ensure_file_with_lock_free_access(
+    temp_dir: &TempDir,
+    path: &Utf8Path,
+) -> io::Result {
     use std::collections::hash_map::RandomState;
     use std::hash::{BuildHasher, Hasher};
 
@@ -229,9 +176,7 @@ fn ensure_file_with_lock_free_access(path: &Utf8Path) -> io::Result
         return Ok(path.to_path_buf());
     }
 
-    let mut to = Utf8PathBuf::from_path_buf(std::env::temp_dir()).unwrap();
-    to.push("rust-analyzer-proc-macros");
-    _ = fs::create_dir(&to);
+    let mut to = Utf8Path::from_path(temp_dir.path()).unwrap().to_owned();
 
     let file_name = path.file_stem().ok_or_else(|| {
         io::Error::new(io::ErrorKind::InvalidInput, format!("File path is invalid: {path}"))
@@ -248,6 +193,60 @@ fn ensure_file_with_lock_free_access(path: &Utf8Path) -> io::Result
 }
 
 #[cfg(unix)]
-fn ensure_file_with_lock_free_access(path: &Utf8Path) -> io::Result {
+fn ensure_file_with_lock_free_access(
+    _temp_dir: &TempDir,
+    path: &Utf8Path,
+) -> io::Result {
     Ok(path.to_owned())
 }
+
+/// Loads dynamic library in platform dependent manner.
+///
+/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
+/// and [here](https://github.com/rust-lang/rust/issues/60593).
+///
+/// Usage of RTLD_DEEPBIND
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
+///
+/// It seems that on Windows that behaviour is default, so we do nothing in that case.
+///
+/// # Safety
+///
+/// The caller is responsible for ensuring that the path is valid proc-macro library
+#[cfg(windows)]
+unsafe fn load_library(file: &Utf8Path) -> Result {
+    // SAFETY: The caller is responsible for ensuring that the path is valid proc-macro library
+    unsafe { Library::new(file) }
+}
+
+/// Loads dynamic library in platform dependent manner.
+///
+/// For unix, you have to use RTLD_DEEPBIND flag to escape problems described
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample)
+/// and [here](https://github.com/rust-lang/rust/issues/60593).
+///
+/// Usage of RTLD_DEEPBIND
+/// [here](https://github.com/fedochet/rust-proc-macro-panic-inside-panic-expample/issues/1)
+///
+/// It seems that on Windows that behaviour is default, so we do nothing in that case.
+///
+/// # Safety
+///
+/// The caller is responsible for ensuring that the path is valid proc-macro library
+#[cfg(unix)]
+unsafe fn load_library(file: &Utf8Path) -> Result {
+    // not defined by POSIX, different values on mips vs other targets
+    #[cfg(target_env = "gnu")]
+    use libc::RTLD_DEEPBIND;
+    use libloading::os::unix::Library as UnixLibrary;
+    // defined by POSIX
+    use libloading::os::unix::RTLD_NOW;
+
+    // MUSL and bionic don't have it..
+    #[cfg(not(target_env = "gnu"))]
+    const RTLD_DEEPBIND: std::os::raw::c_int = 0x0;
+
+    // SAFETY: The caller is responsible for ensuring that the path is valid proc-macro library
+    unsafe { UnixLibrary::open(Some(file), RTLD_NOW | RTLD_DEEPBIND).map(|lib| lib.into()) }
+}
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/proc_macros.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs
similarity index 81%
rename from src/tools/rust-analyzer/crates/proc-macro-srv/src/proc_macros.rs
rename to src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs
index 18532706c4aa..9b5721e370ac 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/proc_macros.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/dylib/proc_macros.rs
@@ -75,20 +75,13 @@ impl ProcMacros {
         Err(bridge::PanicMessage::String(format!("proc-macro `{macro_name}` is missing")).into())
     }
 
-    pub(crate) fn list_macros(&self) -> Vec<(String, ProcMacroKind)> {
-        self.0
-            .iter()
-            .map(|proc_macro| match proc_macro {
-                bridge::client::ProcMacro::CustomDerive { trait_name, .. } => {
-                    (trait_name.to_string(), ProcMacroKind::CustomDerive)
-                }
-                bridge::client::ProcMacro::Bang { name, .. } => {
-                    (name.to_string(), ProcMacroKind::Bang)
-                }
-                bridge::client::ProcMacro::Attr { name, .. } => {
-                    (name.to_string(), ProcMacroKind::Attr)
-                }
-            })
-            .collect()
+    pub(crate) fn list_macros(&self) -> impl Iterator {
+        self.0.iter().map(|proc_macro| match *proc_macro {
+            bridge::client::ProcMacro::CustomDerive { trait_name, .. } => {
+                (trait_name, ProcMacroKind::CustomDerive)
+            }
+            bridge::client::ProcMacro::Bang { name, .. } => (name, ProcMacroKind::Bang),
+            bridge::client::ProcMacro::Attr { name, .. } => (name, ProcMacroKind::Attr),
+        })
     }
 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs
index 223c5a54b703..cb97882c5854 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/lib.rs
@@ -27,7 +27,6 @@ extern crate ra_ap_rustc_lexer as rustc_lexer;
 extern crate rustc_lexer;
 
 mod dylib;
-mod proc_macros;
 mod server_impl;
 
 use std::{
@@ -41,10 +40,13 @@ use std::{
 };
 
 use paths::{Utf8Path, Utf8PathBuf};
-use span::{Span, TokenId};
+use span::Span;
+use temp_dir::TempDir;
 
 use crate::server_impl::TokenStream;
 
+pub use crate::server_impl::token_id::SpanId;
+
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
 pub enum ProcMacroKind {
     CustomDerive,
@@ -57,11 +59,16 @@ pub const RUSTC_VERSION_STRING: &str = env!("RUSTC_VERSION");
 pub struct ProcMacroSrv<'env> {
     expanders: Mutex>>,
     env: &'env EnvSnapshot,
+    temp_dir: TempDir,
 }
 
 impl<'env> ProcMacroSrv<'env> {
     pub fn new(env: &'env EnvSnapshot) -> Self {
-        Self { expanders: Default::default(), env }
+        Self {
+            expanders: Default::default(),
+            env,
+            temp_dir: TempDir::with_prefix("proc-macro-srv").unwrap(),
+        }
     }
 }
 
@@ -71,18 +78,19 @@ impl ProcMacroSrv<'_> {
     pub fn expand(
         &self,
         lib: impl AsRef,
-        env: Vec<(String, String)>,
+        env: &[(String, String)],
         current_dir: Option>,
-        macro_name: String,
+        macro_name: &str,
         macro_body: tt::TopSubtree,
         attribute: Option>,
         def_site: S,
         call_site: S,
         mixed_site: S,
-    ) -> Result>, String> {
+    ) -> Result>, PanicMessage> {
         let snapped_env = self.env;
-        let expander =
-            self.expander(lib.as_ref()).map_err(|err| format!("failed to load macro: {err}"))?;
+        let expander = self.expander(lib.as_ref()).map_err(|err| PanicMessage {
+            message: Some(format!("failed to load macro: {err}")),
+        })?;
 
         let prev_env = EnvChange::apply(snapped_env, env, current_dir.as_ref().map(<_>::as_ref));
 
@@ -91,11 +99,11 @@ impl ProcMacroSrv<'_> {
         let result = thread::scope(|s| {
             let thread = thread::Builder::new()
                 .stack_size(EXPANDER_STACK_SIZE)
-                .name(macro_name.clone())
+                .name(macro_name.to_owned())
                 .spawn_scoped(s, move || {
                     expander
                         .expand(
-                            ¯o_name,
+                            macro_name,
                             server_impl::TopSubtree(macro_body.0.into_vec()),
                             attribute.map(|it| server_impl::TopSubtree(it.0.into_vec())),
                             def_site,
@@ -104,12 +112,7 @@ impl ProcMacroSrv<'_> {
                         )
                         .map(|tt| tt.0)
                 });
-            let res = match thread {
-                Ok(handle) => handle.join(),
-                Err(e) => return Err(e.to_string()),
-            };
-
-            match res {
+            match thread.unwrap().join() {
                 Ok(res) => res,
                 Err(e) => std::panic::resume_unwind(e),
             }
@@ -124,12 +127,12 @@ impl ProcMacroSrv<'_> {
         dylib_path: &Utf8Path,
     ) -> Result, String> {
         let expander = self.expander(dylib_path)?;
-        Ok(expander.list_macros())
+        Ok(expander.list_macros().map(|(k, v)| (k.to_owned(), v)).collect())
     }
 
     fn expander(&self, path: &Utf8Path) -> Result, String> {
         let expander = || {
-            let expander = dylib::Expander::new(path)
+            let expander = dylib::Expander::new(&self.temp_dir, path)
                 .map_err(|err| format!("Cannot create expander for {path}: {err}",));
             expander.map(Arc::new)
         };
@@ -159,8 +162,8 @@ pub trait ProcMacroSrvSpan: Copy + Send {
     fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server;
 }
 
-impl ProcMacroSrvSpan for TokenId {
-    type Server = server_impl::token_id::TokenIdServer;
+impl ProcMacroSrvSpan for SpanId {
+    type Server = server_impl::token_id::SpanIdServer;
 
     fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server {
         Self::Server { call_site, def_site, mixed_site }
@@ -178,6 +181,8 @@ impl ProcMacroSrvSpan for Span {
         }
     }
 }
+
+#[derive(Debug, Clone)]
 pub struct PanicMessage {
     message: Option,
 }
@@ -201,7 +206,7 @@ impl Default for EnvSnapshot {
 static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
 
 struct EnvChange<'snap> {
-    changed_vars: Vec,
+    changed_vars: Vec<&'snap str>,
     prev_working_dir: Option,
     snap: &'snap EnvSnapshot,
     _guard: std::sync::MutexGuard<'snap, ()>,
@@ -210,7 +215,7 @@ struct EnvChange<'snap> {
 impl<'snap> EnvChange<'snap> {
     fn apply(
         snap: &'snap EnvSnapshot,
-        new_vars: Vec<(String, String)>,
+        new_vars: &'snap [(String, String)],
         current_dir: Option<&Path>,
     ) -> EnvChange<'snap> {
         let guard = ENV_LOCK.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
@@ -230,11 +235,11 @@ impl<'snap> EnvChange<'snap> {
         EnvChange {
             snap,
             changed_vars: new_vars
-                .into_iter()
+                .iter()
                 .map(|(k, v)| {
                     // SAFETY: We have acquired the environment lock
-                    unsafe { env::set_var(&k, v) };
-                    k
+                    unsafe { env::set_var(k, v) };
+                    &**k
                 })
                 .collect(),
             prev_working_dir,
@@ -257,14 +262,14 @@ impl Drop for EnvChange<'_> {
             }
         }
 
-        if let Some(dir) = &self.prev_working_dir {
-            if let Err(err) = std::env::set_current_dir(dir) {
-                eprintln!(
-                    "Failed to set the current working dir to {}. Error: {:?}",
-                    dir.display(),
-                    err
-                )
-            }
+        if let Some(dir) = &self.prev_working_dir
+            && let Err(err) = std::env::set_current_dir(dir)
+        {
+            eprintln!(
+                "Failed to set the current working dir to {}. Error: {:?}",
+                dir.display(),
+                err
+            )
         }
     }
 }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
index 662f6257642f..32ad32731ba6 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl.rs
@@ -209,7 +209,7 @@ pub(super) fn from_token_tree(
                 token_trees.push(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
                     spacing: tt::Spacing::Alone,
                     span: literal.span,
-                    char: '-' as char,
+                    char: '-',
                 })));
                 symbol = Symbol::intern(&symbol.as_str()[1..]);
             }
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs
index b493b325e830..91e70ea243ae 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs
@@ -1,4 +1,4 @@
-//! proc-macro server backend based on [`proc_macro_api::msg::TokenId`] as the backing span.
+//! proc-macro server backend based on [`proc_macro_api::msg::SpanId`] as the backing span.
 //! This backend is rather inflexible, used by RustRover and older rust-analyzer versions.
 use std::ops::{Bound, Range};
 
@@ -7,25 +7,34 @@ use proc_macro::bridge::{self, server};
 
 use crate::server_impl::{from_token_tree, literal_from_str, token_stream::TokenStreamBuilder};
 
-type Span = span::TokenId;
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SpanId(pub u32);
+
+impl std::fmt::Debug for SpanId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+type Span = SpanId;
 type TokenStream = crate::server_impl::TokenStream;
 
 pub struct FreeFunctions;
 
-pub struct TokenIdServer {
+pub struct SpanIdServer {
     pub call_site: Span,
     pub def_site: Span,
     pub mixed_site: Span,
 }
 
-impl server::Types for TokenIdServer {
+impl server::Types for SpanIdServer {
     type FreeFunctions = FreeFunctions;
     type TokenStream = TokenStream;
     type Span = Span;
     type Symbol = Symbol;
 }
 
-impl server::FreeFunctions for TokenIdServer {
+impl server::FreeFunctions for SpanIdServer {
     fn injected_env_var(&mut self, _: &str) -> Option {
         None
     }
@@ -41,7 +50,7 @@ impl server::FreeFunctions for TokenIdServer {
     fn emit_diagnostic(&mut self, _: bridge::Diagnostic) {}
 }
 
-impl server::TokenStream for TokenIdServer {
+impl server::TokenStream for SpanIdServer {
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
@@ -102,12 +111,12 @@ impl server::TokenStream for TokenIdServer {
         &mut self,
         stream: Self::TokenStream,
     ) -> Vec> {
-        // Can't join with `TokenId`.
+        // Can't join with `SpanId`.
         stream.into_bridge(&mut |first, _second| first)
     }
 }
 
-impl server::Span for TokenIdServer {
+impl server::Span for SpanIdServer {
     fn debug(&mut self, span: Self::Span) -> String {
         format!("{:?}", span.0)
     }
@@ -174,14 +183,14 @@ impl server::Span for TokenIdServer {
     }
 }
 
-impl server::Symbol for TokenIdServer {
+impl server::Symbol for SpanIdServer {
     fn normalize_and_validate_ident(&mut self, string: &str) -> Result {
         // FIXME: nfc-normalize and validate idents
         Ok(::intern_symbol(string))
     }
 }
 
-impl server::Server for TokenIdServer {
+impl server::Server for SpanIdServer {
     fn globals(&mut self) -> bridge::ExpnGlobals {
         bridge::ExpnGlobals {
             def_site: self.def_site,
@@ -201,8 +210,6 @@ impl server::Server for TokenIdServer {
 
 #[cfg(test)]
 mod tests {
-    use span::TokenId;
-
     use super::*;
 
     #[test]
@@ -211,18 +218,18 @@ mod tests {
             token_trees: vec![
                 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                     sym: Symbol::intern("struct"),
-                    span: TokenId(0),
+                    span: SpanId(0),
                     is_raw: tt::IdentIsRaw::No,
                 })),
                 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                     sym: Symbol::intern("T"),
-                    span: TokenId(0),
+                    span: SpanId(0),
                     is_raw: tt::IdentIsRaw::No,
                 })),
                 tt::TokenTree::Subtree(tt::Subtree {
                     delimiter: tt::Delimiter {
-                        open: TokenId(0),
-                        close: TokenId(0),
+                        open: SpanId(0),
+                        close: SpanId(0),
                         kind: tt::DelimiterKind::Brace,
                     },
                     len: 0,
@@ -238,8 +245,8 @@ mod tests {
         let subtree_paren_a = vec![
             tt::TokenTree::Subtree(tt::Subtree {
                 delimiter: tt::Delimiter {
-                    open: TokenId(0),
-                    close: TokenId(0),
+                    open: SpanId(0),
+                    close: SpanId(0),
                     kind: tt::DelimiterKind::Parenthesis,
                 },
                 len: 1,
@@ -247,24 +254,24 @@ mod tests {
             tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                 is_raw: tt::IdentIsRaw::No,
                 sym: Symbol::intern("a"),
-                span: TokenId(0),
+                span: SpanId(0),
             })),
         ];
 
-        let t1 = TokenStream::from_str("(a)", TokenId(0)).unwrap();
+        let t1 = TokenStream::from_str("(a)", SpanId(0)).unwrap();
         assert_eq!(t1.token_trees.len(), 2);
         assert!(t1.token_trees[0..2] == subtree_paren_a);
 
-        let t2 = TokenStream::from_str("(a);", TokenId(0)).unwrap();
+        let t2 = TokenStream::from_str("(a);", SpanId(0)).unwrap();
         assert_eq!(t2.token_trees.len(), 3);
         assert!(t2.token_trees[0..2] == subtree_paren_a);
 
-        let underscore = TokenStream::from_str("_", TokenId(0)).unwrap();
+        let underscore = TokenStream::from_str("_", SpanId(0)).unwrap();
         assert!(
             underscore.token_trees[0]
                 == tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                     sym: Symbol::intern("_"),
-                    span: TokenId(0),
+                    span: SpanId(0),
                     is_raw: tt::IdentIsRaw::No,
                 }))
         );
diff --git a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs
index 10af5662b5c0..f5a76e30bbcb 100644
--- a/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs
+++ b/src/tools/rust-analyzer/crates/proc-macro-srv/src/tests/utils.rs
@@ -1,14 +1,12 @@
 //! utils used in proc-macro tests
 
 use expect_test::Expect;
-use span::{
-    EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TokenId,
-};
+use span::{EditionedFileId, FileId, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext};
 use tt::TextRange;
 
-use crate::{EnvSnapshot, ProcMacroSrv, dylib, proc_macro_test_dylib_path};
+use crate::{EnvSnapshot, ProcMacroSrv, SpanId, dylib, proc_macro_test_dylib_path};
 
-fn parse_string(call_site: TokenId, src: &str) -> crate::server_impl::TokenStream {
+fn parse_string(call_site: SpanId, src: &str) -> crate::server_impl::TokenStream {
     crate::server_impl::TokenStream::with_subtree(crate::server_impl::TopSubtree(
         syntax_bridge::parse_to_token_tree_static_span(span::Edition::CURRENT, call_site, src)
             .unwrap()
@@ -57,11 +55,11 @@ fn assert_expand_impl(
     expect_spanned: Expect,
 ) {
     let path = proc_macro_test_dylib_path();
-    let expander = dylib::Expander::new(&path).unwrap();
+    let expander = dylib::Expander::new(&temp_dir::TempDir::new().unwrap(), &path).unwrap();
 
-    let def_site = TokenId(0);
-    let call_site = TokenId(1);
-    let mixed_site = TokenId(2);
+    let def_site = SpanId(0);
+    let call_site = SpanId(1);
+    let mixed_site = SpanId(2);
     let input_ts = parse_string(call_site, input).into_subtree(call_site);
     let attr_ts = attr.map(|attr| parse_string(call_site, attr).into_subtree(call_site));
     let input_ts_string = format!("{input_ts:?}");
diff --git a/src/tools/rust-analyzer/crates/profile/src/stop_watch.rs b/src/tools/rust-analyzer/crates/profile/src/stop_watch.rs
index 9f3e636ef816..00c37c01d25e 100644
--- a/src/tools/rust-analyzer/crates/profile/src/stop_watch.rs
+++ b/src/tools/rust-analyzer/crates/profile/src/stop_watch.rs
@@ -37,10 +37,10 @@ impl StopWatch {
                     .build()
                     .map_err(|err| eprintln!("Failed to create perf counter: {err}"))
                     .ok();
-                if let Some(counter) = &mut counter {
-                    if let Err(err) = counter.enable() {
-                        eprintln!("Failed to start perf counter: {err}")
-                    }
+                if let Some(counter) = &mut counter
+                    && let Err(err) = counter.enable()
+                {
+                    eprintln!("Failed to start perf counter: {err}")
                 }
                 counter
             } else {
diff --git a/src/tools/rust-analyzer/crates/project-model/src/build_dependencies.rs b/src/tools/rust-analyzer/crates/project-model/src/build_dependencies.rs
index 5bea74bed7ed..3a682d5a4d83 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/build_dependencies.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/build_dependencies.rs
@@ -9,7 +9,7 @@
 use std::{cell::RefCell, io, mem, process::Command};
 
 use base_db::Env;
-use cargo_metadata::{Message, camino::Utf8Path};
+use cargo_metadata::{Message, PackageId, camino::Utf8Path};
 use cfg::CfgAtom;
 use itertools::Itertools;
 use la_arena::ArenaMap;
@@ -18,6 +18,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
 use serde::Deserialize as _;
 use stdx::never;
 use toolchain::Tool;
+use triomphe::Arc;
 
 use crate::{
     CargoConfig, CargoFeatures, CargoWorkspace, InvocationStrategy, ManifestPath, Package, Sysroot,
@@ -141,7 +142,7 @@ impl WorkspaceBuildScripts {
                 if let Some(&(package, workspace)) = by_id.get(package) {
                     cb(&workspaces[workspace][package].name, &mut res[workspace].outputs[package]);
                 } else {
-                    never!("Received compiler message for unknown package: {}", package);
+                    tracing::error!("Received compiler message for unknown package: {}", package);
                 }
             },
             progress,
@@ -284,7 +285,7 @@ impl WorkspaceBuildScripts {
         // NB: Cargo.toml could have been modified between `cargo metadata` and
         // `cargo check`. We shouldn't assume that package ids we see here are
         // exactly those from `config`.
-        let mut by_id: FxHashMap = FxHashMap::default();
+        let mut by_id: FxHashMap, Package> = FxHashMap::default();
         for package in workspace.packages() {
             outputs.insert(package, BuildScriptOutput::default());
             by_id.insert(workspace[package].id.clone(), package);
@@ -323,7 +324,7 @@ impl WorkspaceBuildScripts {
         // ideally this would be something like:
         // with_output_for: impl FnMut(&str, dyn FnOnce(&mut BuildScriptOutput)),
         // but owned trait objects aren't a thing
-        mut with_output_for: impl FnMut(&str, &mut dyn FnMut(&str, &mut BuildScriptOutput)),
+        mut with_output_for: impl FnMut(&PackageId, &mut dyn FnMut(&str, &mut BuildScriptOutput)),
         progress: &dyn Fn(String),
     ) -> io::Result> {
         let errors = RefCell::new(String::new());
@@ -346,10 +347,8 @@ impl WorkspaceBuildScripts {
 
                 match message {
                     Message::BuildScriptExecuted(mut message) => {
-                        with_output_for(&message.package_id.repr, &mut |name, data| {
-                            progress(format!(
-                                "building compile-time-deps: build script {name} run"
-                            ));
+                        with_output_for(&message.package_id, &mut |name, data| {
+                            progress(format!("build script {name} run"));
                             let cfgs = {
                                 let mut acc = Vec::new();
                                 for cfg in &message.cfgs {
@@ -379,10 +378,8 @@ impl WorkspaceBuildScripts {
                         });
                     }
                     Message::CompilerArtifact(message) => {
-                        with_output_for(&message.package_id.repr, &mut |name, data| {
-                            progress(format!(
-                                "building compile-time-deps: proc-macro {name} built"
-                            ));
+                        with_output_for(&message.package_id, &mut |name, data| {
+                            progress(format!("proc-macro {name} built"));
                             if data.proc_macro_dylib_path == ProcMacroDylibPath::NotBuilt {
                                 data.proc_macro_dylib_path = ProcMacroDylibPath::NotProcMacro;
                             }
diff --git a/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs b/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs
index e613fd590c70..76ba01f3a263 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/cargo_workspace.rs
@@ -5,7 +5,7 @@ use std::str::from_utf8;
 
 use anyhow::Context;
 use base_db::Env;
-use cargo_metadata::{CargoOpt, MetadataCommand};
+use cargo_metadata::{CargoOpt, MetadataCommand, PackageId};
 use la_arena::{Arena, Idx};
 use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf};
 use rustc_hash::{FxHashMap, FxHashSet};
@@ -13,11 +13,12 @@ use serde_derive::Deserialize;
 use serde_json::from_value;
 use span::Edition;
 use stdx::process::spawn_with_streaming_output;
-use toolchain::Tool;
+use toolchain::{NO_RUSTUP_AUTO_INSTALL_ENV, Tool};
+use triomphe::Arc;
 
-use crate::cargo_config_file::make_lockfile_copy;
-use crate::{CfgOverrides, InvocationStrategy};
-use crate::{ManifestPath, Sysroot};
+use crate::{
+    CfgOverrides, InvocationStrategy, ManifestPath, Sysroot, cargo_config_file::make_lockfile_copy,
+};
 
 pub(crate) const MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH: semver::Version =
     semver::Version {
@@ -48,8 +49,9 @@ pub struct CargoWorkspace {
     is_virtual_workspace: bool,
     /// Whether this workspace represents the sysroot workspace.
     is_sysroot: bool,
-    /// Environment variables set in the `.cargo/config` file.
-    config_env: Env,
+    /// Environment variables set in the `.cargo/config` file and the extraEnv
+    /// configuration option.
+    env: Env,
     requires_rustc_private: bool,
 }
 
@@ -155,8 +157,8 @@ pub struct PackageData {
     pub features: FxHashMap>,
     /// List of features enabled on this package
     pub active_features: Vec,
-    /// String representation of package id
-    pub id: String,
+    /// Package id
+    pub id: Arc,
     /// Authors as given in the `Cargo.toml`
     pub authors: Vec,
     /// Description as given in the `Cargo.toml`
@@ -173,6 +175,10 @@ pub struct PackageData {
     pub rust_version: Option,
     /// The contents of [package.metadata.rust-analyzer]
     pub metadata: RustAnalyzerPackageMetaData,
+    /// If this package is a member of the workspace, store all direct and transitive
+    /// dependencies as long as they are workspace members, to track dependency relationships
+    /// between members.
+    pub all_member_deps: Option>,
 }
 
 #[derive(Deserialize, Default, Debug, Clone, Eq, PartialEq)]
@@ -320,7 +326,7 @@ impl CargoWorkspace {
     pub fn new(
         mut meta: cargo_metadata::Metadata,
         ws_manifest_path: ManifestPath,
-        cargo_config_env: Env,
+        cargo_env: Env,
         is_sysroot: bool,
     ) -> CargoWorkspace {
         let mut pkg_by_id = FxHashMap::default();
@@ -334,6 +340,8 @@ impl CargoWorkspace {
         let mut is_virtual_workspace = true;
         let mut requires_rustc_private = false;
 
+        let mut members = FxHashSet::default();
+
         meta.packages.sort_by(|a, b| a.id.cmp(&b.id));
         for meta_pkg in meta.packages {
             let cargo_metadata::Package {
@@ -356,6 +364,7 @@ impl CargoWorkspace {
                 rust_version,
                 ..
             } = meta_pkg;
+            let id = Arc::new(id);
             let meta = from_value::(metadata).unwrap_or_default();
             let edition = match edition {
                 cargo_metadata::Edition::E2015 => Edition::Edition2015,
@@ -375,7 +384,7 @@ impl CargoWorkspace {
             let manifest = ManifestPath::try_from(AbsPathBuf::assert(manifest_path)).unwrap();
             is_virtual_workspace &= manifest != ws_manifest_path;
             let pkg = packages.alloc(PackageData {
-                id: id.repr.clone(),
+                id: id.clone(),
                 name: name.to_string(),
                 version,
                 manifest: manifest.clone(),
@@ -395,7 +404,11 @@ impl CargoWorkspace {
                 features: features.into_iter().collect(),
                 active_features: Vec::new(),
                 metadata: meta.rust_analyzer.unwrap_or_default(),
+                all_member_deps: None,
             });
+            if is_member {
+                members.insert(pkg);
+            }
             let pkg_data = &mut packages[pkg];
             requires_rustc_private |= pkg_data.metadata.rustc_private;
             pkg_by_id.insert(id, pkg);
@@ -440,6 +453,43 @@ impl CargoWorkspace {
                 .extend(node.features.into_iter().map(|it| it.to_string()));
         }
 
+        fn saturate_all_member_deps(
+            packages: &mut Arena,
+            to_visit: Package,
+            visited: &mut FxHashSet,
+            members: &FxHashSet,
+        ) {
+            let pkg_data = &mut packages[to_visit];
+
+            if !visited.insert(to_visit) {
+                return;
+            }
+
+            let deps: Vec<_> = pkg_data
+                .dependencies
+                .iter()
+                .filter_map(|dep| {
+                    let pkg = dep.pkg;
+                    if members.contains(&pkg) { Some(pkg) } else { None }
+                })
+                .collect();
+
+            let mut all_member_deps = FxHashSet::from_iter(deps.iter().copied());
+            for dep in deps {
+                saturate_all_member_deps(packages, dep, visited, members);
+                if let Some(transitives) = &packages[dep].all_member_deps {
+                    all_member_deps.extend(transitives);
+                }
+            }
+
+            packages[to_visit].all_member_deps = Some(all_member_deps);
+        }
+
+        let mut visited = FxHashSet::default();
+        for member in members.iter() {
+            saturate_all_member_deps(&mut packages, *member, &mut visited, &members);
+        }
+
         CargoWorkspace {
             packages,
             targets,
@@ -449,7 +499,7 @@ impl CargoWorkspace {
             is_virtual_workspace,
             requires_rustc_private,
             is_sysroot,
-            config_env: cargo_config_env,
+            env: cargo_env,
         }
     }
 
@@ -540,7 +590,7 @@ impl CargoWorkspace {
     }
 
     pub fn env(&self) -> &Env {
-        &self.config_env
+        &self.env
     }
 
     pub fn is_sysroot(&self) -> bool {
@@ -583,6 +633,7 @@ impl FetchMetadata {
     ) -> Self {
         let cargo = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
         let mut command = MetadataCommand::new();
+        command.env(NO_RUSTUP_AUTO_INSTALL_ENV.0, NO_RUSTUP_AUTO_INSTALL_ENV.1);
         command.cargo_path(cargo.get_program());
         cargo.get_envs().for_each(|(var, val)| _ = command.env(var, val.unwrap_or_default()));
         command.manifest_path(cargo_toml.to_path_buf());
diff --git a/src/tools/rust-analyzer/crates/project-model/src/env.rs b/src/tools/rust-analyzer/crates/project-model/src/env.rs
index d281492fc98c..ae0458af7aa7 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/env.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/env.rs
@@ -1,6 +1,7 @@
 //! Cargo-like environment variables injection.
 use base_db::Env;
 use paths::Utf8Path;
+use rustc_hash::FxHashMap;
 use toolchain::Tool;
 
 use crate::{ManifestPath, PackageData, TargetKind, cargo_config_file::CargoConfigFile};
@@ -60,8 +61,14 @@ pub(crate) fn inject_rustc_tool_env(env: &mut Env, cargo_name: &str, kind: Targe
     env.set("CARGO_CRATE_NAME", cargo_name.replace('-', "_"));
 }
 
-pub(crate) fn cargo_config_env(manifest: &ManifestPath, config: &Option) -> Env {
+pub(crate) fn cargo_config_env(
+    manifest: &ManifestPath,
+    config: &Option,
+    extra_env: &FxHashMap>,
+) -> Env {
     let mut env = Env::default();
+    env.extend(extra_env.iter().filter_map(|(k, v)| v.as_ref().map(|v| (k.clone(), v.clone()))));
+
     let Some(serde_json::Value::Object(env_json)) = config.as_ref().and_then(|c| c.get("env"))
     else {
         return env;
@@ -72,22 +79,34 @@ pub(crate) fn cargo_config_env(manifest: &ManifestPath, config: &Option>::as_ref(manifest.parent());
 
     for (key, entry) in env_json {
-        let serde_json::Value::Object(entry) = entry else {
-            continue;
-        };
-        let Some(value) = entry.get("value").and_then(|v| v.as_str()) else {
-            continue;
+        let value = match entry {
+            serde_json::Value::String(s) => s.clone(),
+            serde_json::Value::Object(entry) => {
+                // Each entry MUST have a `value` key.
+                let Some(value) = entry.get("value").and_then(|v| v.as_str()) else {
+                    continue;
+                };
+                // If the entry already exists in the environment AND the `force` key is not set to
+                // true, then don't overwrite the value.
+                if extra_env.get(key).is_some_and(Option::is_some)
+                    && !entry.get("force").and_then(|v| v.as_bool()).unwrap_or(false)
+                {
+                    continue;
+                }
+
+                if entry
+                    .get("relative")
+                    .and_then(|v| v.as_bool())
+                    .is_some_and(std::convert::identity)
+                {
+                    base.join(value).to_string()
+                } else {
+                    value.to_owned()
+                }
+            }
+            _ => continue,
         };
 
-        let value = if entry
-            .get("relative")
-            .and_then(|v| v.as_bool())
-            .is_some_and(std::convert::identity)
-        {
-            base.join(value).to_string()
-        } else {
-            value.to_owned()
-        };
         env.insert(key, value);
     }
 
@@ -113,7 +132,19 @@ fn parse_output_cargo_config_env_works() {
     },
     "TEST": {
       "value": "test"
-    }
+    },
+    "FORCED": {
+      "value": "test",
+      "force": true
+    },
+    "UNFORCED": {
+      "value": "test",
+      "force": false
+    },
+    "OVERWRITTEN": {
+      "value": "test"
+    },
+    "NOT_AN_OBJECT": "value"
   }
 }
 "#;
@@ -121,9 +152,22 @@ fn parse_output_cargo_config_env_works() {
     let cwd = paths::Utf8PathBuf::try_from(std::env::current_dir().unwrap()).unwrap();
     let manifest = paths::AbsPathBuf::assert(cwd.join("Cargo.toml"));
     let manifest = ManifestPath::try_from(manifest).unwrap();
-    let env = cargo_config_env(&manifest, &Some(config));
+    let extra_env = [
+        ("FORCED", Some("ignored")),
+        ("UNFORCED", Some("newvalue")),
+        ("OVERWRITTEN", Some("newvalue")),
+        ("TEST", None),
+    ]
+    .iter()
+    .map(|(k, v)| (k.to_string(), v.map(ToString::to_string)))
+    .collect();
+    let env = cargo_config_env(&manifest, &Some(config), &extra_env);
     assert_eq!(env.get("CARGO_WORKSPACE_DIR").as_deref(), Some(cwd.join("").as_str()));
     assert_eq!(env.get("RELATIVE").as_deref(), Some(cwd.join("../relative").as_str()));
     assert_eq!(env.get("INVALID").as_deref(), Some("../relative"));
     assert_eq!(env.get("TEST").as_deref(), Some("test"));
+    assert_eq!(env.get("FORCED").as_deref(), Some("test"));
+    assert_eq!(env.get("UNFORCED").as_deref(), Some("newvalue"));
+    assert_eq!(env.get("OVERWRITTEN").as_deref(), Some("newvalue"));
+    assert_eq!(env.get("NOT_AN_OBJECT").as_deref(), Some("value"));
 }
diff --git a/src/tools/rust-analyzer/crates/project-model/src/lib.rs b/src/tools/rust-analyzer/crates/project-model/src/lib.rs
index d39781b15066..e36b90488151 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/lib.rs
@@ -18,7 +18,7 @@
 pub mod project_json;
 pub mod toolchain_info {
     pub mod rustc_cfg;
-    pub mod target_data_layout;
+    pub mod target_data;
     pub mod target_tuple;
     pub mod version;
 
diff --git a/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs b/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
index c0a5009afba3..5cc399bfe76d 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/sysroot.rs
@@ -30,7 +30,7 @@ pub struct Sysroot {
 
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum RustLibSrcWorkspace {
-    Workspace(CargoWorkspace),
+    Workspace { ws: CargoWorkspace, metadata_err: Option },
     Json(ProjectJson),
     Stitched(stitched::Stitched),
     Empty,
@@ -39,7 +39,9 @@ pub enum RustLibSrcWorkspace {
 impl fmt::Display for RustLibSrcWorkspace {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            RustLibSrcWorkspace::Workspace(ws) => write!(f, "workspace {}", ws.workspace_root()),
+            RustLibSrcWorkspace::Workspace { ws, .. } => {
+                write!(f, "workspace {}", ws.workspace_root())
+            }
             RustLibSrcWorkspace::Json(json) => write!(f, "json {}", json.manifest_or_root()),
             RustLibSrcWorkspace::Stitched(stitched) => {
                 write!(f, "stitched with {} crates", stitched.crates.len())
@@ -74,7 +76,7 @@ impl Sysroot {
 
     pub fn is_rust_lib_src_empty(&self) -> bool {
         match &self.workspace {
-            RustLibSrcWorkspace::Workspace(ws) => ws.packages().next().is_none(),
+            RustLibSrcWorkspace::Workspace { ws, .. } => ws.packages().next().is_none(),
             RustLibSrcWorkspace::Json(project_json) => project_json.n_crates() == 0,
             RustLibSrcWorkspace::Stitched(stitched) => stitched.crates.is_empty(),
             RustLibSrcWorkspace::Empty => true,
@@ -85,9 +87,16 @@ impl Sysroot {
         self.error.as_deref()
     }
 
+    pub fn metadata_error(&self) -> Option<&str> {
+        match &self.workspace {
+            RustLibSrcWorkspace::Workspace { metadata_err, .. } => metadata_err.as_deref(),
+            _ => None,
+        }
+    }
+
     pub fn num_packages(&self) -> usize {
         match &self.workspace {
-            RustLibSrcWorkspace::Workspace(ws) => ws.packages().count(),
+            RustLibSrcWorkspace::Workspace { ws, .. } => ws.packages().count(),
             RustLibSrcWorkspace::Json(project_json) => project_json.n_crates(),
             RustLibSrcWorkspace::Stitched(stitched) => stitched.crates.len(),
             RustLibSrcWorkspace::Empty => 0,
@@ -210,7 +219,6 @@ impl Sysroot {
         &self,
         sysroot_source_config: &RustSourceWorkspaceConfig,
         no_deps: bool,
-        current_dir: &AbsPath,
         target_dir: &Utf8Path,
         progress: &dyn Fn(String),
     ) -> Option {
@@ -224,7 +232,7 @@ impl Sysroot {
             if fs::metadata(&library_manifest).is_ok() {
                 match self.load_library_via_cargo(
                     &library_manifest,
-                    current_dir,
+                    src_root,
                     target_dir,
                     cargo_config,
                     no_deps,
@@ -294,7 +302,9 @@ impl Sysroot {
             && let Some(src_root) = &self.rust_lib_src_root
         {
             let has_core = match &self.workspace {
-                RustLibSrcWorkspace::Workspace(ws) => ws.packages().any(|p| ws[p].name == "core"),
+                RustLibSrcWorkspace::Workspace { ws: workspace, .. } => {
+                    workspace.packages().any(|p| workspace[p].name == "core")
+                }
                 RustLibSrcWorkspace::Json(project_json) => project_json
                     .crates()
                     .filter_map(|(_, krate)| krate.display_name.clone())
@@ -333,7 +343,7 @@ impl Sysroot {
 
         // Make sure we never attempt to write to the sysroot
         let locked = true;
-        let (mut res, _) =
+        let (mut res, err) =
             FetchMetadata::new(library_manifest, current_dir, &cargo_config, self, no_deps)
                 .exec(target_dir, locked, progress)?;
 
@@ -388,7 +398,10 @@ impl Sysroot {
 
         let cargo_workspace =
             CargoWorkspace::new(res, library_manifest.clone(), Default::default(), true);
-        Ok(RustLibSrcWorkspace::Workspace(cargo_workspace))
+        Ok(RustLibSrcWorkspace::Workspace {
+            ws: cargo_workspace,
+            metadata_err: err.map(|e| format!("{e:#}")),
+        })
     }
 }
 
diff --git a/src/tools/rust-analyzer/crates/project-model/src/tests.rs b/src/tools/rust-analyzer/crates/project-model/src/tests.rs
index ed72520f40d4..711cdd11b9a8 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/tests.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/tests.rs
@@ -9,7 +9,6 @@ use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf};
 use rustc_hash::FxHashMap;
 use serde::de::DeserializeOwned;
 use span::FileId;
-use triomphe::Arc;
 
 use crate::{
     CargoWorkspace, CfgOverrides, ManifestPath, ProjectJson, ProjectJsonData, ProjectWorkspace,
@@ -47,7 +46,7 @@ fn load_workspace_from_metadata(file: &str) -> ProjectWorkspace {
         sysroot: Sysroot::empty(),
         rustc_cfg: Vec::new(),
         toolchain: None,
-        target_layout: Err("target_data_layout not loaded".into()),
+        target: Err("target_data_layout not loaded".into()),
         extra_includes: Vec::new(),
         set_test: true,
     }
@@ -62,7 +61,7 @@ fn load_rust_project(file: &str) -> (CrateGraphBuilder, ProcMacroPaths) {
         sysroot,
         rustc_cfg: Vec::new(),
         toolchain: None,
-        target_layout: Err(Arc::from("test has no data layout")),
+        target: Err("test has no target data".into()),
         cfg_overrides: Default::default(),
         extra_includes: Vec::new(),
         set_test: true,
@@ -242,7 +241,6 @@ fn smoke_test_real_sysroot_cargo() {
     let loaded_sysroot = sysroot.load_workspace(
         &RustSourceWorkspaceConfig::default_cargo(),
         false,
-        &cwd,
         &Utf8PathBuf::default(),
         &|_| (),
     );
@@ -250,7 +248,7 @@ fn smoke_test_real_sysroot_cargo() {
         sysroot.set_workspace(loaded_sysroot);
     }
     assert!(
-        matches!(sysroot.workspace(), RustLibSrcWorkspace::Workspace(_)),
+        matches!(sysroot.workspace(), RustLibSrcWorkspace::Workspace { .. }),
         "got {}",
         sysroot.workspace()
     );
@@ -265,7 +263,7 @@ fn smoke_test_real_sysroot_cargo() {
         rustc_cfg: Vec::new(),
         cfg_overrides: Default::default(),
         toolchain: None,
-        target_layout: Err("target_data_layout not loaded".into()),
+        target: Err("target_data_layout not loaded".into()),
         extra_includes: Vec::new(),
         set_test: true,
     };
diff --git a/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data.rs b/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data.rs
new file mode 100644
index 000000000000..b815c0b79718
--- /dev/null
+++ b/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data.rs
@@ -0,0 +1,107 @@
+//! Runs `rustc --print target-spec-json` to get the target_data_layout.
+
+use anyhow::Context;
+use base_db::target;
+use rustc_hash::FxHashMap;
+use serde_derive::Deserialize;
+use toolchain::Tool;
+
+use crate::{Sysroot, toolchain_info::QueryConfig, utf8_stdout};
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum Arch {
+    Wasm32,
+    Wasm64,
+    #[serde(other)]
+    Other,
+}
+
+impl From for target::Arch {
+    fn from(value: Arch) -> Self {
+        match value {
+            Arch::Wasm32 => target::Arch::Wasm32,
+            Arch::Wasm64 => target::Arch::Wasm64,
+            Arch::Other => target::Arch::Other,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct TargetSpec {
+    pub data_layout: String,
+    pub arch: Arch,
+}
+
+/// Uses `rustc --print target-spec-json`.
+pub fn get(
+    config: QueryConfig<'_>,
+    target: Option<&str>,
+    extra_env: &FxHashMap>,
+) -> anyhow::Result {
+    const RUSTC_ARGS: [&str; 2] = ["--print", "target-spec-json"];
+    let process = |output: String| {
+        let target_spec = serde_json::from_str::(&output).map_err(|_| {
+            anyhow::format_err!("could not parse target-spec-json from command output")
+        })?;
+        Ok(target::TargetData {
+            arch: target_spec.arch.into(),
+            data_layout: target_spec.data_layout.into_boxed_str(),
+        })
+    };
+    let (sysroot, current_dir) = match config {
+        QueryConfig::Cargo(sysroot, cargo_toml, _) => {
+            let mut cmd = sysroot.tool(Tool::Cargo, cargo_toml.parent(), extra_env);
+            cmd.env("RUSTC_BOOTSTRAP", "1");
+            cmd.args(["rustc", "-Z", "unstable-options"]).args(RUSTC_ARGS);
+            if let Some(target) = target {
+                cmd.args(["--target", target]);
+            }
+            cmd.args(["--", "-Z", "unstable-options"]);
+            match utf8_stdout(&mut cmd) {
+                Ok(output) => return process(output),
+                Err(e) => {
+                    tracing::warn!(%e, "failed to run `{cmd:?}`, falling back to invoking rustc directly");
+                    (sysroot, cargo_toml.parent().as_ref())
+                }
+            }
+        }
+        QueryConfig::Rustc(sysroot, current_dir) => (sysroot, current_dir),
+    };
+
+    let mut cmd = Sysroot::tool(sysroot, Tool::Rustc, current_dir, extra_env);
+    cmd.env("RUSTC_BOOTSTRAP", "1").args(["-Z", "unstable-options"]).args(RUSTC_ARGS);
+    if let Some(target) = target {
+        cmd.args(["--target", target]);
+    }
+    utf8_stdout(&mut cmd)
+        .with_context(|| format!("unable to fetch target-data-layout via `{cmd:?}`"))
+        .and_then(process)
+}
+
+#[cfg(test)]
+mod tests {
+    use paths::{AbsPathBuf, Utf8PathBuf};
+
+    use crate::{ManifestPath, Sysroot};
+
+    use super::*;
+
+    #[test]
+    fn cargo() {
+        let manifest_path = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
+        let sysroot = Sysroot::empty();
+        let manifest_path =
+            ManifestPath::try_from(AbsPathBuf::assert(Utf8PathBuf::from(manifest_path))).unwrap();
+        let cfg = QueryConfig::Cargo(&sysroot, &manifest_path, &None);
+        assert!(get(cfg, None, &FxHashMap::default()).is_ok());
+    }
+
+    #[test]
+    fn rustc() {
+        let sysroot = Sysroot::empty();
+        let cfg = QueryConfig::Rustc(&sysroot, env!("CARGO_MANIFEST_DIR").as_ref());
+        assert!(get(cfg, None, &FxHashMap::default()).is_ok());
+    }
+}
diff --git a/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data_layout.rs b/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data_layout.rs
deleted file mode 100644
index a28f468e692d..000000000000
--- a/src/tools/rust-analyzer/crates/project-model/src/toolchain_info/target_data_layout.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-//! Runs `rustc --print target-spec-json` to get the target_data_layout.
-
-use anyhow::Context;
-use rustc_hash::FxHashMap;
-use toolchain::Tool;
-
-use crate::{Sysroot, toolchain_info::QueryConfig, utf8_stdout};
-
-/// Uses `rustc --print target-spec-json`.
-pub fn get(
-    config: QueryConfig<'_>,
-    target: Option<&str>,
-    extra_env: &FxHashMap>,
-) -> anyhow::Result {
-    const RUSTC_ARGS: [&str; 2] = ["--print", "target-spec-json"];
-    let process = |output: String| {
-        (|| Some(output.split_once(r#""data-layout": ""#)?.1.split_once('"')?.0.to_owned()))()
-            .ok_or_else(|| {
-                anyhow::format_err!("could not parse target-spec-json from command output")
-            })
-    };
-    let (sysroot, current_dir) = match config {
-        QueryConfig::Cargo(sysroot, cargo_toml, _) => {
-            let mut cmd = sysroot.tool(Tool::Cargo, cargo_toml.parent(), extra_env);
-            cmd.env("RUSTC_BOOTSTRAP", "1");
-            cmd.args(["rustc", "-Z", "unstable-options"]).args(RUSTC_ARGS).args([
-                "--",
-                "-Z",
-                "unstable-options",
-            ]);
-            if let Some(target) = target {
-                cmd.args(["--target", target]);
-            }
-            match utf8_stdout(&mut cmd) {
-                Ok(output) => return process(output),
-                Err(e) => {
-                    tracing::warn!(%e, "failed to run `{cmd:?}`, falling back to invoking rustc directly");
-                    (sysroot, cargo_toml.parent().as_ref())
-                }
-            }
-        }
-        QueryConfig::Rustc(sysroot, current_dir) => (sysroot, current_dir),
-    };
-
-    let mut cmd = Sysroot::tool(sysroot, Tool::Rustc, current_dir, extra_env);
-    cmd.env("RUSTC_BOOTSTRAP", "1").args(["-Z", "unstable-options"]).args(RUSTC_ARGS);
-    if let Some(target) = target {
-        cmd.args(["--target", target]);
-    }
-    utf8_stdout(&mut cmd)
-        .with_context(|| format!("unable to fetch target-data-layout via `{cmd:?}`"))
-        .and_then(process)
-}
-
-#[cfg(test)]
-mod tests {
-    use paths::{AbsPathBuf, Utf8PathBuf};
-
-    use crate::{ManifestPath, Sysroot};
-
-    use super::*;
-
-    #[test]
-    fn cargo() {
-        let manifest_path = concat!(env!("CARGO_MANIFEST_DIR"), "/Cargo.toml");
-        let sysroot = Sysroot::empty();
-        let manifest_path =
-            ManifestPath::try_from(AbsPathBuf::assert(Utf8PathBuf::from(manifest_path))).unwrap();
-        let cfg = QueryConfig::Cargo(&sysroot, &manifest_path, &None);
-        assert!(get(cfg, None, &FxHashMap::default()).is_ok());
-    }
-
-    #[test]
-    fn rustc() {
-        let sysroot = Sysroot::empty();
-        let cfg = QueryConfig::Rustc(&sysroot, env!("CARGO_MANIFEST_DIR").as_ref());
-        assert!(get(cfg, None, &FxHashMap::default()).is_ok());
-    }
-}
diff --git a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
index 5b36e10fd692..b88db419574d 100644
--- a/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
+++ b/src/tools/rust-analyzer/crates/project-model/src/workspace.rs
@@ -8,7 +8,7 @@ use anyhow::Context;
 use base_db::{
     CrateBuilderId, CrateDisplayName, CrateGraphBuilder, CrateName, CrateOrigin,
     CrateWorkspaceData, DependencyBuilder, Env, LangCrateOrigin, ProcMacroLoadingError,
-    ProcMacroPaths, TargetLayoutLoadResult,
+    ProcMacroPaths, target::TargetLoadResult,
 };
 use cfg::{CfgAtom, CfgDiff, CfgOptions};
 use intern::{Symbol, sym};
@@ -16,8 +16,9 @@ use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
 use rustc_hash::{FxHashMap, FxHashSet};
 use semver::Version;
 use span::{Edition, FileId};
-use toolchain::Tool;
+use toolchain::{NO_RUSTUP_AUTO_INSTALL_ENV, Tool};
 use tracing::instrument;
+use tracing::{debug, error, info};
 use triomphe::Arc;
 
 use crate::{
@@ -30,10 +31,9 @@ use crate::{
     env::{cargo_config_env, inject_cargo_env, inject_cargo_package_env, inject_rustc_tool_env},
     project_json::{Crate, CrateArrayIdx},
     sysroot::RustLibSrcWorkspace,
-    toolchain_info::{QueryConfig, rustc_cfg, target_data_layout, target_tuple, version},
+    toolchain_info::{QueryConfig, rustc_cfg, target_data, target_tuple, version},
     utf8_stdout,
 };
-use tracing::{debug, error, info};
 
 pub type FileLoader<'a> = &'a mut dyn for<'b> FnMut(&'b AbsPath) -> Option;
 
@@ -63,7 +63,7 @@ pub struct ProjectWorkspace {
     /// The toolchain version used by this workspace.
     pub toolchain: Option,
     /// The target data layout queried for workspace.
-    pub target_layout: TargetLayoutLoadResult,
+    pub target: TargetLoadResult,
     /// A set of cfg overrides for this workspace.
     pub cfg_overrides: CfgOverrides,
     /// Additional includes to add for the VFS.
@@ -115,7 +115,7 @@ impl fmt::Debug for ProjectWorkspace {
             sysroot,
             rustc_cfg,
             toolchain,
-            target_layout,
+            target: target_layout,
             cfg_overrides,
             extra_includes,
             set_test,
@@ -157,7 +157,6 @@ impl fmt::Debug for ProjectWorkspace {
                 .field("file", &file)
                 .field("cargo_script", &cargo_script.is_some())
                 .field("n_sysroot_crates", &sysroot.num_packages())
-                .field("cargo_script", &cargo_script.is_some())
                 .field("n_rustc_cfg", &rustc_cfg.len())
                 .field("toolchain", &toolchain)
                 .field("data_layout", &target_layout)
@@ -310,8 +309,8 @@ impl ProjectWorkspace {
             let rustc_cfg = s.spawn(|| {
                 rustc_cfg::get(toolchain_config, targets.first().map(Deref::deref), extra_env)
             });
-            let data_layout = s.spawn(|| {
-                target_data_layout::get(
+            let target_data = s.spawn(|| {
+                target_data::get(
                     toolchain_config,
                     targets.first().map(Deref::deref),
                     extra_env,
@@ -384,41 +383,40 @@ impl ProjectWorkspace {
                         toolchain.clone(),
                     )),
                     config.no_deps,
-                    workspace_dir,
                     &target_dir,
                     progress,
                 )
             });
-            let cargo_config_extra_env =
-                s.spawn(move || cargo_config_env(cargo_toml, &config_file));
+            let cargo_env =
+                s.spawn(move || cargo_config_env(cargo_toml, &config_file, &config.extra_env));
             thread::Result::Ok((
                 rustc_cfg.join()?,
-                data_layout.join()?,
+                target_data.join()?,
                 rustc_dir.join()?,
                 loaded_sysroot.join()?,
                 cargo_metadata.join()?,
-                cargo_config_extra_env.join()?,
+                cargo_env.join()?,
             ))
         });
 
-        let (
-            rustc_cfg,
-            data_layout,
-            mut rustc,
-            loaded_sysroot,
-            cargo_metadata,
-            cargo_config_extra_env,
-        ) = match join {
-            Ok(it) => it,
-            Err(e) => std::panic::resume_unwind(e),
-        };
+        let (rustc_cfg, data_layout, mut rustc, loaded_sysroot, cargo_metadata, mut cargo_env) =
+            match join {
+                Ok(it) => it,
+                Err(e) => std::panic::resume_unwind(e),
+            };
+
+        for (key, value) in config.extra_env.iter() {
+            if let Some(value) = value {
+                cargo_env.insert(key.clone(), value.clone());
+            }
+        }
 
         let (meta, error) = cargo_metadata.with_context(|| {
             format!(
                 "Failed to read Cargo metadata from Cargo.toml file {cargo_toml}, {toolchain:?}",
             )
         })?;
-        let cargo = CargoWorkspace::new(meta, cargo_toml.clone(), cargo_config_extra_env, false);
+        let cargo = CargoWorkspace::new(meta, cargo_toml.clone(), cargo_env, false);
         if let Some(loaded_sysroot) = loaded_sysroot {
             tracing::info!(src_root = ?sysroot.rust_lib_src_root(), root = %loaded_sysroot, "Loaded sysroot");
             sysroot.set_workspace(loaded_sysroot);
@@ -443,7 +441,7 @@ impl ProjectWorkspace {
             rustc_cfg,
             cfg_overrides: cfg_overrides.clone(),
             toolchain,
-            target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
+            target: data_layout.map_err(|it| it.to_string().into()),
             extra_includes: extra_includes.clone(),
             set_test: *set_test,
         })
@@ -481,18 +479,13 @@ impl ProjectWorkspace {
                 rustc_cfg::get(query_config, targets.first().map(Deref::deref), &config.extra_env)
             });
             let data_layout = s.spawn(|| {
-                target_data_layout::get(
-                    query_config,
-                    targets.first().map(Deref::deref),
-                    &config.extra_env,
-                )
+                target_data::get(query_config, targets.first().map(Deref::deref), &config.extra_env)
             });
             let loaded_sysroot = s.spawn(|| {
                 if let Some(sysroot_project) = sysroot_project {
                     sysroot.load_workspace(
                         &RustSourceWorkspaceConfig::Json(*sysroot_project),
                         config.no_deps,
-                        project_root,
                         &target_dir,
                         progress,
                     )
@@ -504,7 +497,6 @@ impl ProjectWorkspace {
                             toolchain.clone(),
                         )),
                         config.no_deps,
-                        project_root,
                         &target_dir,
                         progress,
                     )
@@ -514,7 +506,7 @@ impl ProjectWorkspace {
             thread::Result::Ok((rustc_cfg.join()?, data_layout.join()?, loaded_sysroot.join()?))
         });
 
-        let (rustc_cfg, target_layout, loaded_sysroot) = match join {
+        let (rustc_cfg, target_data, loaded_sysroot) = match join {
             Ok(it) => it,
             Err(e) => std::panic::resume_unwind(e),
         };
@@ -528,7 +520,7 @@ impl ProjectWorkspace {
             sysroot,
             rustc_cfg,
             toolchain,
-            target_layout: target_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
+            target: target_data.map_err(|it| it.to_string().into()),
             cfg_overrides: config.cfg_overrides.clone(),
             extra_includes: config.extra_includes.clone(),
             set_test: config.set_test,
@@ -552,7 +544,7 @@ impl ProjectWorkspace {
         let targets = target_tuple::get(query_config, config.target.as_deref(), &config.extra_env)
             .unwrap_or_default();
         let rustc_cfg = rustc_cfg::get(query_config, None, &config.extra_env);
-        let data_layout = target_data_layout::get(query_config, None, &config.extra_env);
+        let target_data = target_data::get(query_config, None, &config.extra_env);
         let target_dir = config
             .target_dir
             .clone()
@@ -566,7 +558,6 @@ impl ProjectWorkspace {
                 toolchain.clone(),
             )),
             config.no_deps,
-            dir,
             &target_dir,
             &|_| (),
         );
@@ -595,7 +586,8 @@ impl ProjectWorkspace {
             .unwrap_or_else(|| dir.join("target").into());
         let cargo_script =
             fetch_metadata.exec(&target_dir, false, &|_| ()).ok().map(|(ws, error)| {
-                let cargo_config_extra_env = cargo_config_env(detached_file, &config_file);
+                let cargo_config_extra_env =
+                    cargo_config_env(detached_file, &config_file, &config.extra_env);
                 (
                     CargoWorkspace::new(ws, detached_file.clone(), cargo_config_extra_env, false),
                     WorkspaceBuildScripts::default(),
@@ -611,7 +603,7 @@ impl ProjectWorkspace {
             sysroot,
             rustc_cfg,
             toolchain,
-            target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
+            target: target_data.map_err(|it| it.to_string().into()),
             cfg_overrides: config.cfg_overrides.clone(),
             extra_includes: config.extra_includes.clone(),
             set_test: config.set_test,
@@ -752,7 +744,7 @@ impl ProjectWorkspace {
     pub fn to_roots(&self) -> Vec {
         let mk_sysroot = || {
             let mut r = match self.sysroot.workspace() {
-                RustLibSrcWorkspace::Workspace(ws) => ws
+                RustLibSrcWorkspace::Workspace { ws, .. } => ws
                     .packages()
                     .filter_map(|pkg| {
                         if ws[pkg].is_local {
@@ -943,7 +935,7 @@ impl ProjectWorkspace {
         let Self { kind, sysroot, cfg_overrides, rustc_cfg, .. } = self;
         let crate_ws_data = Arc::new(CrateWorkspaceData {
             toolchain: self.toolchain.clone(),
-            data_layout: self.target_layout.clone(),
+            target: self.target.clone(),
         });
         let (crate_graph, proc_macros) = match kind {
             ProjectWorkspaceKind::Json(project) => project_json_to_crate_graph(
@@ -1001,13 +993,15 @@ impl ProjectWorkspace {
     }
 
     pub fn eq_ignore_build_data(&self, other: &Self) -> bool {
-        let Self { kind, sysroot, rustc_cfg, toolchain, target_layout, cfg_overrides, .. } = self;
+        let Self {
+            kind, sysroot, rustc_cfg, toolchain, target: target_layout, cfg_overrides, ..
+        } = self;
         let Self {
             kind: o_kind,
             sysroot: o_sysroot,
             rustc_cfg: o_rustc_cfg,
             toolchain: o_toolchain,
-            target_layout: o_target_layout,
+            target: o_target_layout,
             cfg_overrides: o_cfg_overrides,
             ..
         } = other;
@@ -1096,7 +1090,13 @@ fn project_json_to_crate_graph(
                 },
                 file_id,
             )| {
-                let env = env.clone().into_iter().collect();
+                let mut env = env.clone().into_iter().collect::();
+                // Override existing env vars with those from `extra_env`
+                env.extend(
+                    extra_env
+                        .iter()
+                        .filter_map(|(k, v)| v.as_ref().map(|v| (k.clone(), v.clone()))),
+                );
 
                 let target_cfgs = match target.as_deref() {
                     Some(target) => cfg_cache.entry(target).or_insert_with(|| {
@@ -1738,7 +1738,7 @@ fn sysroot_to_crate_graph(
 ) -> (SysrootPublicDeps, Option) {
     let _p = tracing::info_span!("sysroot_to_crate_graph").entered();
     match sysroot.workspace() {
-        RustLibSrcWorkspace::Workspace(cargo) => {
+        RustLibSrcWorkspace::Workspace { ws: cargo, .. } => {
             let (sysroot_cg, sysroot_pm) = cargo_to_crate_graph(
                 load,
                 None,
@@ -1910,6 +1910,7 @@ fn cargo_target_dir(
 ) -> Option {
     let cargo = sysroot.tool(Tool::Cargo, manifest.parent(), extra_env);
     let mut meta = cargo_metadata::MetadataCommand::new();
+    meta.env(NO_RUSTUP_AUTO_INSTALL_ENV.0, NO_RUSTUP_AUTO_INSTALL_ENV.1);
     meta.cargo_path(cargo.get_program());
     meta.manifest_path(manifest);
     // `--no-deps` doesn't (over)write lockfiles as it doesn't do any package resolve.
diff --git a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model.txt b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model.txt
index 3722e2c72168..4f6ce4dc9537 100644
--- a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model.txt
+++ b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model.txt
@@ -70,7 +70,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -155,7 +155,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -240,7 +240,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -325,7 +325,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -406,7 +406,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
diff --git a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_selective_overrides.txt b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_selective_overrides.txt
index 3722e2c72168..4f6ce4dc9537 100644
--- a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_selective_overrides.txt
+++ b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_selective_overrides.txt
@@ -70,7 +70,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -155,7 +155,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -240,7 +240,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -325,7 +325,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -406,7 +406,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
diff --git a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_wildcard_overrides.txt b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_wildcard_overrides.txt
index 7b156ea63a58..6862918e09ae 100644
--- a/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_wildcard_overrides.txt
+++ b/src/tools/rust-analyzer/crates/project-model/test_data/output/cargo_hello_world_project_model_with_wildcard_overrides.txt
@@ -69,7 +69,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -153,7 +153,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -237,7 +237,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -321,7 +321,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
@@ -402,7 +402,7 @@
             },
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
+            target: Err(
                 "target_data_layout not loaded",
             ),
             toolchain: None,
diff --git a/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_cfg_groups.txt b/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_cfg_groups.txt
index 98fe598eb3a3..28ad3236ae81 100644
--- a/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_cfg_groups.txt
+++ b/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_cfg_groups.txt
@@ -43,8 +43,8 @@
             entries: {},
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
-                "test has no data layout",
+            target: Err(
+                "test has no target data",
             ),
             toolchain: None,
         },
@@ -93,8 +93,8 @@
             entries: {},
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
-                "test has no data layout",
+            target: Err(
+                "test has no target data",
             ),
             toolchain: None,
         },
diff --git a/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_hello_world_project_model.txt b/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_hello_world_project_model.txt
index 0dc373b5b47e..dabb3aa67441 100644
--- a/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_hello_world_project_model.txt
+++ b/src/tools/rust-analyzer/crates/project-model/test_data/output/rust_project_hello_world_project_model.txt
@@ -40,8 +40,8 @@
             entries: {},
         },
         ws_data: CrateWorkspaceData {
-            data_layout: Err(
-                "test has no data layout",
+            target: Err(
+                "test has no target data",
             ),
             toolchain: None,
         },
diff --git a/src/tools/rust-analyzer/crates/query-group-macro/src/lib.rs b/src/tools/rust-analyzer/crates/query-group-macro/src/lib.rs
index ec4b6b2a4ac3..277cc0b269d7 100644
--- a/src/tools/rust-analyzer/crates/query-group-macro/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/query-group-macro/src/lib.rs
@@ -278,15 +278,15 @@ pub(crate) fn query_group_impl(
                 return Err(syn::Error::new(signature.span(), "Queries must have a return type"));
             };
 
-            if let syn::Type::Path(ref ty_path) = *return_ty {
-                if matches!(query_kind, QueryKind::Input) {
-                    let field = InputStructField {
-                        name: method_name.to_token_stream(),
-                        ty: ty_path.path.to_token_stream(),
-                    };
+            if let syn::Type::Path(ref ty_path) = *return_ty
+                && matches!(query_kind, QueryKind::Input)
+            {
+                let field = InputStructField {
+                    name: method_name.to_token_stream(),
+                    ty: ty_path.path.to_token_stream(),
+                };
 
-                    input_struct_fields.push(field);
-                }
+                input_struct_fields.push(field);
             }
 
             if let Some(block) = &mut method.default {
diff --git a/src/tools/rust-analyzer/crates/query-group-macro/src/queries.rs b/src/tools/rust-analyzer/crates/query-group-macro/src/queries.rs
index c151cca07272..7698ce5fff13 100644
--- a/src/tools/rust-analyzer/crates/query-group-macro/src/queries.rs
+++ b/src/tools/rust-analyzer/crates/query-group-macro/src/queries.rs
@@ -48,7 +48,8 @@ impl ToTokens for TrackedQuery {
                 quote!(#(#options),*)
             })
             .into_iter()
-            .chain(self.lru.map(|lru| quote!(lru = #lru)));
+            .chain(self.lru.map(|lru| quote!(lru = #lru)))
+            .chain(Some(quote!(unsafe(non_update_return_type))));
         let annotation = quote!(#[salsa_macros::tracked( #(#options),* )]);
 
         let pat_and_tys = &self.pat_and_tys;
@@ -326,7 +327,8 @@ impl ToTokens for Lookup {
         let wrapper_struct = self.interned_struct_path.to_token_stream();
         let method = quote! {
             #sig {
-                #wrapper_struct::ingredient(self).data(self.as_dyn_database(), id.as_id()).0.clone()
+                let zalsa = self.zalsa();
+                #wrapper_struct::ingredient(zalsa).data(zalsa, id.as_id()).0.clone()
             }
         };
 
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/Cargo.toml b/src/tools/rust-analyzer/crates/rust-analyzer/Cargo.toml
index b301a7189b3c..c746f848b6a0 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/Cargo.toml
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/Cargo.toml
@@ -75,6 +75,8 @@ vfs-notify.workspace = true
 vfs.workspace = true
 paths.workspace = true
 
+ra-ap-rustc_type_ir.workspace = true
+
 [target.'cfg(windows)'.dependencies]
 windows-sys = { version = "0.60", features = [
   "Win32_System_Diagnostics_Debug",
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/bin/main.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/bin/main.rs
index 4dba97c8ec49..cc8db1b841ea 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/bin/main.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/bin/main.rs
@@ -122,12 +122,12 @@ fn setup_logging(log_file_flag: Option) -> anyhow::Result<()> {
         // directory which we set to the project workspace.
         // https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/general-environment-variables
         // https://docs.microsoft.com/en-us/windows/win32/api/dbghelp/nf-dbghelp-syminitialize
-        if let Ok(path) = env::current_exe() {
-            if let Some(path) = path.parent() {
-                // SAFETY: This is safe because this is single-threaded.
-                unsafe {
-                    env::set_var("_NT_SYMBOL_PATH", path);
-                }
+        if let Ok(path) = env::current_exe()
+            && let Some(path) = path.parent()
+        {
+            // SAFETY: This is safe because this is single-threaded.
+            unsafe {
+                env::set_var("_NT_SYMBOL_PATH", path);
             }
         }
     }
@@ -160,9 +160,9 @@ fn setup_logging(log_file_flag: Option) -> anyhow::Result<()> {
 
     rust_analyzer::tracing::Config {
         writer,
-        // Deliberately enable all `error` logs if the user has not set RA_LOG, as there is usually
+        // Deliberately enable all `warn` logs if the user has not set RA_LOG, as there is usually
         // useful information in there for debugging.
-        filter: env::var("RA_LOG").ok().unwrap_or_else(|| "error".to_owned()),
+        filter: env::var("RA_LOG").ok().unwrap_or_else(|| "warn".to_owned()),
         chalk_filter: env::var("CHALK_DEBUG").ok(),
         profile_filter: env::var("RA_PROFILE").ok(),
         json_profile_filter: std::env::var("RA_PROFILE_JSON").ok(),
@@ -208,13 +208,24 @@ fn run_server() -> anyhow::Result<()> {
     tracing::info!("InitializeParams: {}", initialize_params);
     let lsp_types::InitializeParams {
         root_uri,
-        capabilities,
+        mut capabilities,
         workspace_folders,
         initialization_options,
         client_info,
         ..
     } = from_json::("InitializeParams", &initialize_params)?;
 
+    // lsp-types has a typo in the `/capabilities/workspace/diagnostics` field, its typoed as `diagnostic`
+    if let Some(val) = initialize_params.pointer("/capabilities/workspace/diagnostics")
+        && let Ok(diag_caps) = from_json::(
+            "DiagnosticWorkspaceClientCapabilities",
+            val,
+        )
+    {
+        tracing::info!("Patching lsp-types workspace diagnostics capabilities: {diag_caps:#?}");
+        capabilities.workspace.get_or_insert_default().diagnostic.get_or_insert(diag_caps);
+    }
+
     let root_path = match root_uri
         .and_then(|it| it.to_file_path().ok())
         .map(patch_path_prefix)
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
index 4f75d14834c6..de24bc09ff0f 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/analysis_stats.rs
@@ -10,21 +10,22 @@ use std::{
 
 use cfg::{CfgAtom, CfgDiff};
 use hir::{
-    Adt, AssocItem, Crate, DefWithBody, HasSource, HirDisplay, ImportPathConfig, ModuleDef, Name,
+    Adt, AssocItem, Crate, DefWithBody, FindPathConfig, HasCrate, HasSource, HirDisplay, ModuleDef,
+    Name,
     db::{DefDatabase, ExpandDatabase, HirDatabase},
+    next_solver::{DbInterner, GenericArgs},
 };
 use hir_def::{
     SyntheticSyntax,
     expr_store::BodySourceMap,
     hir::{ExprId, PatId},
 };
-use hir_ty::{Interner, Substitution, TyExt, TypeFlags};
 use ide::{
     Analysis, AnalysisHost, AnnotationConfig, DiagnosticsConfig, Edition, InlayFieldsToResolve,
     InlayHintsConfig, LineCol, RootDatabase,
 };
 use ide_db::{
-    EditionedFileId, LineIndexDatabase, SnippetCap,
+    EditionedFileId, LineIndexDatabase, MiniCore, SnippetCap,
     base_db::{SourceDatabase, salsa::Database},
 };
 use itertools::Itertools;
@@ -34,6 +35,7 @@ use profile::StopWatch;
 use project_model::{CargoConfig, CfgOverrides, ProjectManifest, ProjectWorkspace, RustLibSource};
 use rayon::prelude::*;
 use rustc_hash::{FxHashMap, FxHashSet};
+use rustc_type_ir::inherent::Ty as _;
 use syntax::AstNode;
 use vfs::{AbsPathBuf, Vfs, VfsPath};
 
@@ -136,34 +138,30 @@ impl flags::AnalysisStats {
         for source_root_id in source_roots {
             let source_root = db.source_root(source_root_id).source_root(db);
             for file_id in source_root.iter() {
-                if let Some(p) = source_root.path_for_file(&file_id) {
-                    if let Some((_, Some("rs"))) = p.name_and_extension() {
-                        // measure workspace/project code
-                        if !source_root.is_library || self.with_deps {
-                            let length = db.file_text(file_id).text(db).lines().count();
-                            let item_stats = db
-                                .file_item_tree(
-                                    EditionedFileId::current_edition(db, file_id).into(),
-                                )
-                                .item_tree_stats()
-                                .into();
+                if let Some(p) = source_root.path_for_file(&file_id)
+                    && let Some((_, Some("rs"))) = p.name_and_extension()
+                {
+                    // measure workspace/project code
+                    if !source_root.is_library || self.with_deps {
+                        let length = db.file_text(file_id).text(db).lines().count();
+                        let item_stats = db
+                            .file_item_tree(EditionedFileId::current_edition(db, file_id).into())
+                            .item_tree_stats()
+                            .into();
 
-                            workspace_loc += length;
-                            workspace_item_trees += 1;
-                            workspace_item_stats += item_stats;
-                        } else {
-                            let length = db.file_text(file_id).text(db).lines().count();
-                            let item_stats = db
-                                .file_item_tree(
-                                    EditionedFileId::current_edition(db, file_id).into(),
-                                )
-                                .item_tree_stats()
-                                .into();
+                        workspace_loc += length;
+                        workspace_item_trees += 1;
+                        workspace_item_stats += item_stats;
+                    } else {
+                        let length = db.file_text(file_id).text(db).lines().count();
+                        let item_stats = db
+                            .file_item_tree(EditionedFileId::current_edition(db, file_id).into())
+                            .item_tree_stats()
+                            .into();
 
-                            dep_loc += length;
-                            dep_item_trees += 1;
-                            dep_item_stats += item_stats;
-                        }
+                        dep_loc += length;
+                        dep_item_trees += 1;
+                        dep_item_stats += item_stats;
                     }
                 }
             }
@@ -314,34 +312,41 @@ impl flags::AnalysisStats {
             shuffle(&mut rng, &mut bodies);
         }
 
-        if !self.skip_lowering {
-            self.run_body_lowering(db, &vfs, &bodies, verbosity);
-        }
+        hir::attach_db(db, || {
+            if !self.skip_lowering {
+                self.run_body_lowering(db, &vfs, &bodies, verbosity);
+            }
 
-        if !self.skip_inference {
-            self.run_inference(db, &vfs, &bodies, verbosity);
-        }
+            if !self.skip_inference {
+                self.run_inference(db, &vfs, &bodies, verbosity);
+            }
 
-        if !self.skip_mir_stats {
-            self.run_mir_lowering(db, &bodies, verbosity);
-        }
+            if !self.skip_mir_stats {
+                self.run_mir_lowering(db, &bodies, verbosity);
+            }
 
-        if !self.skip_data_layout {
-            self.run_data_layout(db, &adts, verbosity);
-        }
+            if !self.skip_data_layout {
+                self.run_data_layout(db, &adts, verbosity);
+            }
 
-        if !self.skip_const_eval {
-            self.run_const_eval(db, &bodies, verbosity);
-        }
+            if !self.skip_const_eval {
+                self.run_const_eval(db, &bodies, verbosity);
+            }
+        });
+
+        file_ids.sort();
+        file_ids.dedup();
 
         if self.run_all_ide_things {
-            self.run_ide_things(host.analysis(), file_ids.clone());
+            self.run_ide_things(host.analysis(), &file_ids, db, &vfs, verbosity);
         }
 
         if self.run_term_search {
-            self.run_term_search(&workspace, db, &vfs, file_ids, verbosity);
+            self.run_term_search(&workspace, db, &vfs, &file_ids, verbosity);
         }
 
+        hir::clear_tls_solver_cache();
+
         let db = host.raw_database_mut();
         db.trigger_lru_eviction();
 
@@ -365,6 +370,7 @@ impl flags::AnalysisStats {
         let mut all = 0;
         let mut fail = 0;
         for &a in adts {
+            let interner = DbInterner::new_with(db, Some(a.krate(db).base()), None);
             let generic_params = db.generic_params(a.into());
             if generic_params.iter_type_or_consts().next().is_some()
                 || generic_params.iter_lt().next().is_some()
@@ -375,7 +381,7 @@ impl flags::AnalysisStats {
             all += 1;
             let Err(e) = db.layout_of_adt(
                 hir_def::AdtId::from(a),
-                Substitution::empty(Interner),
+                GenericArgs::new_from_iter(interner, []),
                 db.trait_environment(a.into()),
             ) else {
                 continue;
@@ -394,15 +400,27 @@ impl flags::AnalysisStats {
     }
 
     fn run_const_eval(&self, db: &RootDatabase, bodies: &[DefWithBody], verbosity: Verbosity) {
+        let len = bodies
+            .iter()
+            .filter(|body| matches!(body, DefWithBody::Const(_) | DefWithBody::Static(_)))
+            .count();
+        let mut bar = match verbosity {
+            Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
+            _ if self.parallel || self.output.is_some() => ProgressReport::hidden(),
+            _ => ProgressReport::new(len),
+        };
+
         let mut sw = self.stop_watch();
         let mut all = 0;
         let mut fail = 0;
         for &b in bodies {
+            bar.set_message(move || format!("const eval: {}", full_name(db, b, b.module(db))));
             let res = match b {
                 DefWithBody::Const(c) => c.eval(db),
                 DefWithBody::Static(s) => s.eval(db),
                 _ => continue,
             };
+            bar.inc(1);
             all += 1;
             let Err(error) = res else {
                 continue;
@@ -410,10 +428,11 @@ impl flags::AnalysisStats {
             if verbosity.is_spammy() {
                 let full_name =
                     full_name_of_item(db, b.module(db), b.name(db).unwrap_or(Name::missing()));
-                println!("Const eval for {full_name} failed due {error:?}");
+                bar.println(format!("Const eval for {full_name} failed due {error:?}"));
             }
             fail += 1;
         }
+        bar.finish_and_clear();
         let const_eval_time = sw.elapsed();
         eprintln!("{:<20} {}", "Const evaluation:", const_eval_time);
         eprintln!("Failed const evals: {fail} ({}%)", percentage(fail, all));
@@ -421,12 +440,13 @@ impl flags::AnalysisStats {
         report_metric("const eval time", const_eval_time.time.as_millis() as u64, "ms");
     }
 
+    /// Invariant: `file_ids` must be sorted and deduped before passing into here
     fn run_term_search(
         &self,
         ws: &ProjectWorkspace,
         db: &RootDatabase,
         vfs: &Vfs,
-        mut file_ids: Vec,
+        file_ids: &[EditionedFileId],
         verbosity: Verbosity,
     ) {
         let cargo_config = CargoConfig {
@@ -444,9 +464,6 @@ impl flags::AnalysisStats {
             _ => ProgressReport::new(file_ids.len()),
         };
 
-        file_ids.sort();
-        file_ids.dedup();
-
         #[derive(Debug, Default)]
         struct Acc {
             tail_expr_syntax_hits: u64,
@@ -460,7 +477,7 @@ impl flags::AnalysisStats {
         bar.tick();
         let mut sw = self.stop_watch();
 
-        for &file_id in &file_ids {
+        for &file_id in file_ids {
             let file_id = file_id.editioned_file_id(db);
             let sema = hir::Semantics::new(db);
             let display_target = match sema.first_crate(file_id.file_id()) {
@@ -539,7 +556,7 @@ impl flags::AnalysisStats {
                         .gen_source_code(
                             &scope,
                             &mut formatter,
-                            ImportPathConfig {
+                            FindPathConfig {
                                 prefer_no_std: false,
                                 prefer_prelude: true,
                                 prefer_absolute: false,
@@ -560,29 +577,35 @@ impl flags::AnalysisStats {
                         std::fs::write(path, txt).unwrap();
 
                         let res = ws.run_build_scripts(&cargo_config, &|_| ()).unwrap();
-                        if let Some(err) = res.error() {
-                            if err.contains("error: could not compile") {
-                                if let Some(mut err_idx) = err.find("error[E") {
-                                    err_idx += 7;
-                                    let err_code = &err[err_idx..err_idx + 4];
-                                    match err_code {
-                                        "0282" | "0283" => continue, // Byproduct of testing method
-                                        "0277" | "0308" if generated.contains(&todo) => continue, // See https://github.com/rust-lang/rust/issues/69882
-                                        // FIXME: In some rare cases `AssocItem::container_or_implemented_trait` returns `None` for trait methods.
-                                        // Generated code is valid in case traits are imported
-                                        "0599" if err.contains("the following trait is implemented but not in scope") => continue,
-                                        _ => (),
+                        if let Some(err) = res.error()
+                            && err.contains("error: could not compile")
+                        {
+                            if let Some(mut err_idx) = err.find("error[E") {
+                                err_idx += 7;
+                                let err_code = &err[err_idx..err_idx + 4];
+                                match err_code {
+                                    "0282" | "0283" => continue, // Byproduct of testing method
+                                    "0277" | "0308" if generated.contains(&todo) => continue, // See https://github.com/rust-lang/rust/issues/69882
+                                    // FIXME: In some rare cases `AssocItem::container_or_implemented_trait` returns `None` for trait methods.
+                                    // Generated code is valid in case traits are imported
+                                    "0599"
+                                        if err.contains(
+                                            "the following trait is implemented but not in scope",
+                                        ) =>
+                                    {
+                                        continue;
                                     }
-                                    bar.println(err);
-                                    bar.println(generated);
-                                    acc.error_codes
-                                        .entry(err_code.to_owned())
-                                        .and_modify(|n| *n += 1)
-                                        .or_insert(1);
-                                } else {
-                                    acc.syntax_errors += 1;
-                                    bar.println(format!("Syntax error: \n{err}"));
+                                    _ => (),
                                 }
+                                bar.println(err);
+                                bar.println(generated);
+                                acc.error_codes
+                                    .entry(err_code.to_owned())
+                                    .and_modify(|n| *n += 1)
+                                    .or_insert(1);
+                            } else {
+                                acc.syntax_errors += 1;
+                                bar.println(format!("Syntax error: \n{err}"));
                             }
                         }
                     }
@@ -657,6 +680,10 @@ impl flags::AnalysisStats {
         let mut all = 0;
         let mut fail = 0;
         for &body_id in bodies {
+            bar.set_message(move || {
+                format!("mir lowering: {}", full_name(db, body_id, body_id.module(db)))
+            });
+            bar.inc(1);
             if matches!(body_id, DefWithBody::Variant(_)) {
                 continue;
             }
@@ -731,12 +758,11 @@ impl flags::AnalysisStats {
             let name = body_id.name(db).unwrap_or_else(Name::missing);
             let module = body_id.module(db);
             let display_target = module.krate().to_display_target(db);
-            if let Some(only_name) = self.only.as_deref() {
-                if name.display(db, Edition::LATEST).to_string() != only_name
-                    && full_name(db, body_id, module) != only_name
-                {
-                    continue;
-                }
+            if let Some(only_name) = self.only.as_deref()
+                && name.display(db, Edition::LATEST).to_string() != only_name
+                && full_name(db, body_id, module) != only_name
+            {
+                continue;
             }
             let msg = move || {
                 if verbosity.is_verbose() {
@@ -793,7 +819,7 @@ impl flags::AnalysisStats {
             for (expr_id, _) in body.exprs() {
                 let ty = &inference_result[expr_id];
                 num_exprs += 1;
-                let unknown_or_partial = if ty.is_unknown() {
+                let unknown_or_partial = if ty.is_ty_error() {
                     num_exprs_unknown += 1;
                     if verbosity.is_spammy() {
                         if let Some((path, start, end)) = expr_syntax_range(db, vfs, &sm(), expr_id)
@@ -815,8 +841,7 @@ impl flags::AnalysisStats {
                     }
                     true
                 } else {
-                    let is_partially_unknown =
-                        ty.data(Interner).flags.contains(TypeFlags::HAS_ERROR);
+                    let is_partially_unknown = ty.references_non_lt_error();
                     if is_partially_unknown {
                         num_exprs_partially_unknown += 1;
                     }
@@ -898,7 +923,7 @@ impl flags::AnalysisStats {
             for (pat_id, _) in body.pats() {
                 let ty = &inference_result[pat_id];
                 num_pats += 1;
-                let unknown_or_partial = if ty.is_unknown() {
+                let unknown_or_partial = if ty.is_ty_error() {
                     num_pats_unknown += 1;
                     if verbosity.is_spammy() {
                         if let Some((path, start, end)) = pat_syntax_range(db, vfs, &sm(), pat_id) {
@@ -919,8 +944,7 @@ impl flags::AnalysisStats {
                     }
                     true
                 } else {
-                    let is_partially_unknown =
-                        ty.data(Interner).flags.contains(TypeFlags::HAS_ERROR);
+                    let is_partially_unknown = ty.references_non_lt_error();
                     if is_partially_unknown {
                         num_pats_partially_unknown += 1;
                     }
@@ -1085,12 +1109,28 @@ impl flags::AnalysisStats {
         report_metric("body lowering time", body_lowering_time.time.as_millis() as u64, "ms");
     }
 
-    fn run_ide_things(&self, analysis: Analysis, mut file_ids: Vec) {
-        file_ids.sort();
-        file_ids.dedup();
+    /// Invariant: `file_ids` must be sorted and deduped before passing into here
+    fn run_ide_things(
+        &self,
+        analysis: Analysis,
+        file_ids: &[EditionedFileId],
+        db: &RootDatabase,
+        vfs: &Vfs,
+        verbosity: Verbosity,
+    ) {
+        let len = file_ids.len();
+        let create_bar = || match verbosity {
+            Verbosity::Quiet | Verbosity::Spammy => ProgressReport::hidden(),
+            _ if self.parallel || self.output.is_some() => ProgressReport::hidden(),
+            _ => ProgressReport::new(len),
+        };
+
         let mut sw = self.stop_watch();
 
-        for &file_id in &file_ids {
+        let mut bar = create_bar();
+        for &file_id in file_ids {
+            let msg = format!("diagnostics: {}", vfs.file_path(file_id.file_id(db)));
+            bar.set_message(move || msg.clone());
             _ = analysis.full_diagnostics(
                 &DiagnosticsConfig {
                     enabled: true,
@@ -1117,8 +1157,14 @@ impl flags::AnalysisStats {
                 ide::AssistResolveStrategy::All,
                 analysis.editioned_file_id_to_vfs(file_id),
             );
+            bar.inc(1);
         }
-        for &file_id in &file_ids {
+        bar.finish_and_clear();
+
+        let mut bar = create_bar();
+        for &file_id in file_ids {
+            let msg = format!("inlay hints: {}", vfs.file_path(file_id.file_id(db)));
+            bar.set_message(move || msg.clone());
             _ = analysis.inlay_hints(
                 &InlayHintsConfig {
                     render_colons: false,
@@ -1133,6 +1179,7 @@ impl flags::AnalysisStats {
                     },
                     chaining_hints: true,
                     adjustment_hints: ide::AdjustmentHints::Always,
+                    adjustment_hints_disable_reborrows: true,
                     adjustment_hints_mode: ide::AdjustmentHintsMode::Postfix,
                     adjustment_hints_hide_outside_unsafe: false,
                     closure_return_type_hints: ide::ClosureReturnTypeHints::Always,
@@ -1149,31 +1196,40 @@ impl flags::AnalysisStats {
                     closing_brace_hints_min_lines: Some(20),
                     fields_to_resolve: InlayFieldsToResolve::empty(),
                     range_exclusive_hints: true,
+                    minicore: MiniCore::default(),
                 },
                 analysis.editioned_file_id_to_vfs(file_id),
                 None,
             );
+            bar.inc(1);
         }
-        for &file_id in &file_ids {
+        bar.finish_and_clear();
+
+        let mut bar = create_bar();
+        let annotation_config = AnnotationConfig {
+            binary_target: true,
+            annotate_runnables: true,
+            annotate_impls: true,
+            annotate_references: false,
+            annotate_method_references: false,
+            annotate_enum_variant_references: false,
+            location: ide::AnnotationLocation::AboveName,
+            minicore: MiniCore::default(),
+        };
+        for &file_id in file_ids {
+            let msg = format!("annotations: {}", vfs.file_path(file_id.file_id(db)));
+            bar.set_message(move || msg.clone());
             analysis
-                .annotations(
-                    &AnnotationConfig {
-                        binary_target: true,
-                        annotate_runnables: true,
-                        annotate_impls: true,
-                        annotate_references: false,
-                        annotate_method_references: false,
-                        annotate_enum_variant_references: false,
-                        location: ide::AnnotationLocation::AboveName,
-                    },
-                    analysis.editioned_file_id_to_vfs(file_id),
-                )
+                .annotations(&annotation_config, analysis.editioned_file_id_to_vfs(file_id))
                 .unwrap()
                 .into_iter()
                 .for_each(|annotation| {
-                    _ = analysis.resolve_annotation(annotation);
+                    _ = analysis.resolve_annotation(&annotation_config, annotation);
                 });
+            bar.inc(1);
         }
+        bar.finish_and_clear();
+
         let ide_time = sw.elapsed();
         eprintln!("{:<20} {} ({} files)", "IDE:", ide_time, file_ids.len());
     }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
index 7b12cb14009f..82590c8e707f 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/diagnostics.rs
@@ -9,7 +9,7 @@ use ide::{AnalysisHost, AssistResolveStrategy, Diagnostic, DiagnosticsConfig, Se
 use ide_db::{LineIndexDatabase, base_db::SourceDatabase};
 use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace_at};
 
-use crate::cli::flags;
+use crate::cli::{flags, progress_report::ProgressReport};
 
 impl flags::Diagnostics {
     pub fn run(self) -> anyhow::Result<()> {
@@ -50,23 +50,26 @@ impl flags::Diagnostics {
 
         let mut found_error = false;
         let mut visited_files = FxHashSet::default();
+        let min_severity = self.severity.unwrap_or(flags::Severity::Weak);
 
-        let work = all_modules(db).into_iter().filter(|module| {
-            let file_id = module.definition_source_file_id(db).original_file(db);
-            let source_root = db.file_source_root(file_id.file_id(db)).source_root_id(db);
-            let source_root = db.source_root(source_root).source_root(db);
-            !source_root.is_library
-        });
+        let work = all_modules(db)
+            .into_iter()
+            .filter(|module| {
+                let file_id = module.definition_source_file_id(db).original_file(db);
+                let source_root = db.file_source_root(file_id.file_id(db)).source_root_id(db);
+                let source_root = db.source_root(source_root).source_root(db);
+                !source_root.is_library
+            })
+            .collect::>();
 
+        let mut bar = ProgressReport::new(work.len());
         for module in work {
             let file_id = module.definition_source_file_id(db).original_file(db);
             if !visited_files.contains(&file_id) {
+                let message = format!("processing {}", _vfs.file_path(file_id.file_id(db)));
+                bar.set_message(move || message.clone());
                 let crate_name =
                     module.krate().display_name(db).as_deref().unwrap_or(&sym::unknown).to_owned();
-                println!(
-                    "processing crate: {crate_name}, module: {}",
-                    _vfs.file_path(file_id.file_id(db))
-                );
                 for diagnostic in analysis
                     .full_diagnostics(
                         &DiagnosticsConfig::test_sample(),
@@ -75,6 +78,16 @@ impl flags::Diagnostics {
                     )
                     .unwrap()
                 {
+                    let severity = match diagnostic.severity {
+                        Severity::Error => flags::Severity::Error,
+                        Severity::Warning => flags::Severity::Warning,
+                        Severity::WeakWarning => flags::Severity::Weak,
+                        Severity::Allow => continue,
+                    };
+                    if severity < min_severity {
+                        continue;
+                    }
+
                     if matches!(diagnostic.severity, Severity::Error) {
                         found_error = true;
                     }
@@ -83,12 +96,17 @@ impl flags::Diagnostics {
                     let line_index = db.line_index(range.file_id);
                     let start = line_index.line_col(range.range.start());
                     let end = line_index.line_col(range.range.end());
-                    println!("{severity:?} {code:?} from {start:?} to {end:?}: {message}");
+                    bar.println(format!(
+                        "at crate {crate_name}, file {}: {severity:?} {code:?} from {start:?} to {end:?}: {message}",
+                        _vfs.file_path(file_id.file_id(db))
+                    ));
                 }
 
                 visited_files.insert(file_id);
             }
+            bar.inc(1);
         }
+        bar.finish_and_clear();
 
         println!();
         println!("diagnostic scan complete");
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
index 16f351272b69..75030bedfca3 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/flags.rs
@@ -124,6 +124,9 @@ xflags::xflags! {
             optional --disable-proc-macros
             /// Run the proc-macro-srv binary at the specified path.
             optional --proc-macro-srv path: PathBuf
+
+            /// The minimum severity.
+            optional --severity severity: Severity
         }
 
         /// Report unresolved references
@@ -281,6 +284,7 @@ pub struct Diagnostics {
     pub disable_build_scripts: bool,
     pub disable_proc_macros: bool,
     pub proc_macro_srv: Option,
+    pub severity: Option,
 }
 
 #[derive(Debug)]
@@ -376,3 +380,23 @@ impl FromStr for OutputFormat {
         }
     }
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum Severity {
+    Weak,
+    Warning,
+    Error,
+}
+
+impl FromStr for Severity {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result {
+        match &*s.to_ascii_lowercase() {
+            "weak" => Ok(Self::Weak),
+            "warning" => Ok(Self::Warning),
+            "error" => Ok(Self::Error),
+            _ => Err(format!("unknown severity `{s}`")),
+        }
+    }
+}
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/progress_report.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/progress_report.rs
index 1b9b870a7c74..028311388c56 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/progress_report.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/progress_report.rs
@@ -83,11 +83,11 @@ impl<'a> ProgressReport<'a> {
         output.extend(text.chars().skip(common_prefix_length));
 
         // If the new text is shorter than the old one: delete overlapping characters
-        if let Some(overlap_count) = self.text.len().checked_sub(text.len()) {
-            if overlap_count > 0 {
-                output += &" ".repeat(overlap_count);
-                output += &"\x08".repeat(overlap_count);
-            }
+        if let Some(overlap_count) = self.text.len().checked_sub(text.len())
+            && overlap_count > 0
+        {
+            output += &" ".repeat(overlap_count);
+            output += &"\x08".repeat(overlap_count);
         }
 
         let _ = io::stdout().write(output.as_bytes());
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
index 30ac93fb6f83..20567149bb4b 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/rustc_tests.rs
@@ -11,7 +11,7 @@ use ide_db::base_db;
 use itertools::Either;
 use paths::Utf8PathBuf;
 use profile::StopWatch;
-use project_model::toolchain_info::{QueryConfig, target_data_layout};
+use project_model::toolchain_info::{QueryConfig, target_data};
 use project_model::{
     CargoConfig, ManifestPath, ProjectWorkspace, ProjectWorkspaceKind, RustLibSource,
     RustSourceWorkspaceConfig, Sysroot,
@@ -19,7 +19,6 @@ use project_model::{
 
 use load_cargo::{LoadCargoConfig, ProcMacroServerChoice, load_workspace};
 use rustc_hash::FxHashMap;
-use triomphe::Arc;
 use vfs::{AbsPathBuf, FileId};
 use walkdir::WalkDir;
 
@@ -79,7 +78,6 @@ impl Tester {
         let loaded_sysroot = sysroot.load_workspace(
             &RustSourceWorkspaceConfig::default_cargo(),
             false,
-            &path,
             &Utf8PathBuf::default(),
             &|_| (),
         );
@@ -87,7 +85,7 @@ impl Tester {
             sysroot.set_workspace(loaded_sysroot);
         }
 
-        let data_layout = target_data_layout::get(
+        let target_data = target_data::get(
             QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
             None,
             &cargo_config.extra_env,
@@ -101,7 +99,7 @@ impl Tester {
             sysroot,
             rustc_cfg: vec![],
             toolchain: None,
-            target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
+            target: target_data.map_err(|it| it.to_string().into()),
             cfg_overrides: Default::default(),
             extra_includes: vec![],
             set_test: true,
@@ -305,10 +303,10 @@ impl flags::RustcTests {
         for i in walk_dir {
             let i = i?;
             let p = i.into_path();
-            if let Some(f) = &self.filter {
-                if !p.as_os_str().to_string_lossy().contains(f) {
-                    continue;
-                }
+            if let Some(f) = &self.filter
+                && !p.as_os_str().to_string_lossy().contains(f)
+            {
+                continue;
             }
             if p.extension().is_none_or(|x| x != "rs") {
                 continue;
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/symbols.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/symbols.rs
index 9fad6723afcd..d7af56d3e15b 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/symbols.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/cli/symbols.rs
@@ -1,5 +1,5 @@
 //! Read Rust code on stdin, print syntax tree on stdout.
-use ide::Analysis;
+use ide::{Analysis, FileStructureConfig};
 
 use crate::cli::{flags, read_stdin};
 
@@ -7,7 +7,12 @@ impl flags::Symbols {
     pub fn run(self) -> anyhow::Result<()> {
         let text = read_stdin()?;
         let (analysis, file_id) = Analysis::from_single_file(text);
-        let structure = analysis.file_structure(file_id).unwrap();
+        let structure = analysis
+            // The default setting in config.rs (document_symbol_search_excludeLocals) is to exclude
+            // locals because it is unlikely that users want document search to return the names of
+            // local variables, but here we include them deliberately.
+            .file_structure(&FileStructureConfig { exclude_locals: false }, file_id)
+            .unwrap();
         for s in structure {
             println!("{s:?}");
         }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/command.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/command.rs
index d6c80c399ba2..674e8623b2d0 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/command.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/command.rs
@@ -3,13 +3,15 @@
 
 use std::{
     ffi::OsString,
-    fmt, io,
+    fmt,
+    io::{self, BufWriter, Write},
     marker::PhantomData,
     path::PathBuf,
     process::{ChildStderr, ChildStdout, Command, Stdio},
 };
 
 use crossbeam_channel::Sender;
+use paths::Utf8PathBuf;
 use process_wrap::std::{StdChildWrapper, StdCommandWrap};
 use stdx::process::streaming_output;
 
@@ -40,7 +42,7 @@ impl CargoActor {
 }
 
 impl CargoActor {
-    fn run(self) -> io::Result<(bool, String)> {
+    fn run(self, outfile: Option) -> io::Result<(bool, String)> {
         // We manually read a line at a time, instead of using serde's
         // stream deserializers, because the deserializer cannot recover
         // from an error, resulting in it getting stuck, because we try to
@@ -50,6 +52,15 @@ impl CargoActor {
         // simply skip a line if it doesn't parse, which just ignores any
         // erroneous output.
 
+        let mut stdout = outfile.as_ref().and_then(|path| {
+            _ = std::fs::create_dir_all(path);
+            Some(BufWriter::new(std::fs::File::create(path.join("stdout")).ok()?))
+        });
+        let mut stderr = outfile.as_ref().and_then(|path| {
+            _ = std::fs::create_dir_all(path);
+            Some(BufWriter::new(std::fs::File::create(path.join("stderr")).ok()?))
+        });
+
         let mut stdout_errors = String::new();
         let mut stderr_errors = String::new();
         let mut read_at_least_one_stdout_message = false;
@@ -67,11 +78,19 @@ impl CargoActor {
             self.stdout,
             self.stderr,
             &mut |line| {
+                if let Some(stdout) = &mut stdout {
+                    _ = stdout.write_all(line.as_bytes());
+                    _ = stdout.write_all(b"\n");
+                }
                 if process_line(line, &mut stdout_errors) {
                     read_at_least_one_stdout_message = true;
                 }
             },
             &mut |line| {
+                if let Some(stderr) = &mut stderr {
+                    _ = stderr.write_all(line.as_bytes());
+                    _ = stderr.write_all(b"\n");
+                }
                 if process_line(line, &mut stderr_errors) {
                     read_at_least_one_stderr_message = true;
                 }
@@ -130,6 +149,7 @@ impl CommandHandle {
         mut command: Command,
         parser: impl CargoParser,
         sender: Sender,
+        out_file: Option,
     ) -> std::io::Result {
         command.stdout(Stdio::piped()).stderr(Stdio::piped()).stdin(Stdio::null());
 
@@ -150,7 +170,7 @@ impl CommandHandle {
         let actor = CargoActor::::new(parser, sender, stdout, stderr);
         let thread =
             stdx::thread::Builder::new(stdx::thread::ThreadIntent::Worker, "CommandHandle")
-                .spawn(move || actor.run())
+                .spawn(move || actor.run(out_file))
                 .expect("failed to spawn thread");
         Ok(CommandHandle { program, arguments, current_dir, child, thread, _phantom: PhantomData })
     }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
index 9456fd8809b4..652c2e32ffa6 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config.rs
@@ -8,14 +8,14 @@ use std::{env, fmt, iter, ops::Not, sync::OnceLock};
 use cfg::{CfgAtom, CfgDiff};
 use hir::Symbol;
 use ide::{
-    AssistConfig, CallHierarchyConfig, CallableSnippets, CompletionConfig,
-    CompletionFieldsToResolve, DiagnosticsConfig, GenericParameterHints, HighlightConfig,
-    HighlightRelatedConfig, HoverConfig, HoverDocFormat, InlayFieldsToResolve, InlayHintsConfig,
-    JoinLinesConfig, MemoryLayoutHoverConfig, MemoryLayoutHoverRenderKind, Snippet, SnippetScope,
-    SourceRootId,
+    AnnotationConfig, AssistConfig, CallHierarchyConfig, CallableSnippets, CompletionConfig,
+    CompletionFieldsToResolve, DiagnosticsConfig, GenericParameterHints, GotoDefinitionConfig,
+    HighlightConfig, HighlightRelatedConfig, HoverConfig, HoverDocFormat, InlayFieldsToResolve,
+    InlayHintsConfig, JoinLinesConfig, MemoryLayoutHoverConfig, MemoryLayoutHoverRenderKind,
+    Snippet, SnippetScope, SourceRootId,
 };
 use ide_db::{
-    SnippetCap,
+    MiniCore, SnippetCap,
     assists::ExprFillDefaultMode,
     imports::insert_use::{ImportGranularity, InsertUseConfig, PrefixKind},
 };
@@ -61,233 +61,322 @@ pub enum MaxSubstitutionLength {
     Limit(usize),
 }
 
-// Defines the server-side configuration of the rust-analyzer. We generate
-// *parts* of VS Code's `package.json` config from this. Run `cargo test` to
-// re-generate that file.
+// Defines the server-side configuration of the rust-analyzer. We generate *parts* of VS Code's
+// `package.json` config from this. Run `cargo test` to re-generate that file.
 //
-// However, editor specific config, which the server doesn't know about, should
-// be specified directly in `package.json`.
+// However, editor specific config, which the server doesn't know about, should be specified
+// directly in `package.json`.
 //
-// To deprecate an option by replacing it with another name use `new_name` | `old_name` so that we keep
-// parsing the old name.
+// To deprecate an option by replacing it with another name use `new_name` | `old_name` so that we
+// keep parsing the old name.
 config_data! {
-    /// Configs that apply on a workspace-wide scope. There are 2 levels on which a global configuration can be configured
+    /// Configs that apply on a workspace-wide scope. There are 2 levels on which a global
+    /// configuration can be configured
     ///
-    /// 1. `rust-analyzer.toml` file under user's config directory (e.g ~/.config/rust-analyzer/rust-analyzer.toml)
+    /// 1. `rust-analyzer.toml` file under user's config directory (e.g
+    ///    ~/.config/rust-analyzer/rust-analyzer.toml)
     /// 2. Client's own configurations (e.g `settings.json` on VS Code)
     ///
-    /// A config is searched for by traversing a "config tree" in a bottom up fashion. It is chosen by the nearest first principle.
+    /// A config is searched for by traversing a "config tree" in a bottom up fashion. It is chosen
+    /// by the nearest first principle.
     global: struct GlobalDefaultConfigData <- GlobalConfigInput -> {
         /// Warm up caches on project load.
         cachePriming_enable: bool = true,
-        /// How many worker threads to handle priming caches. The default `0` means to pick automatically.
+
+        /// How many worker threads to handle priming caches. The default `0` means to pick
+        /// automatically.
         cachePriming_numThreads: NumThreads = NumThreads::Physical,
 
         /// Custom completion snippets.
-        completion_snippets_custom: FxIndexMap = Config::completion_snippets_default(),
+        completion_snippets_custom: FxIndexMap =
+            Config::completion_snippets_default(),
 
-
-        /// These paths (file/directories) will be ignored by rust-analyzer. They are
-        /// relative to the workspace root, and globs are not supported. You may
-        /// also need to add the folders to Code's `files.watcherExclude`.
+        /// List of files to ignore
+        ///
+        /// These paths (file/directories) will be ignored by rust-analyzer. They are relative to
+        /// the workspace root, and globs are not supported. You may also need to add the folders to
+        /// Code's `files.watcherExclude`.
         files_exclude | files_excludeDirs: Vec = vec![],
 
-
-
-        /// Enables highlighting of related return values while the cursor is on any `match`, `if`, or match arm arrow (`=>`).
+        /// Highlight related return values while the cursor is on any `match`, `if`, or match arm
+        /// arrow (`=>`).
         highlightRelated_branchExitPoints_enable: bool = true,
-        /// Enables highlighting of related references while the cursor is on `break`, `loop`, `while`, or `for` keywords.
+
+        /// Highlight related references while the cursor is on `break`, `loop`, `while`, or `for`
+        /// keywords.
         highlightRelated_breakPoints_enable: bool = true,
-        /// Enables highlighting of all captures of a closure while the cursor is on the `|` or move keyword of a closure.
+
+        /// Highlight all captures of a closure while the cursor is on the `|` or move keyword of a closure.
         highlightRelated_closureCaptures_enable: bool = true,
-        /// Enables highlighting of all exit points while the cursor is on any `return`, `?`, `fn`, or return type arrow (`->`).
+
+        /// Highlight all exit points while the cursor is on any `return`, `?`, `fn`, or return type
+        /// arrow (`->`).
         highlightRelated_exitPoints_enable: bool = true,
-        /// Enables highlighting of related references while the cursor is on any identifier.
+
+        /// Highlight related references while the cursor is on any identifier.
         highlightRelated_references_enable: bool = true,
-        /// Enables highlighting of all break points for a loop or block context while the cursor is on any `async` or `await` keywords.
+
+        /// Highlight all break points for a loop or block context while the cursor is on any
+        /// `async` or `await` keywords.
         highlightRelated_yieldPoints_enable: bool = true,
 
-        /// Whether to show `Debug` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` is set.
-        hover_actions_debug_enable: bool           = true,
-        /// Whether to show HoverActions in Rust files.
-        hover_actions_enable: bool          = true,
-        /// Whether to show `Go to Type Definition` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` is set.
-        hover_actions_gotoTypeDef_enable: bool     = true,
-        /// Whether to show `Implementations` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` is set.
-        hover_actions_implementations_enable: bool = true,
-        /// Whether to show `References` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` is set.
-        hover_actions_references_enable: bool      = false,
-        /// Whether to show `Run` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` is set.
-        hover_actions_run_enable: bool             = true,
-        /// Whether to show `Update Test` action. Only applies when
-        /// `#rust-analyzer.hover.actions.enable#` and `#rust-analyzer.hover.actions.run.enable#` are set.
-        hover_actions_updateTest_enable: bool     = true,
+        /// Show `Debug` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.
+        hover_actions_debug_enable: bool = true,
 
-        /// Whether to show documentation on hover.
-        hover_documentation_enable: bool           = true,
-        /// Whether to show keyword hover popups. Only applies when
+        /// Show HoverActions in Rust files.
+        hover_actions_enable: bool = true,
+
+        /// Show `Go to Type Definition` action. Only applies when
+        /// `#rust-analyzer.hover.actions.enable#` is set.
+        hover_actions_gotoTypeDef_enable: bool = true,
+
+        /// Show `Implementations` action. Only applies when `#rust-analyzer.hover.actions.enable#`
+        /// is set.
+        hover_actions_implementations_enable: bool = true,
+
+        /// Show `References` action. Only applies when `#rust-analyzer.hover.actions.enable#` is
+        /// set.
+        hover_actions_references_enable: bool = false,
+
+        /// Show `Run` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.
+        hover_actions_run_enable: bool = true,
+
+        /// Show `Update Test` action. Only applies when `#rust-analyzer.hover.actions.enable#` and
+        /// `#rust-analyzer.hover.actions.run.enable#` are set.
+        hover_actions_updateTest_enable: bool = true,
+
+        /// Show documentation on hover.
+        hover_documentation_enable: bool = true,
+
+        /// Show keyword hover popups. Only applies when
         /// `#rust-analyzer.hover.documentation.enable#` is set.
-        hover_documentation_keywords_enable: bool  = true,
-        /// Whether to show drop glue information on hover.
-        hover_dropGlue_enable: bool                = true,
+        hover_documentation_keywords_enable: bool = true,
+
+        /// Show drop glue information on hover.
+        hover_dropGlue_enable: bool = true,
+
         /// Use markdown syntax for links on hover.
         hover_links_enable: bool = true,
-        /// Whether to show what types are used as generic arguments in calls etc. on hover, and what is their max length to show such types, beyond it they will be shown with ellipsis.
+
+        /// Show what types are used as generic arguments in calls etc. on hover, and limit the max
+        /// length to show such types, beyond which they will be shown with ellipsis.
         ///
-        /// This can take three values: `null` means "unlimited", the string `"hide"` means to not show generic substitutions at all, and a number means to limit them to X characters.
+        /// This can take three values: `null` means "unlimited", the string `"hide"` means to not
+        /// show generic substitutions at all, and a number means to limit them to X characters.
         ///
         /// The default is 20 characters.
-        hover_maxSubstitutionLength: Option = Some(MaxSubstitutionLength::Limit(20)),
+        hover_maxSubstitutionLength: Option =
+            Some(MaxSubstitutionLength::Limit(20)),
+
         /// How to render the align information in a memory layout hover.
-        hover_memoryLayout_alignment: Option = Some(MemoryLayoutHoverRenderKindDef::Hexadecimal),
-        /// Whether to show memory layout data on hover.
+        hover_memoryLayout_alignment: Option =
+            Some(MemoryLayoutHoverRenderKindDef::Hexadecimal),
+
+        /// Show memory layout data on hover.
         hover_memoryLayout_enable: bool = true,
+
         /// How to render the niche information in a memory layout hover.
         hover_memoryLayout_niches: Option = Some(false),
+
         /// How to render the offset information in a memory layout hover.
-        hover_memoryLayout_offset: Option = Some(MemoryLayoutHoverRenderKindDef::Hexadecimal),
+        hover_memoryLayout_offset: Option =
+            Some(MemoryLayoutHoverRenderKindDef::Hexadecimal),
+
         /// How to render the padding information in a memory layout hover.
         hover_memoryLayout_padding: Option = None,
+
         /// How to render the size information in a memory layout hover.
-        hover_memoryLayout_size: Option = Some(MemoryLayoutHoverRenderKindDef::Both),
+        hover_memoryLayout_size: Option =
+            Some(MemoryLayoutHoverRenderKindDef::Both),
 
         /// How many variants of an enum to display when hovering on. Show none if empty.
         hover_show_enumVariants: Option = Some(5),
-        /// How many fields of a struct, variant or union to display when hovering on. Show none if empty.
+
+        /// How many fields of a struct, variant or union to display when hovering on. Show none if
+        /// empty.
         hover_show_fields: Option = Some(5),
+
         /// How many associated items of a trait to display when hovering a trait.
         hover_show_traitAssocItems: Option = None,
 
-        /// Whether to show inlay type hints for binding modes.
-        inlayHints_bindingModeHints_enable: bool                   = false,
-        /// Whether to show inlay type hints for method chains.
-        inlayHints_chainingHints_enable: bool                      = true,
-        /// Whether to show inlay hints after a closing `}` to indicate what item it belongs to.
-        inlayHints_closingBraceHints_enable: bool                  = true,
+        /// Show inlay type hints for binding modes.
+        inlayHints_bindingModeHints_enable: bool = false,
+
+        /// Show inlay type hints for method chains.
+        inlayHints_chainingHints_enable: bool = true,
+
+        /// Show inlay hints after a closing `}` to indicate what item it belongs to.
+        inlayHints_closingBraceHints_enable: bool = true,
+
         /// Minimum number of lines required before the `}` until the hint is shown (set to 0 or 1
         /// to always show them).
-        inlayHints_closingBraceHints_minLines: usize               = 25,
-        /// Whether to show inlay hints for closure captures.
-        inlayHints_closureCaptureHints_enable: bool                          = false,
-        /// Whether to show inlay type hints for return types of closures.
-        inlayHints_closureReturnTypeHints_enable: ClosureReturnTypeHintsDef  = ClosureReturnTypeHintsDef::Never,
-        /// Closure notation in type and chaining inlay hints.
-        inlayHints_closureStyle: ClosureStyle                                = ClosureStyle::ImplFn,
-        /// Whether to show enum variant discriminant hints.
-        inlayHints_discriminantHints_enable: DiscriminantHintsDef            = DiscriminantHintsDef::Never,
-        /// Whether to show inlay hints for type adjustments.
-        inlayHints_expressionAdjustmentHints_enable: AdjustmentHintsDef = AdjustmentHintsDef::Never,
-        /// Whether to hide inlay hints for type adjustments outside of `unsafe` blocks.
-        inlayHints_expressionAdjustmentHints_hideOutsideUnsafe: bool = false,
-        /// Whether to show inlay hints as postfix ops (`.*` instead of `*`, etc).
-        inlayHints_expressionAdjustmentHints_mode: AdjustmentHintsModeDef = AdjustmentHintsModeDef::Prefix,
-        /// Whether to show const generic parameter name inlay hints.
-        inlayHints_genericParameterHints_const_enable: bool= true,
-        /// Whether to show generic lifetime parameter name inlay hints.
-        inlayHints_genericParameterHints_lifetime_enable: bool = false,
-        /// Whether to show generic type parameter name inlay hints.
-        inlayHints_genericParameterHints_type_enable: bool = false,
-        /// Whether to show implicit drop hints.
-        inlayHints_implicitDrops_enable: bool                      = false,
-        /// Whether to show inlay hints for the implied type parameter `Sized` bound.
-        inlayHints_implicitSizedBoundHints_enable: bool            = false,
-        /// Whether to show inlay type hints for elided lifetimes in function signatures.
-        inlayHints_lifetimeElisionHints_enable: LifetimeElisionDef = LifetimeElisionDef::Never,
-        /// Whether to prefer using parameter names as the name for elided lifetime hints if possible.
-        inlayHints_lifetimeElisionHints_useParameterNames: bool    = false,
-        /// Maximum length for inlay hints. Set to null to have an unlimited length.
-        inlayHints_maxLength: Option                        = Some(25),
-        /// Whether to show function parameter name inlay hints at the call
-        /// site.
-        inlayHints_parameterHints_enable: bool                     = true,
-        /// Whether to show exclusive range inlay hints.
-        inlayHints_rangeExclusiveHints_enable: bool                = false,
-        /// Whether to show inlay hints for compiler inserted reborrows.
-        /// This setting is deprecated in favor of #rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.
-        inlayHints_reborrowHints_enable: ReborrowHintsDef          = ReborrowHintsDef::Never,
-        /// Whether to render leading colons for type hints, and trailing colons for parameter hints.
-        inlayHints_renderColons: bool                              = true,
-        /// Whether to show inlay type hints for variables.
-        inlayHints_typeHints_enable: bool                          = true,
-        /// Whether to hide inlay type hints for `let` statements that initialize to a closure.
-        /// Only applies to closures with blocks, same as `#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.
-        inlayHints_typeHints_hideClosureInitialization: bool       = false,
-        /// Whether to hide inlay parameter type hints for closures.
-        inlayHints_typeHints_hideClosureParameter:bool             = false,
-        /// Whether to hide inlay type hints for constructors.
-        inlayHints_typeHints_hideNamedConstructor: bool            = false,
+        inlayHints_closingBraceHints_minLines: usize = 25,
 
-        /// Enables the experimental support for interpreting tests.
+        /// Show inlay hints for closure captures.
+        inlayHints_closureCaptureHints_enable: bool = false,
+
+        /// Show inlay type hints for return types of closures.
+        inlayHints_closureReturnTypeHints_enable: ClosureReturnTypeHintsDef =
+            ClosureReturnTypeHintsDef::Never,
+
+        /// Closure notation in type and chaining inlay hints.
+        inlayHints_closureStyle: ClosureStyle = ClosureStyle::ImplFn,
+
+        /// Show enum variant discriminant hints.
+        inlayHints_discriminantHints_enable: DiscriminantHintsDef =
+            DiscriminantHintsDef::Never,
+
+        /// Disable reborrows in expression adjustments inlay hints.
+        ///
+        /// Reborrows are a pair of a builtin deref then borrow, i.e. `&*`. They are inserted by the compiler but are mostly useless to the programmer.
+        ///
+        /// Note: if the deref is not builtin (an overloaded deref), or the borrow is `&raw const`/`&raw mut`, they are not removed.
+        inlayHints_expressionAdjustmentHints_disableReborrows: bool =
+            true,
+
+        /// Show inlay hints for type adjustments.
+        inlayHints_expressionAdjustmentHints_enable: AdjustmentHintsDef =
+            AdjustmentHintsDef::Never,
+
+        /// Hide inlay hints for type adjustments outside of `unsafe` blocks.
+        inlayHints_expressionAdjustmentHints_hideOutsideUnsafe: bool = false,
+
+        /// Show inlay hints as postfix ops (`.*` instead of `*`, etc).
+        inlayHints_expressionAdjustmentHints_mode: AdjustmentHintsModeDef =
+            AdjustmentHintsModeDef::Prefix,
+
+        /// Show const generic parameter name inlay hints.
+        inlayHints_genericParameterHints_const_enable: bool = true,
+
+        /// Show generic lifetime parameter name inlay hints.
+        inlayHints_genericParameterHints_lifetime_enable: bool = false,
+
+        /// Show generic type parameter name inlay hints.
+        inlayHints_genericParameterHints_type_enable: bool = false,
+
+        /// Show implicit drop hints.
+        inlayHints_implicitDrops_enable: bool = false,
+
+        /// Show inlay hints for the implied type parameter `Sized` bound.
+        inlayHints_implicitSizedBoundHints_enable: bool = false,
+
+        /// Show inlay type hints for elided lifetimes in function signatures.
+        inlayHints_lifetimeElisionHints_enable: LifetimeElisionDef = LifetimeElisionDef::Never,
+
+        /// Prefer using parameter names as the name for elided lifetime hints if possible.
+        inlayHints_lifetimeElisionHints_useParameterNames: bool = false,
+
+        /// Maximum length for inlay hints. Set to null to have an unlimited length.
+        ///
+        /// **Note:** This is mostly a hint, and we don't guarantee to strictly follow the limit.
+        inlayHints_maxLength: Option = Some(25),
+
+        /// Show function parameter name inlay hints at the call site.
+        inlayHints_parameterHints_enable: bool = true,
+
+        /// Show exclusive range inlay hints.
+        inlayHints_rangeExclusiveHints_enable: bool = false,
+
+        /// Show inlay hints for compiler inserted reborrows.
+        ///
+        /// This setting is deprecated in favor of
+        /// #rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.
+        inlayHints_reborrowHints_enable: ReborrowHintsDef = ReborrowHintsDef::Never,
+
+        /// Whether to render leading colons for type hints, and trailing colons for parameter hints.
+        inlayHints_renderColons: bool = true,
+
+        /// Show inlay type hints for variables.
+        inlayHints_typeHints_enable: bool = true,
+
+        /// Hide inlay type hints for `let` statements that initialize to a closure.
+        ///
+        /// Only applies to closures with blocks, same as
+        /// `#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.
+        inlayHints_typeHints_hideClosureInitialization: bool = false,
+
+        /// Hide inlay parameter type hints for closures.
+        inlayHints_typeHints_hideClosureParameter: bool = false,
+
+        /// Hide inlay type hints for constructors.
+        inlayHints_typeHints_hideNamedConstructor: bool = false,
+
+        /// Enable the experimental support for interpreting tests.
         interpret_tests: bool = false,
 
         /// Join lines merges consecutive declaration and initialization of an assignment.
         joinLines_joinAssignments: bool = true,
+
         /// Join lines inserts else between consecutive ifs.
         joinLines_joinElseIf: bool = true,
+
         /// Join lines removes trailing commas.
         joinLines_removeTrailingComma: bool = true,
+
         /// Join lines unwraps trivial blocks.
         joinLines_unwrapTrivialBlock: bool = true,
 
-        /// Whether to show `Debug` lens. Only applies when
-        /// `#rust-analyzer.lens.enable#` is set.
-        lens_debug_enable: bool            = true,
-        /// Whether to show CodeLens in Rust files.
-        lens_enable: bool           = true,
-        /// Whether to show `Implementations` lens. Only applies when
-        /// `#rust-analyzer.lens.enable#` is set.
-        lens_implementations_enable: bool  = true,
+        /// Show `Debug` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
+        lens_debug_enable: bool = true,
+
+        /// Show CodeLens in Rust files.
+        lens_enable: bool = true,
+
+        /// Show `Implementations` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
+        lens_implementations_enable: bool = true,
+
         /// Where to render annotations.
         lens_location: AnnotationLocation = AnnotationLocation::AboveName,
-        /// Whether to show `References` lens for Struct, Enum, and Union.
-        /// Only applies when `#rust-analyzer.lens.enable#` is set.
+
+        /// Show `References` lens for Struct, Enum, and Union. Only applies when
+        /// `#rust-analyzer.lens.enable#` is set.
         lens_references_adt_enable: bool = false,
-        /// Whether to show `References` lens for Enum Variants.
-        /// Only applies when `#rust-analyzer.lens.enable#` is set.
+
+        /// Show `References` lens for Enum Variants. Only applies when
+        /// `#rust-analyzer.lens.enable#` is set.
         lens_references_enumVariant_enable: bool = false,
-        /// Whether to show `Method References` lens. Only applies when
-        /// `#rust-analyzer.lens.enable#` is set.
+
+        /// Show `Method References` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
         lens_references_method_enable: bool = false,
-        /// Whether to show `References` lens for Trait.
-        /// Only applies when `#rust-analyzer.lens.enable#` is set.
+
+        /// Show `References` lens for Trait. Only applies when `#rust-analyzer.lens.enable#` is
+        /// set.
         lens_references_trait_enable: bool = false,
-        /// Whether to show `Run` lens. Only applies when
-        /// `#rust-analyzer.lens.enable#` is set.
-        lens_run_enable: bool              = true,
-        /// Whether to show `Update Test` lens. Only applies when
-        /// `#rust-analyzer.lens.enable#` and `#rust-analyzer.lens.run.enable#` are set.
+
+        /// Show `Run` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
+        lens_run_enable: bool = true,
+
+        /// Show `Update Test` lens. Only applies when `#rust-analyzer.lens.enable#` and
+        /// `#rust-analyzer.lens.run.enable#` are set.
         lens_updateTest_enable: bool = true,
 
-        /// Disable project auto-discovery in favor of explicitly specified set
-        /// of projects.
+        /// Disable project auto-discovery in favor of explicitly specified set of projects.
         ///
-        /// Elements must be paths pointing to `Cargo.toml`,
-        /// `rust-project.json`, `.rs` files (which will be treated as standalone files) or JSON
-        /// objects in `rust-project.json` format.
+        /// Elements must be paths pointing to `Cargo.toml`, `rust-project.json`, `.rs` files (which
+        /// will be treated as standalone files) or JSON objects in `rust-project.json` format.
         linkedProjects: Vec = vec![],
 
         /// Number of syntax trees rust-analyzer keeps in memory. Defaults to 128.
-        lru_capacity: Option                 = None,
-        /// Sets the LRU capacity of the specified queries.
+        lru_capacity: Option = None,
+
+        /// The LRU capacity of the specified queries.
         lru_query_capacities: FxHashMap, u16> = FxHashMap::default(),
 
-        /// Whether to show `can't find Cargo.toml` error message.
-        notifications_cargoTomlNotFound: bool      = true,
+        /// Show `can't find Cargo.toml` error message.
+        notifications_cargoTomlNotFound: bool = true,
 
-        /// How many worker threads in the main loop. The default `null` means to pick automatically.
+        /// The number of worker threads in the main loop. The default `null` means to pick
+        /// automatically.
         numThreads: Option = None,
 
         /// Expand attribute macros. Requires `#rust-analyzer.procMacro.enable#` to be set.
         procMacro_attributes_enable: bool = true,
+
         /// Enable support for procedural macros, implies `#rust-analyzer.cargo.buildScripts.enable#`.
-        procMacro_enable: bool                     = true,
+        procMacro_enable: bool = true,
+
         /// Internal config, path to proc-macro server executable.
-        procMacro_server: Option          = None,
+        procMacro_server: Option = None,
 
         /// Exclude imports from find-all-references.
         references_excludeImports: bool = false,
@@ -295,36 +384,53 @@ config_data! {
         /// Exclude tests from find-all-references and call-hierarchy.
         references_excludeTests: bool = false,
 
+        /// Use semantic tokens for comments.
+        ///
+        /// In some editors (e.g. vscode) semantic tokens override other highlighting grammars.
+        /// By disabling semantic tokens for comments, other grammars can be used to highlight
+        /// their contents.
+        semanticHighlighting_comments_enable: bool = true,
+
         /// Inject additional highlighting into doc comments.
         ///
         /// When enabled, rust-analyzer will highlight rust source in doc comments as well as intra
         /// doc links.
         semanticHighlighting_doc_comment_inject_enable: bool = true,
-        /// Whether the server is allowed to emit non-standard tokens and modifiers.
+
+        /// Emit non-standard tokens and modifiers
+        ///
+        /// When enabled, rust-analyzer will emit tokens and modifiers that are not part of the
+        /// standard set of semantic tokens.
         semanticHighlighting_nonStandardTokens: bool = true,
+
         /// Use semantic tokens for operators.
         ///
         /// When disabled, rust-analyzer will emit semantic tokens only for operator tokens when
         /// they are tagged with modifiers.
         semanticHighlighting_operator_enable: bool = true,
+
         /// Use specialized semantic tokens for operators.
         ///
         /// When enabled, rust-analyzer will emit special token types for operator tokens instead
         /// of the generic `operator` token type.
         semanticHighlighting_operator_specialization_enable: bool = false,
+
         /// Use semantic tokens for punctuation.
         ///
         /// When disabled, rust-analyzer will emit semantic tokens only for punctuation tokens when
         /// they are tagged with modifiers or have a special role.
         semanticHighlighting_punctuation_enable: bool = false,
+
         /// When enabled, rust-analyzer will emit a punctuation semantic token for the `!` of macro
         /// calls.
         semanticHighlighting_punctuation_separate_macro_bang: bool = false,
+
         /// Use specialized semantic tokens for punctuation.
         ///
         /// When enabled, rust-analyzer will emit special token types for punctuation tokens instead
         /// of the generic `punctuation` token type.
         semanticHighlighting_punctuation_specialization_enable: bool = false,
+
         /// Use semantic tokens for strings.
         ///
         /// In some editors (e.g. vscode) semantic tokens override other highlighting grammars.
@@ -333,16 +439,21 @@ config_data! {
         semanticHighlighting_strings_enable: bool = true,
 
         /// Show full signature of the callable. Only shows parameters if disabled.
-        signatureInfo_detail: SignatureDetail                           = SignatureDetail::Full,
+        signatureInfo_detail: SignatureDetail = SignatureDetail::Full,
+
         /// Show documentation.
-        signatureInfo_documentation_enable: bool                       = true,
+        signatureInfo_documentation_enable: bool = true,
 
         /// Specify the characters allowed to invoke special on typing triggers.
-        /// - typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing expression
+        ///
+        /// - typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing
+        ///   expression
         /// - typing `=` between two expressions adds `;` when in statement position
-        /// - typing `=` to turn an assignment into an equality comparison removes `;` when in expression position
+        /// - typing `=` to turn an assignment into an equality comparison removes `;` when in
+        ///   expression position
         /// - typing `.` in a chain method call auto-indents
-        /// - typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the expression
+        /// - typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the
+        ///   expression
         /// - typing `{` in a use item adds a closing `}` in the right place
         /// - typing `>` to complete a return type `->` will insert a whitespace after it
         /// - typing `<` in a path or type position inserts a closing `>` after the path or type.
@@ -374,8 +485,8 @@ config_data! {
         ///
         /// **Warning**: This format is provisional and subject to change.
         ///
-        /// [`DiscoverWorkspaceConfig::command`] *must* return a JSON object
-        /// corresponding to `DiscoverProjectData::Finished`:
+        /// [`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to
+        /// `DiscoverProjectData::Finished`:
         ///
         /// ```norun
         /// #[derive(Debug, Clone, Deserialize, Serialize)]
@@ -405,12 +516,11 @@ config_data! {
         /// }
         /// ```
         ///
-        /// It is encouraged, but not required, to use the other variants on
-        /// `DiscoverProjectData` to provide a more polished end-user experience.
+        /// It is encouraged, but not required, to use the other variants on `DiscoverProjectData`
+        /// to provide a more polished end-user experience.
         ///
-        /// `DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`,
-        /// which will be substituted with the JSON-serialized form of the following
-        /// enum:
+        /// `DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be
+        /// substituted with the JSON-serialized form of the following enum:
         ///
         /// ```norun
         /// #[derive(PartialEq, Clone, Debug, Serialize)]
@@ -437,11 +547,10 @@ config_data! {
         /// }
         /// ```
         ///
-        /// `DiscoverArgument::Path` is used to find and generate a `rust-project.json`,
-        /// and therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to
-        /// to update an existing workspace. As a reference for implementors,
-        /// buck2's `rust-project` will likely be useful:
-        /// https://github.com/facebook/buck2/tree/main/integrations/rust-project.
+        /// `DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and
+        /// therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an
+        /// existing workspace. As a reference for implementors, buck2's `rust-project` will likely
+        /// be useful: https://github.com/facebook/buck2/tree/main/integrations/rust-project.
         workspace_discoverConfig: Option = None,
     }
 }
@@ -449,109 +558,154 @@ config_data! {
 config_data! {
     /// Local configurations can be defined per `SourceRoot`. This almost always corresponds to a `Crate`.
     local: struct LocalDefaultConfigData <- LocalConfigInput ->  {
-        /// Whether to insert #[must_use] when generating `as_` methods
-        /// for enum variants.
-        assist_emitMustUse: bool               = false,
+        /// Insert #[must_use] when generating `as_` methods for enum variants.
+        assist_emitMustUse: bool = false,
+
         /// Placeholder expression to use for missing expressions in assists.
-        assist_expressionFillDefault: ExprFillDefaultDef              = ExprFillDefaultDef::Todo,
-        /// When inserting a type (e.g. in "fill match arms" assist), prefer to use `Self` over the type name where possible.
+        assist_expressionFillDefault: ExprFillDefaultDef = ExprFillDefaultDef::Todo,
+
+        /// Prefer to use `Self` over the type name when inserting a type (e.g. in "fill match arms" assist).
         assist_preferSelf: bool = false,
-        /// Enable borrow checking for term search code assists. If set to false, also there will be more suggestions, but some of them may not borrow-check.
+
+        /// Enable borrow checking for term search code assists. If set to false, also there will be
+        /// more suggestions, but some of them may not borrow-check.
         assist_termSearch_borrowcheck: bool = true,
+
         /// Term search fuel in "units of work" for assists (Defaults to 1800).
         assist_termSearch_fuel: usize = 1800,
 
-
-        /// Whether to automatically add a semicolon when completing unit-returning functions.
+        /// Automatically add a semicolon when completing unit-returning functions.
         ///
         /// In `match` arms it completes a comma instead.
         completion_addSemicolonToUnit: bool = true,
-        /// Toggles the additional completions that automatically show method calls and field accesses with `await` prefixed to them when completing on a future.
-        completion_autoAwait_enable: bool        = true,
-        /// Toggles the additional completions that automatically show method calls with `iter()` or `into_iter()` prefixed to them when completing on a type that has them.
-        completion_autoIter_enable: bool        = true,
-        /// Toggles the additional completions that automatically add imports when completed.
-        /// Note that your client must specify the `additionalTextEdits` LSP client capability to truly have this feature enabled.
-        completion_autoimport_enable: bool       = true,
+
+        /// Show method calls and field accesses completions with `await` prefixed to them when
+        /// completing on a future.
+        completion_autoAwait_enable: bool = true,
+
+        /// Show method call completions with `iter()` or `into_iter()` prefixed to them when
+        /// completing on a type that has them.
+        completion_autoIter_enable: bool = true,
+
+        /// Show completions that automatically add imports when completed.
+        ///
+        /// Note that your client must specify the `additionalTextEdits` LSP client capability to
+        /// truly have this feature enabled.
+        completion_autoimport_enable: bool = true,
+
         /// A list of full paths to items to exclude from auto-importing completions.
         ///
         /// Traits in this list won't have their methods suggested in completions unless the trait
         /// is in scope.
         ///
-        /// You can either specify a string path which defaults to type "always" or use the more verbose
-        /// form `{ "path": "path::to::item", type: "always" }`.
+        /// You can either specify a string path which defaults to type "always" or use the more
+        /// verbose form `{ "path": "path::to::item", type: "always" }`.
         ///
-        /// For traits the type "methods" can be used to only exclude the methods but not the trait itself.
+        /// For traits the type "methods" can be used to only exclude the methods but not the trait
+        /// itself.
         ///
         /// This setting also inherits `#rust-analyzer.completion.excludeTraits#`.
         completion_autoimport_exclude: Vec = vec![
             AutoImportExclusion::Verbose { path: "core::borrow::Borrow".to_owned(), r#type: AutoImportExclusionType::Methods },
             AutoImportExclusion::Verbose { path: "core::borrow::BorrowMut".to_owned(), r#type: AutoImportExclusionType::Methods },
         ],
-        /// Toggles the additional completions that automatically show method calls and field accesses
-        /// with `self` prefixed to them when inside a method.
-        completion_autoself_enable: bool        = true,
-        /// Whether to add parenthesis and argument snippets when completing function.
-        completion_callable_snippets: CallableCompletionDef  = CallableCompletionDef::FillArguments,
+
+        /// Show method calls and field access completions with `self` prefixed to them when
+        /// inside a method.
+        completion_autoself_enable: bool = true,
+
+        /// Add parenthesis and argument snippets when completing function.
+        completion_callable_snippets: CallableCompletionDef = CallableCompletionDef::FillArguments,
+
         /// A list of full paths to traits whose methods to exclude from completion.
         ///
-        /// Methods from these traits won't be completed, even if the trait is in scope. However, they will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or `T where T: Trait`.
+        /// Methods from these traits won't be completed, even if the trait is in scope. However,
+        /// they will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or
+        /// `T where T: Trait`.
         ///
         /// Note that the trait themselves can still be completed.
         completion_excludeTraits: Vec = Vec::new(),
-        /// Whether to show full function/method signatures in completion docs.
+
+        /// Show full function / method signatures in completion docs.
         completion_fullFunctionSignatures_enable: bool = false,
-        /// Whether to omit deprecated items from autocompletion. By default they are marked as deprecated but not hidden.
+
+        /// Omit deprecated items from completions. By default they are marked as deprecated but not
+        /// hidden.
         completion_hideDeprecated: bool = false,
+
         /// Maximum number of completions to return. If `None`, the limit is infinite.
         completion_limit: Option = None,
-        /// Whether to show postfix snippets like `dbg`, `if`, `not`, etc.
-        completion_postfix_enable: bool         = true,
-        /// Enables completions of private items and fields that are defined in the current workspace even if they are not visible at the current position.
+
+        /// Show postfix snippets like `dbg`, `if`, `not`, etc.
+        completion_postfix_enable: bool = true,
+
+        /// Show completions of private items and fields that are defined in the current workspace
+        /// even if they are not visible at the current position.
         completion_privateEditable_enable: bool = false,
-        /// Whether to enable term search based snippets like `Some(foo.bar().baz())`.
+
+        /// Enable term search based snippets like `Some(foo.bar().baz())`.
         completion_termSearch_enable: bool = false,
+
         /// Term search fuel in "units of work" for autocompletion (Defaults to 1000).
         completion_termSearch_fuel: usize = 1000,
 
         /// List of rust-analyzer diagnostics to disable.
         diagnostics_disabled: FxHashSet = FxHashSet::default(),
-        /// Whether to show native rust-analyzer diagnostics.
-        diagnostics_enable: bool                = true,
-        /// Whether to show experimental rust-analyzer diagnostics that might
-        /// have more false positives than usual.
-        diagnostics_experimental_enable: bool    = false,
-        /// Map of prefixes to be substituted when parsing diagnostic file paths.
-        /// This should be the reverse mapping of what is passed to `rustc` as `--remap-path-prefix`.
+
+        /// Show native rust-analyzer diagnostics.
+        diagnostics_enable: bool = true,
+
+        /// Show experimental rust-analyzer diagnostics that might have more false positives than
+        /// usual.
+        diagnostics_experimental_enable: bool = false,
+
+        /// Map of prefixes to be substituted when parsing diagnostic file paths. This should be the
+        /// reverse mapping of what is passed to `rustc` as `--remap-path-prefix`.
         diagnostics_remapPrefix: FxHashMap = FxHashMap::default(),
-        /// Whether to run additional style lints.
-        diagnostics_styleLints_enable: bool =    false,
+
+        /// Run additional style lints.
+        diagnostics_styleLints_enable: bool = false,
+
         /// List of warnings that should be displayed with hint severity.
         ///
-        /// The warnings will be indicated by faded text or three dots in code
-        /// and will not show up in the `Problems Panel`.
+        /// The warnings will be indicated by faded text or three dots in code and will not show up
+        /// in the `Problems Panel`.
         diagnostics_warningsAsHint: Vec = vec![],
+
         /// List of warnings that should be displayed with info severity.
         ///
-        /// The warnings will be indicated by a blue squiggly underline in code
-        /// and a blue icon in the `Problems Panel`.
+        /// The warnings will be indicated by a blue squiggly underline in code and a blue icon in
+        /// the `Problems Panel`.
         diagnostics_warningsAsInfo: Vec = vec![],
 
-        /// Whether to enforce the import granularity setting for all files. If set to false rust-analyzer will try to keep import styles consistent per file.
-        imports_granularity_enforce: bool              = false,
+        /// Enforce the import granularity setting for all files. If set to false rust-analyzer will
+        /// try to keep import styles consistent per file.
+        imports_granularity_enforce: bool = false,
+
         /// How imports should be grouped into use statements.
-        imports_granularity_group: ImportGranularityDef  = ImportGranularityDef::Crate,
-        /// Group inserted imports by the [following order](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are separated by newlines.
-        imports_group_enable: bool                           = true,
-        /// Whether to allow import insertion to merge new imports into single path glob imports like `use std::fmt::*;`.
-        imports_merge_glob: bool           = true,
+        imports_granularity_group: ImportGranularityDef = ImportGranularityDef::Crate,
+
+        /// Group inserted imports by the [following
+        /// order](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are
+        /// separated by newlines.
+        imports_group_enable: bool = true,
+
+        /// Allow import insertion to merge new imports into single path glob imports like `use
+        /// std::fmt::*;`.
+        imports_merge_glob: bool = true,
+
         /// Prefer to unconditionally use imports of the core and alloc crate, over the std crate.
         imports_preferNoStd | imports_prefer_no_std: bool = false,
-         /// Whether to prefer import paths containing a `prelude` module.
-        imports_preferPrelude: bool                       = false,
+
+        /// Prefer import paths containing a `prelude` module.
+        imports_preferPrelude: bool = false,
+
         /// The path structure for newly inserted paths to use.
-        imports_prefix: ImportPrefixDef               = ImportPrefixDef::ByCrate,
-        /// Whether to prefix external (including std, core) crate imports with `::`. e.g. "use ::std::io::Read;".
+        imports_prefix: ImportPrefixDef = ImportPrefixDef::ByCrate,
+
+        /// Prefix external (including std, core) crate imports with `::`.
+        ///
+        /// E.g. `use ::std::io::Read;`.
         imports_prefixExternPrelude: bool = false,
     }
 }
@@ -589,7 +743,9 @@ config_data! {
         /// ```bash
         /// cargo check --quiet --workspace --message-format=json --all-targets --keep-going
         /// ```
-        /// .
+        ///
+        /// Note: The option must be specified as an array of command line arguments, with
+        /// the first argument being the name of the command to run.
         cargo_buildScripts_overrideCommand: Option> = None,
         /// Rerun proc-macros building/build-scripts running when proc-macro
         /// or build-script sources change and are saved.
@@ -703,7 +859,9 @@ config_data! {
         /// ```bash
         /// cargo check --workspace --message-format=json --all-targets
         /// ```
-        /// .
+        ///
+        /// Note: The option must be specified as an array of command line arguments, with
+        /// the first argument being the name of the command to run.
         check_overrideCommand | checkOnSave_overrideCommand: Option>             = None,
         /// Check for specific targets. Defaults to `#rust-analyzer.cargo.target#` if empty.
         ///
@@ -717,6 +875,9 @@ config_data! {
         /// check will be performed.
         check_workspace: bool = true,
 
+        /// Exclude all locals from document symbol search.
+        document_symbol_search_excludeLocals: bool = true,
+
         /// These proc-macros will be ignored when trying to expand them.
         ///
         /// This config takes a map of crate names with the exported proc-macro names to ignore as values.
@@ -734,7 +895,7 @@ config_data! {
         /// [custom test harness](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-harness-field),
         /// they will end up being interpreted as options to
         /// [`rustc`’s built-in test harness (“libtest”)](https://doc.rust-lang.org/rustc/tests/index.html#cli-arguments).
-        runnables_extraTestBinaryArgs: Vec = vec!["--show-output".to_owned()],
+        runnables_extraTestBinaryArgs: Vec = vec!["--nocapture".to_owned()],
 
         /// Path to the Cargo.toml of the rust compiler workspace, for usage in rustc_private
         /// projects, or "discover" to try to automatically find it if the `rustc-dev` component
@@ -753,6 +914,9 @@ config_data! {
         /// not that of `cargo fmt`. The file contents will be passed on the
         /// standard input and the formatted result will be read from the
         /// standard output.
+        ///
+        /// Note: The option must be specified as an array of command line arguments, with
+        /// the first argument being the name of the command to run.
         rustfmt_overrideCommand: Option> = None,
         /// Enables the use of rustfmt's unstable range formatting command for the
         /// `textDocument/rangeFormatting` request. The rustfmt option is unstable and only
@@ -1290,6 +1454,23 @@ impl LensConfig {
     pub fn references(&self) -> bool {
         self.method_refs || self.refs_adt || self.refs_trait || self.enum_variant_refs
     }
+
+    pub fn into_annotation_config<'a>(
+        self,
+        binary_target: bool,
+        minicore: MiniCore<'a>,
+    ) -> AnnotationConfig<'a> {
+        AnnotationConfig {
+            binary_target,
+            annotate_runnables: self.runnable(),
+            annotate_impls: self.implementations,
+            annotate_references: self.refs_adt,
+            annotate_method_references: self.method_refs,
+            annotate_enum_variant_references: self.enum_variant_refs,
+            location: self.location.into(),
+            minicore,
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -1337,6 +1518,13 @@ pub enum FilesWatcher {
     Server,
 }
 
+/// Configuration for document symbol search requests.
+#[derive(Debug, Clone)]
+pub struct DocumentSymbolConfig {
+    /// Should locals be excluded.
+    pub search_exclude_locals: bool,
+}
+
 #[derive(Debug, Clone)]
 pub struct NotificationsConfig {
     pub cargo_toml_not_found: bool,
@@ -1517,11 +1705,15 @@ impl Config {
         }
     }
 
-    pub fn call_hierarchy(&self) -> CallHierarchyConfig {
-        CallHierarchyConfig { exclude_tests: self.references_excludeTests().to_owned() }
+    pub fn call_hierarchy<'a>(&self, minicore: MiniCore<'a>) -> CallHierarchyConfig<'a> {
+        CallHierarchyConfig { exclude_tests: self.references_excludeTests().to_owned(), minicore }
     }
 
-    pub fn completion(&self, source_root: Option) -> CompletionConfig<'_> {
+    pub fn completion<'a>(
+        &'a self,
+        source_root: Option,
+        minicore: MiniCore<'a>,
+    ) -> CompletionConfig<'a> {
         let client_capability_fields = self.completion_resolve_support_properties();
         CompletionConfig {
             enable_postfix_completions: self.completion_postfix_enable(source_root).to_owned(),
@@ -1575,6 +1767,7 @@ impl Config {
                 })
                 .collect(),
             exclude_traits: self.completion_excludeTraits(source_root),
+            minicore,
         }
     }
 
@@ -1649,7 +1842,7 @@ impl Config {
         }
     }
 
-    pub fn hover(&self) -> HoverConfig {
+    pub fn hover<'a>(&self, minicore: MiniCore<'a>) -> HoverConfig<'a> {
         let mem_kind = |kind| match kind {
             MemoryLayoutHoverRenderKindDef::Both => MemoryLayoutHoverRenderKind::Both,
             MemoryLayoutHoverRenderKindDef::Decimal => MemoryLayoutHoverRenderKind::Decimal,
@@ -1682,10 +1875,15 @@ impl Config {
                 None => ide::SubstTyLen::Unlimited,
             },
             show_drop_glue: *self.hover_dropGlue_enable(),
+            minicore,
         }
     }
 
-    pub fn inlay_hints(&self) -> InlayHintsConfig {
+    pub fn goto_definition<'a>(&self, minicore: MiniCore<'a>) -> GotoDefinitionConfig<'a> {
+        GotoDefinitionConfig { minicore }
+    }
+
+    pub fn inlay_hints<'a>(&self, minicore: MiniCore<'a>) -> InlayHintsConfig<'a> {
         let client_capability_fields = self.inlay_hint_resolve_support_properties();
 
         InlayHintsConfig {
@@ -1734,12 +1932,14 @@ impl Config {
                 AdjustmentHintsDef::Always => ide::AdjustmentHints::Always,
                 AdjustmentHintsDef::Never => match self.inlayHints_reborrowHints_enable() {
                     ReborrowHintsDef::Always | ReborrowHintsDef::Mutable => {
-                        ide::AdjustmentHints::ReborrowOnly
+                        ide::AdjustmentHints::BorrowsOnly
                     }
                     ReborrowHintsDef::Never => ide::AdjustmentHints::Never,
                 },
-                AdjustmentHintsDef::Reborrow => ide::AdjustmentHints::ReborrowOnly,
+                AdjustmentHintsDef::Borrows => ide::AdjustmentHints::BorrowsOnly,
             },
+            adjustment_hints_disable_reborrows: *self
+                .inlayHints_expressionAdjustmentHints_disableReborrows(),
             adjustment_hints_mode: match self.inlayHints_expressionAdjustmentHints_mode() {
                 AdjustmentHintsModeDef::Prefix => ide::AdjustmentHintsMode::Prefix,
                 AdjustmentHintsModeDef::Postfix => ide::AdjustmentHintsMode::Postfix,
@@ -1765,14 +1965,16 @@ impl Config {
             ),
             implicit_drop_hints: self.inlayHints_implicitDrops_enable().to_owned(),
             range_exclusive_hints: self.inlayHints_rangeExclusiveHints_enable().to_owned(),
+            minicore,
         }
     }
 
     fn insert_use_config(&self, source_root: Option) -> InsertUseConfig {
         InsertUseConfig {
             granularity: match self.imports_granularity_group(source_root) {
-                ImportGranularityDef::Preserve => ImportGranularity::Preserve,
-                ImportGranularityDef::Item => ImportGranularity::Item,
+                ImportGranularityDef::Item | ImportGranularityDef::Preserve => {
+                    ImportGranularity::Item
+                }
                 ImportGranularityDef::Crate => ImportGranularity::Crate,
                 ImportGranularityDef::Module => ImportGranularity::Module,
                 ImportGranularityDef::One => ImportGranularity::One,
@@ -1801,9 +2003,10 @@ impl Config {
         self.semanticHighlighting_nonStandardTokens().to_owned()
     }
 
-    pub fn highlighting_config(&self) -> HighlightConfig {
+    pub fn highlighting_config<'a>(&self, minicore: MiniCore<'a>) -> HighlightConfig<'a> {
         HighlightConfig {
             strings: self.semanticHighlighting_strings_enable().to_owned(),
+            comments: self.semanticHighlighting_comments_enable().to_owned(),
             punctuation: self.semanticHighlighting_punctuation_enable().to_owned(),
             specialize_punctuation: self
                 .semanticHighlighting_punctuation_specialization_enable()
@@ -1815,6 +2018,7 @@ impl Config {
                 .to_owned(),
             inject_doc_comment: self.semanticHighlighting_doc_comment_inject_enable().to_owned(),
             syntactic_name_ref_highlighting: false,
+            minicore,
         }
     }
 
@@ -2294,6 +2498,12 @@ impl Config {
         }
     }
 
+    pub fn document_symbol(&self, source_root: Option) -> DocumentSymbolConfig {
+        DocumentSymbolConfig {
+            search_exclude_locals: *self.document_symbol_search_excludeLocals(source_root),
+        }
+    }
+
     pub fn workspace_symbol(&self, source_root: Option) -> WorkspaceSymbolConfig {
         WorkspaceSymbolConfig {
             search_exclude_imports: *self.workspace_symbol_search_excludeImports(source_root),
@@ -2662,7 +2872,8 @@ enum ReborrowHintsDef {
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "snake_case")]
 enum AdjustmentHintsDef {
-    Reborrow,
+    #[serde(alias = "reborrow")]
+    Borrows,
     #[serde(with = "true_or_always")]
     #[serde(untagged)]
     Always,
@@ -2923,7 +3134,7 @@ macro_rules! _config_data {
     }) => {
         /// Default config values for this grouping.
         #[allow(non_snake_case)]
-        #[derive(Debug, Clone )]
+        #[derive(Debug, Clone)]
         struct $name { $($field: $ty,)* }
 
         impl_for_config_data!{
@@ -3201,8 +3412,10 @@ fn schema(fields: &[SchemaField]) -> serde_json::Value {
         .iter()
         .map(|(field, ty, doc, default)| {
             let name = field.replace('_', ".");
-            let category =
-                name.find('.').map(|end| String::from(&name[..end])).unwrap_or("general".into());
+            let category = name
+                .split_once(".")
+                .map(|(category, _name)| to_title_case(category))
+                .unwrap_or("rust-analyzer".into());
             let name = format!("rust-analyzer.{name}");
             let props = field_props(field, ty, doc, default);
             serde_json::json!({
@@ -3216,6 +3429,29 @@ fn schema(fields: &[SchemaField]) -> serde_json::Value {
     map.into()
 }
 
+/// Translate a field name to a title case string suitable for use in the category names on the
+/// vscode settings page.
+///
+/// First letter of word should be uppercase, if an uppercase letter is encountered, add a space
+/// before it e.g. "fooBar" -> "Foo Bar", "fooBarBaz" -> "Foo Bar Baz", "foo" -> "Foo"
+///
+/// This likely should be in stdx (or just use heck instead), but it doesn't handle any edge cases
+/// and is intentionally simple.
+fn to_title_case(s: &str) -> String {
+    let mut result = String::with_capacity(s.len());
+    let mut chars = s.chars();
+    if let Some(first) = chars.next() {
+        result.push(first.to_ascii_uppercase());
+        for c in chars {
+            if c.is_uppercase() {
+                result.push(' ');
+            }
+            result.push(c);
+        }
+    }
+    result
+}
+
 fn field_props(field: &str, ty: &str, doc: &[&str], default: &str) -> serde_json::Value {
     let doc = doc_comment_to_string(doc);
     let doc = doc.trim_end_matches('\n');
@@ -3298,13 +3534,23 @@ fn field_props(field: &str, ty: &str, doc: &[&str], default: &str) -> serde_json
         },
         "ImportGranularityDef" => set! {
             "type": "string",
-            "enum": ["preserve", "crate", "module", "item", "one"],
-            "enumDescriptions": [
-                "Do not change the granularity of any imports and preserve the original structure written by the developer.",
-                "Merge imports from the same crate into a single use statement. Conversely, imports from different crates are split into separate statements.",
-                "Merge imports from the same module into a single use statement. Conversely, imports from different modules are split into separate statements.",
-                "Flatten imports so that each has its own use statement.",
-                "Merge all imports into a single use statement as long as they have the same visibility and attributes."
+            "anyOf": [
+                {
+                    "enum": ["crate", "module", "item", "one"],
+                    "enumDescriptions": [
+                        "Merge imports from the same crate into a single use statement. Conversely, imports from different crates are split into separate statements.",
+                        "Merge imports from the same module into a single use statement. Conversely, imports from different modules are split into separate statements.",
+                        "Flatten imports so that each has its own use statement.",
+                        "Merge all imports into a single use statement as long as they have the same visibility and attributes."
+                    ],
+                },
+                {
+                    "enum": ["preserve"],
+                    "enumDescriptions": [
+                        "Deprecated - unless `enforceGranularity` is `true`, the style of the current file is preferred over this setting. Behaves like `item`.",
+                    ],
+                    "deprecated": true,
+                }
             ],
         },
         "ImportPrefixDef" => set! {
@@ -3742,17 +3988,16 @@ mod tests {
         for idx in url_offsets {
             let link = &schema[idx..];
             // matching on whitespace to ignore normal links
-            if let Some(link_end) = link.find([' ', '[']) {
-                if link.chars().nth(link_end) == Some('[') {
-                    if let Some(link_text_end) = link.find(']') {
-                        let link_text = link[link_end..(link_text_end + 1)].to_string();
+            if let Some(link_end) = link.find([' ', '['])
+                && link.chars().nth(link_end) == Some('[')
+                && let Some(link_text_end) = link.find(']')
+            {
+                let link_text = link[link_end..(link_text_end + 1)].to_string();
 
-                        schema.replace_range((idx + link_end)..(idx + link_text_end + 1), "");
-                        schema.insert(idx, '(');
-                        schema.insert(idx + link_end + 1, ')');
-                        schema.insert_str(idx, &link_text);
-                    }
-                }
+                schema.replace_range((idx + link_end)..(idx + link_text_end + 1), "");
+                schema.insert(idx, '(');
+                schema.insert(idx + link_end + 1, ')');
+                schema.insert_str(idx, &link_text);
             }
         }
 
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs
index 95857dd8f3b4..389bb7848c01 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/config/patch_old_style.rs
@@ -73,19 +73,19 @@ pub(super) fn patch_json_for_outdated_configs(json: &mut Value) {
     }
 
     // completion.snippets -> completion.snippets.custom;
-    if let Some(Value::Object(obj)) = copy.pointer("/completion/snippets").cloned() {
-        if obj.len() != 1 || obj.get("custom").is_none() {
-            merge(
-                json,
-                json! {{
-                    "completion": {
-                        "snippets": {
-                            "custom": obj
-                        },
+    if let Some(Value::Object(obj)) = copy.pointer("/completion/snippets").cloned()
+        && (obj.len() != 1 || obj.get("custom").is_none())
+    {
+        merge(
+            json,
+            json! {{
+                "completion": {
+                    "snippets": {
+                        "custom": obj
                     },
-                }},
-            );
-        }
+                },
+            }},
+        );
     }
 
     // callInfo_full -> signatureInfo_detail, signatureInfo_documentation_enable
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs
index 438a2a0ba1ea..4bfad98b3997 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics.rs
@@ -26,6 +26,17 @@ pub struct DiagnosticsMapConfig {
 
 pub(crate) type DiagnosticsGeneration = usize;
 
+#[derive(Debug, Clone, Default)]
+pub(crate) struct WorkspaceFlycheckDiagnostic {
+    pub(crate) per_package: FxHashMap>, PackageFlycheckDiagnostic>,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct PackageFlycheckDiagnostic {
+    generation: DiagnosticsGeneration,
+    per_file: FxHashMap>,
+}
+
 #[derive(Debug, Default, Clone)]
 pub(crate) struct DiagnosticCollection {
     // FIXME: should be FxHashMap>
@@ -33,9 +44,7 @@ pub(crate) struct DiagnosticCollection {
         FxHashMap)>,
     pub(crate) native_semantic:
         FxHashMap)>,
-    // FIXME: should be Vec
-    pub(crate) check:
-        Vec>, FxHashMap>>>,
+    pub(crate) check: Vec,
     pub(crate) check_fixes: CheckFixes,
     changes: FxHashSet,
     /// Counter for supplying a new generation number for diagnostics.
@@ -57,7 +66,7 @@ impl DiagnosticCollection {
         let Some(check) = self.check.get_mut(flycheck_id) else {
             return;
         };
-        self.changes.extend(check.drain().flat_map(|(_, v)| v.into_keys()));
+        self.changes.extend(check.per_package.drain().flat_map(|(_, v)| v.per_file.into_keys()));
         if let Some(fixes) = Arc::make_mut(&mut self.check_fixes).get_mut(flycheck_id) {
             fixes.clear();
         }
@@ -66,7 +75,9 @@ impl DiagnosticCollection {
     pub(crate) fn clear_check_all(&mut self) {
         Arc::make_mut(&mut self.check_fixes).clear();
         self.changes.extend(
-            self.check.iter_mut().flat_map(|it| it.drain().flat_map(|(_, v)| v.into_keys())),
+            self.check
+                .iter_mut()
+                .flat_map(|it| it.per_package.drain().flat_map(|(_, v)| v.per_file.into_keys())),
         )
     }
 
@@ -79,14 +90,59 @@ impl DiagnosticCollection {
             return;
         };
         let package_id = Some(package_id);
-        if let Some(checks) = check.remove(&package_id) {
-            self.changes.extend(checks.into_keys());
+        if let Some(checks) = check.per_package.remove(&package_id) {
+            self.changes.extend(checks.per_file.into_keys());
         }
         if let Some(fixes) = Arc::make_mut(&mut self.check_fixes).get_mut(flycheck_id) {
             fixes.remove(&package_id);
         }
     }
 
+    pub(crate) fn clear_check_older_than(
+        &mut self,
+        flycheck_id: usize,
+        generation: DiagnosticsGeneration,
+    ) {
+        if let Some(flycheck) = self.check.get_mut(flycheck_id) {
+            let mut packages = vec![];
+            self.changes.extend(
+                flycheck
+                    .per_package
+                    .extract_if(|_, v| v.generation < generation)
+                    .inspect(|(package_id, _)| packages.push(package_id.clone()))
+                    .flat_map(|(_, v)| v.per_file.into_keys()),
+            );
+            if let Some(fixes) = Arc::make_mut(&mut self.check_fixes).get_mut(flycheck_id) {
+                for package in packages {
+                    fixes.remove(&package);
+                }
+            }
+        }
+    }
+
+    pub(crate) fn clear_check_older_than_for_package(
+        &mut self,
+        flycheck_id: usize,
+        package_id: Arc,
+        generation: DiagnosticsGeneration,
+    ) {
+        let Some(check) = self.check.get_mut(flycheck_id) else {
+            return;
+        };
+        let package_id = Some(package_id);
+        let Some((_, checks)) = check
+            .per_package
+            .extract_if(|k, v| *k == package_id && v.generation < generation)
+            .next()
+        else {
+            return;
+        };
+        self.changes.extend(checks.per_file.into_keys());
+        if let Some(fixes) = Arc::make_mut(&mut self.check_fixes).get_mut(flycheck_id) {
+            fixes.remove(&package_id);
+        }
+    }
+
     pub(crate) fn clear_native_for(&mut self, file_id: FileId) {
         self.native_syntax.remove(&file_id);
         self.native_semantic.remove(&file_id);
@@ -96,19 +152,26 @@ impl DiagnosticCollection {
     pub(crate) fn add_check_diagnostic(
         &mut self,
         flycheck_id: usize,
+        generation: DiagnosticsGeneration,
         package_id: &Option>,
         file_id: FileId,
         diagnostic: lsp_types::Diagnostic,
         fix: Option>,
     ) {
         if self.check.len() <= flycheck_id {
-            self.check.resize_with(flycheck_id + 1, Default::default);
+            self.check.resize_with(flycheck_id + 1, WorkspaceFlycheckDiagnostic::default);
         }
-        let diagnostics = self.check[flycheck_id]
-            .entry(package_id.clone())
-            .or_default()
-            .entry(file_id)
-            .or_default();
+
+        let check = &mut self.check[flycheck_id];
+        let package = check.per_package.entry(package_id.clone()).or_insert_with(|| {
+            PackageFlycheckDiagnostic { generation, per_file: FxHashMap::default() }
+        });
+        // Getting message from old generation. Might happen in restarting checks.
+        if package.generation > generation {
+            return;
+        }
+        package.generation = generation;
+        let diagnostics = package.per_file.entry(file_id).or_default();
         for existing_diagnostic in diagnostics.iter() {
             if are_diagnostics_equal(existing_diagnostic, &diagnostic) {
                 return;
@@ -177,8 +240,8 @@ impl DiagnosticCollection {
         let check = self
             .check
             .iter()
-            .flat_map(|it| it.values())
-            .filter_map(move |it| it.get(&file_id))
+            .flat_map(|it| it.per_package.values())
+            .filter_map(move |it| it.per_file.get(&file_id))
             .flatten();
         native_syntax.chain(native_semantic).chain(check)
     }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics/to_proto.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics/to_proto.rs
index 79d8f678de4d..3f64628de860 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics/to_proto.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/diagnostics/to_proto.rs
@@ -298,10 +298,10 @@ pub(crate) fn map_rust_diagnostic_to_lsp(
     let mut source = String::from("rustc");
     let mut code = rd.code.as_ref().map(|c| c.code.clone());
 
-    if let Some(code_val) = &code {
-        if config.check_ignore.contains(code_val) {
-            return Vec::new();
-        }
+    if let Some(code_val) = &code
+        && config.check_ignore.contains(code_val)
+    {
+        return Vec::new();
     }
 
     if let Some(code_val) = &code {
@@ -373,10 +373,8 @@ pub(crate) fn map_rust_diagnostic_to_lsp(
             let primary_location = primary_location(config, workspace_root, primary_span, snap);
             let message = {
                 let mut message = message.clone();
-                if needs_primary_span_label {
-                    if let Some(primary_span_label) = &primary_span.label {
-                        format_to!(message, "\n{}", primary_span_label);
-                    }
+                if needs_primary_span_label && let Some(primary_span_label) = &primary_span.label {
+                    format_to!(message, "\n{}", primary_span_label);
                 }
                 message
             };
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs
index 24c433610f1a..4ec0c075dd5b 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/discover.rs
@@ -67,7 +67,7 @@ impl DiscoverCommand {
         cmd.args(args);
 
         Ok(DiscoverHandle {
-            _handle: CommandHandle::spawn(cmd, DiscoverProjectParser, self.sender.clone())?,
+            _handle: CommandHandle::spawn(cmd, DiscoverProjectParser, self.sender.clone(), None)?,
             span: info_span!("discover_command").entered(),
         })
     }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs
index 512ce0b9de35..73a51bba3d9a 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/flycheck.rs
@@ -1,13 +1,18 @@
 //! Flycheck provides the functionality needed to run `cargo check` to provide
 //! LSP diagnostics based on the output of the command.
 
-use std::{fmt, io, process::Command, time::Duration};
+use std::{
+    fmt, io,
+    process::Command,
+    sync::atomic::{AtomicUsize, Ordering},
+    time::Duration,
+};
 
 use cargo_metadata::PackageId;
 use crossbeam_channel::{Receiver, Sender, select_biased, unbounded};
 use ide_db::FxHashSet;
 use itertools::Itertools;
-use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
+use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf};
 use rustc_hash::FxHashMap;
 use serde::Deserialize as _;
 use serde_derive::Deserialize;
@@ -18,7 +23,10 @@ pub(crate) use cargo_metadata::diagnostic::{
 use toolchain::Tool;
 use triomphe::Arc;
 
-use crate::command::{CargoParser, CommandHandle};
+use crate::{
+    command::{CargoParser, CommandHandle},
+    diagnostics::DiagnosticsGeneration,
+};
 
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub(crate) enum InvocationStrategy {
@@ -96,11 +104,11 @@ pub(crate) enum FlycheckConfig {
 }
 
 impl FlycheckConfig {
-    pub(crate) fn invocation_strategy_once(&self) -> bool {
+    pub(crate) fn invocation_strategy(&self) -> InvocationStrategy {
         match self {
-            FlycheckConfig::CargoCommand { .. } => false,
+            FlycheckConfig::CargoCommand { .. } => InvocationStrategy::PerWorkspace,
             FlycheckConfig::CustomCommand { invocation_strategy, .. } => {
-                *invocation_strategy == InvocationStrategy::Once
+                invocation_strategy.clone()
             }
         }
     }
@@ -138,36 +146,64 @@ pub(crate) struct FlycheckHandle {
     sender: Sender,
     _thread: stdx::thread::JoinHandle,
     id: usize,
+    generation: AtomicUsize,
 }
 
 impl FlycheckHandle {
     pub(crate) fn spawn(
         id: usize,
+        generation: DiagnosticsGeneration,
         sender: Sender,
         config: FlycheckConfig,
         sysroot_root: Option,
         workspace_root: AbsPathBuf,
         manifest_path: Option,
     ) -> FlycheckHandle {
-        let actor =
-            FlycheckActor::new(id, sender, config, sysroot_root, workspace_root, manifest_path);
+        let actor = FlycheckActor::new(
+            id,
+            generation,
+            sender,
+            config,
+            sysroot_root,
+            workspace_root,
+            manifest_path,
+        );
         let (sender, receiver) = unbounded::();
         let thread =
             stdx::thread::Builder::new(stdx::thread::ThreadIntent::Worker, format!("Flycheck{id}"))
                 .spawn(move || actor.run(receiver))
                 .expect("failed to spawn thread");
-        FlycheckHandle { id, sender, _thread: thread }
+        FlycheckHandle { id, generation: generation.into(), sender, _thread: thread }
     }
 
     /// Schedule a re-start of the cargo check worker to do a workspace wide check.
     pub(crate) fn restart_workspace(&self, saved_file: Option) {
-        self.sender.send(StateChange::Restart { package: None, saved_file, target: None }).unwrap();
+        let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
+        self.sender
+            .send(StateChange::Restart {
+                generation,
+                scope: FlycheckScope::Workspace,
+                saved_file,
+                target: None,
+            })
+            .unwrap();
     }
 
     /// Schedule a re-start of the cargo check worker to do a package wide check.
-    pub(crate) fn restart_for_package(&self, package: String, target: Option) {
+    pub(crate) fn restart_for_package(
+        &self,
+        package: Arc,
+        target: Option,
+        workspace_deps: Option>>,
+    ) {
+        let generation = self.generation.fetch_add(1, Ordering::Relaxed) + 1;
         self.sender
-            .send(StateChange::Restart { package: Some(package), saved_file: None, target })
+            .send(StateChange::Restart {
+                generation,
+                scope: FlycheckScope::Package { package, workspace_deps },
+                saved_file: None,
+                target,
+            })
             .unwrap();
     }
 
@@ -179,23 +215,36 @@ impl FlycheckHandle {
     pub(crate) fn id(&self) -> usize {
         self.id
     }
+
+    pub(crate) fn generation(&self) -> DiagnosticsGeneration {
+        self.generation.load(Ordering::Relaxed)
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum ClearDiagnosticsKind {
+    All(ClearScope),
+    OlderThan(DiagnosticsGeneration, ClearScope),
+}
+
+#[derive(Debug)]
+pub(crate) enum ClearScope {
+    Workspace,
+    Package(Arc),
 }
 
 pub(crate) enum FlycheckMessage {
     /// Request adding a diagnostic with fixes included to a file
     AddDiagnostic {
         id: usize,
+        generation: DiagnosticsGeneration,
         workspace_root: Arc,
         diagnostic: Diagnostic,
         package_id: Option>,
     },
 
     /// Request clearing all outdated diagnostics.
-    ClearDiagnostics {
-        id: usize,
-        /// The package whose diagnostics to clear, or if unspecified, all diagnostics.
-        package_id: Option>,
-    },
+    ClearDiagnostics { id: usize, kind: ClearDiagnosticsKind },
 
     /// Request check progress notification to client
     Progress {
@@ -208,18 +257,23 @@ pub(crate) enum FlycheckMessage {
 impl fmt::Debug for FlycheckMessage {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            FlycheckMessage::AddDiagnostic { id, workspace_root, diagnostic, package_id } => f
+            FlycheckMessage::AddDiagnostic {
+                id,
+                generation,
+                workspace_root,
+                diagnostic,
+                package_id,
+            } => f
                 .debug_struct("AddDiagnostic")
                 .field("id", id)
+                .field("generation", generation)
                 .field("workspace_root", workspace_root)
                 .field("package_id", package_id)
                 .field("diagnostic_code", &diagnostic.code.as_ref().map(|it| &it.code))
                 .finish(),
-            FlycheckMessage::ClearDiagnostics { id, package_id } => f
-                .debug_struct("ClearDiagnostics")
-                .field("id", id)
-                .field("package_id", package_id)
-                .finish(),
+            FlycheckMessage::ClearDiagnostics { id, kind } => {
+                f.debug_struct("ClearDiagnostics").field("id", id).field("kind", kind).finish()
+            }
             FlycheckMessage::Progress { id, progress } => {
                 f.debug_struct("Progress").field("id", id).field("progress", progress).finish()
             }
@@ -236,8 +290,18 @@ pub(crate) enum Progress {
     DidFailToRestart(String),
 }
 
+enum FlycheckScope {
+    Workspace,
+    Package { package: Arc, workspace_deps: Option>> },
+}
+
 enum StateChange {
-    Restart { package: Option, saved_file: Option, target: Option },
+    Restart {
+        generation: DiagnosticsGeneration,
+        scope: FlycheckScope,
+        saved_file: Option,
+        target: Option,
+    },
     Cancel,
 }
 
@@ -246,6 +310,7 @@ struct FlycheckActor {
     /// The workspace id of this flycheck instance.
     id: usize,
 
+    generation: DiagnosticsGeneration,
     sender: Sender,
     config: FlycheckConfig,
     manifest_path: Option,
@@ -253,6 +318,7 @@ struct FlycheckActor {
     /// or the project root of the project.
     root: Arc,
     sysroot_root: Option,
+    scope: FlycheckScope,
     /// CargoHandle exists to wrap around the communication needed to be able to
     /// run `cargo check` without blocking. Currently the Rust standard library
     /// doesn't provide a way to read sub-process output without blocking, so we
@@ -283,6 +349,7 @@ pub(crate) const SAVED_FILE_PLACEHOLDER: &str = "$saved_file";
 impl FlycheckActor {
     fn new(
         id: usize,
+        generation: DiagnosticsGeneration,
         sender: Sender,
         config: FlycheckConfig,
         sysroot_root: Option,
@@ -292,10 +359,12 @@ impl FlycheckActor {
         tracing::info!(%id, ?workspace_root, "Spawning flycheck");
         FlycheckActor {
             id,
+            generation,
             sender,
             config,
             sysroot_root,
             root: Arc::new(workspace_root),
+            scope: FlycheckScope::Workspace,
             manifest_path,
             command_handle: None,
             command_receiver: None,
@@ -327,7 +396,12 @@ impl FlycheckActor {
                     tracing::debug!(flycheck_id = self.id, "flycheck cancelled");
                     self.cancel_check_process();
                 }
-                Event::RequestStateChange(StateChange::Restart { package, saved_file, target }) => {
+                Event::RequestStateChange(StateChange::Restart {
+                    generation,
+                    scope,
+                    saved_file,
+                    target,
+                }) => {
                     // Cancel the previously spawned process
                     self.cancel_check_process();
                     while let Ok(restart) = inbox.recv_timeout(Duration::from_millis(50)) {
@@ -337,9 +411,11 @@ impl FlycheckActor {
                         }
                     }
 
-                    let Some(command) =
-                        self.check_command(package.as_deref(), saved_file.as_deref(), target)
-                    else {
+                    let command = self.check_command(&scope, saved_file.as_deref(), target);
+                    self.scope = scope;
+                    self.generation = generation;
+
+                    let Some(command) = command else {
                         continue;
                     };
 
@@ -347,7 +423,23 @@ impl FlycheckActor {
 
                     tracing::debug!(?command, "will restart flycheck");
                     let (sender, receiver) = unbounded();
-                    match CommandHandle::spawn(command, CargoCheckParser, sender) {
+                    match CommandHandle::spawn(
+                        command,
+                        CargoCheckParser,
+                        sender,
+                        match &self.config {
+                            FlycheckConfig::CargoCommand { options, .. } => Some(
+                                options
+                                    .target_dir
+                                    .as_deref()
+                                    .unwrap_or(
+                                        Utf8Path::new("target").join("rust-analyzer").as_path(),
+                                    )
+                                    .join(format!("flycheck{}", self.id)),
+                            ),
+                            _ => None,
+                        },
+                    ) {
                         Ok(command_handle) => {
                             tracing::debug!(command = formatted_command, "did restart flycheck");
                             self.command_handle = Some(command_handle);
@@ -381,10 +473,55 @@ impl FlycheckActor {
                         tracing::trace!(flycheck_id = self.id, "clearing diagnostics");
                         // We finished without receiving any diagnostics.
                         // Clear everything for good measure
-                        self.send(FlycheckMessage::ClearDiagnostics {
-                            id: self.id,
-                            package_id: None,
-                        });
+                        match &self.scope {
+                            FlycheckScope::Workspace => {
+                                self.send(FlycheckMessage::ClearDiagnostics {
+                                    id: self.id,
+                                    kind: ClearDiagnosticsKind::All(ClearScope::Workspace),
+                                });
+                            }
+                            FlycheckScope::Package { package, workspace_deps } => {
+                                for pkg in
+                                    std::iter::once(package).chain(workspace_deps.iter().flatten())
+                                {
+                                    self.send(FlycheckMessage::ClearDiagnostics {
+                                        id: self.id,
+                                        kind: ClearDiagnosticsKind::All(ClearScope::Package(
+                                            pkg.clone(),
+                                        )),
+                                    });
+                                }
+                            }
+                        }
+                    } else if res.is_ok() {
+                        // We clear diagnostics for packages on
+                        // `[CargoCheckMessage::CompilerArtifact]` but there seem to be setups where
+                        // cargo may not report an artifact to our runner at all. To handle such
+                        // cases, clear stale diagnostics when flycheck completes successfully.
+                        match &self.scope {
+                            FlycheckScope::Workspace => {
+                                self.send(FlycheckMessage::ClearDiagnostics {
+                                    id: self.id,
+                                    kind: ClearDiagnosticsKind::OlderThan(
+                                        self.generation,
+                                        ClearScope::Workspace,
+                                    ),
+                                });
+                            }
+                            FlycheckScope::Package { package, workspace_deps } => {
+                                for pkg in
+                                    std::iter::once(package).chain(workspace_deps.iter().flatten())
+                                {
+                                    self.send(FlycheckMessage::ClearDiagnostics {
+                                        id: self.id,
+                                        kind: ClearDiagnosticsKind::OlderThan(
+                                            self.generation,
+                                            ClearScope::Package(pkg.clone()),
+                                        ),
+                                    });
+                                }
+                            }
+                        }
                     }
                     self.clear_diagnostics_state();
 
@@ -412,7 +549,7 @@ impl FlycheckActor {
                             );
                             self.send(FlycheckMessage::ClearDiagnostics {
                                 id: self.id,
-                                package_id: Some(package_id),
+                                kind: ClearDiagnosticsKind::All(ClearScope::Package(package_id)),
                             });
                         }
                     }
@@ -435,7 +572,9 @@ impl FlycheckActor {
                                 );
                                 self.send(FlycheckMessage::ClearDiagnostics {
                                     id: self.id,
-                                    package_id: Some(package_id.clone()),
+                                    kind: ClearDiagnosticsKind::All(ClearScope::Package(
+                                        package_id.clone(),
+                                    )),
                                 });
                             }
                         } else if self.diagnostics_received
@@ -444,11 +583,12 @@ impl FlycheckActor {
                             self.diagnostics_received = DiagnosticsReceived::YesAndClearedForAll;
                             self.send(FlycheckMessage::ClearDiagnostics {
                                 id: self.id,
-                                package_id: None,
+                                kind: ClearDiagnosticsKind::All(ClearScope::Workspace),
                             });
                         }
                         self.send(FlycheckMessage::AddDiagnostic {
                             id: self.id,
+                            generation: self.generation,
                             package_id,
                             workspace_root: self.root.clone(),
                             diagnostic,
@@ -465,7 +605,7 @@ impl FlycheckActor {
         if let Some(command_handle) = self.command_handle.take() {
             tracing::debug!(
                 command = ?command_handle,
-                "did  cancel flycheck"
+                "did cancel flycheck"
             );
             command_handle.cancel();
             self.command_receiver.take();
@@ -484,7 +624,7 @@ impl FlycheckActor {
     /// return None.
     fn check_command(
         &self,
-        package: Option<&str>,
+        scope: &FlycheckScope,
         saved_file: Option<&AbsPath>,
         target: Option,
     ) -> Option {
@@ -492,18 +632,18 @@ impl FlycheckActor {
             FlycheckConfig::CargoCommand { command, options, ansi_color_output } => {
                 let mut cmd =
                     toolchain::command(Tool::Cargo.path(), &*self.root, &options.extra_env);
-                if let Some(sysroot_root) = &self.sysroot_root {
-                    if !options.extra_env.contains_key("RUSTUP_TOOLCHAIN")
-                        && std::env::var_os("RUSTUP_TOOLCHAIN").is_none()
-                    {
-                        cmd.env("RUSTUP_TOOLCHAIN", AsRef::::as_ref(sysroot_root));
-                    }
+                if let Some(sysroot_root) = &self.sysroot_root
+                    && !options.extra_env.contains_key("RUSTUP_TOOLCHAIN")
+                    && std::env::var_os("RUSTUP_TOOLCHAIN").is_none()
+                {
+                    cmd.env("RUSTUP_TOOLCHAIN", AsRef::::as_ref(sysroot_root));
                 }
+                cmd.env("CARGO_LOG", "cargo::core::compiler::fingerprint=info");
                 cmd.arg(command);
 
-                match package {
-                    Some(pkg) => cmd.arg("-p").arg(pkg),
-                    None => cmd.arg("--workspace"),
+                match scope {
+                    FlycheckScope::Workspace => cmd.arg("--workspace"),
+                    FlycheckScope::Package { package, .. } => cmd.arg("-p").arg(&package.repr),
                 };
 
                 if let Some(tgt) = target {
@@ -525,6 +665,7 @@ impl FlycheckActor {
                     cmd.arg("--manifest-path");
                     cmd.arg(manifest_path);
                     if manifest_path.extension() == Some("rs") {
+                        cmd.env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "nightly");
                         cmd.arg("-Zscript");
                     }
                 }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
index 62a28a1a685d..f557dd5cb092 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/global_state.rs
@@ -9,10 +9,14 @@ use std::{
     time::{Duration, Instant},
 };
 
+use cargo_metadata::PackageId;
 use crossbeam_channel::{Receiver, Sender, unbounded};
 use hir::ChangeWithProcMacros;
 use ide::{Analysis, AnalysisHost, Cancellable, FileId, SourceRootId};
-use ide_db::base_db::{Crate, ProcMacroPaths, SourceDatabase};
+use ide_db::{
+    MiniCore,
+    base_db::{Crate, ProcMacroPaths, SourceDatabase},
+};
 use itertools::Itertools;
 use load_cargo::SourceRootConfig;
 use lsp_types::{SemanticTokens, Url};
@@ -183,6 +187,18 @@ pub(crate) struct GlobalState {
     /// this queue should run only *after* [`GlobalState::process_changes`] has
     /// been called.
     pub(crate) deferred_task_queue: TaskQueue,
+    /// HACK: Workaround for https://github.com/rust-lang/rust-analyzer/issues/19709
+    /// This is marked true if we failed to load a crate root file at crate graph creation,
+    /// which will usually end up causing a bunch of incorrect diagnostics on startup.
+    pub(crate) incomplete_crate_graph: bool,
+
+    pub(crate) minicore: MiniCoreRustAnalyzerInternalOnly,
+}
+
+// FIXME: This should move to the VFS once the rewrite is done.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct MiniCoreRustAnalyzerInternalOnly {
+    pub(crate) minicore_text: Option,
 }
 
 /// An immutable snapshot of the world's state at a point in time.
@@ -199,6 +215,7 @@ pub(crate) struct GlobalStateSnapshot {
     // FIXME: Can we derive this from somewhere else?
     pub(crate) proc_macros_loaded: bool,
     pub(crate) flycheck: Arc<[FlycheckHandle]>,
+    minicore: MiniCoreRustAnalyzerInternalOnly,
 }
 
 impl std::panic::UnwindSafe for GlobalStateSnapshot {}
@@ -298,6 +315,9 @@ impl GlobalState {
             discover_workspace_queue: OpQueue::default(),
 
             deferred_task_queue: task_queue,
+            incomplete_crate_graph: false,
+
+            minicore: MiniCoreRustAnalyzerInternalOnly::default(),
         };
         // Apply any required database inputs from the config.
         this.update_configuration(config);
@@ -448,7 +468,7 @@ impl GlobalState {
                     tracing::info!(%vfs_path, ?change_kind, "Processing rust-analyzer.toml changes");
                     if vfs_path.as_path() == user_config_abs_path {
                         tracing::info!(%vfs_path, ?change_kind, "Use config rust-analyzer.toml changes");
-                        change.change_user_config(Some(db.file_text(file_id).text(db)));
+                        change.change_user_config(Some(db.file_text(file_id).text(db).clone()));
                     }
 
                     // If change has been made to a ratoml file that
@@ -462,14 +482,14 @@ impl GlobalState {
                             change.change_workspace_ratoml(
                                 source_root_id,
                                 vfs_path.clone(),
-                                Some(db.file_text(file_id).text(db)),
+                                Some(db.file_text(file_id).text(db).clone()),
                             )
                         } else {
                             tracing::info!(%vfs_path, ?source_root_id, "crate rust-analyzer.toml changes");
                             change.change_ratoml(
                                 source_root_id,
                                 vfs_path.clone(),
-                                Some(db.file_text(file_id).text(db)),
+                                Some(db.file_text(file_id).text(db).clone()),
                             )
                         };
 
@@ -544,6 +564,7 @@ impl GlobalState {
             workspaces: Arc::clone(&self.workspaces),
             analysis: self.analysis_host.analysis(),
             vfs: Arc::clone(&self.vfs),
+            minicore: self.minicore.clone(),
             check_fixes: Arc::clone(&self.diagnostics.check_fixes),
             mem_docs: self.mem_docs.clone(),
             semantic_tokens_cache: Arc::clone(&self.semantic_tokens_cache),
@@ -591,10 +612,10 @@ impl GlobalState {
 
     pub(crate) fn respond(&mut self, response: lsp_server::Response) {
         if let Some((method, start)) = self.req_queue.incoming.complete(&response.id) {
-            if let Some(err) = &response.error {
-                if err.message.starts_with("server panicked") {
-                    self.poke_rust_analyzer_developer(format!("{}, check the log", err.message));
-                }
+            if let Some(err) = &response.error
+                && err.message.starts_with("server panicked")
+            {
+                self.poke_rust_analyzer_developer(format!("{}, check the log", err.message));
             }
 
             let duration = start.elapsed();
@@ -663,18 +684,18 @@ impl GlobalState {
 
     pub(crate) fn check_workspaces_msrv(&self) -> impl Iterator + '_ {
         self.workspaces.iter().filter_map(|ws| {
-            if let Some(toolchain) = &ws.toolchain {
-                if *toolchain < crate::MINIMUM_SUPPORTED_TOOLCHAIN_VERSION {
-                    return Some(format!(
-                        "Workspace `{}` is using an outdated toolchain version `{}` but \
+            if let Some(toolchain) = &ws.toolchain
+                && *toolchain < crate::MINIMUM_SUPPORTED_TOOLCHAIN_VERSION
+            {
+                return Some(format!(
+                    "Workspace `{}` is using an outdated toolchain version `{}` but \
                         rust-analyzer only supports `{}` and higher.\n\
                         Consider using the rust-analyzer rustup component for your toolchain or
                         upgrade your toolchain to a supported version.\n\n",
-                        ws.manifest_or_root(),
-                        toolchain,
-                        crate::MINIMUM_SUPPORTED_TOOLCHAIN_VERSION,
-                    ));
-                }
+                    ws.manifest_or_root(),
+                    toolchain,
+                    crate::MINIMUM_SUPPORTED_TOOLCHAIN_VERSION,
+                ));
             }
             None
         })
@@ -779,6 +800,7 @@ impl GlobalStateSnapshot {
                         cargo_toml: package_data.manifest.clone(),
                         crate_id,
                         package: cargo.package_flag(package_data),
+                        package_id: package_data.id.clone(),
                         target: target_data.name.clone(),
                         target_kind: target_data.kind,
                         required_features: target_data.required_features.clone(),
@@ -807,9 +829,38 @@ impl GlobalStateSnapshot {
         None
     }
 
+    pub(crate) fn all_workspace_dependencies_for_package(
+        &self,
+        package: &Arc,
+    ) -> Option>> {
+        for workspace in self.workspaces.iter() {
+            match &workspace.kind {
+                ProjectWorkspaceKind::Cargo { cargo, .. }
+                | ProjectWorkspaceKind::DetachedFile { cargo: Some((cargo, _, _)), .. } => {
+                    let package = cargo.packages().find(|p| cargo[*p].id == *package)?;
+
+                    return cargo[package]
+                        .all_member_deps
+                        .as_ref()
+                        .map(|deps| deps.iter().map(|dep| cargo[*dep].id.clone()).collect());
+                }
+                _ => {}
+            }
+        }
+        None
+    }
+
     pub(crate) fn file_exists(&self, file_id: FileId) -> bool {
         self.vfs.read().0.exists(file_id)
     }
+
+    #[inline]
+    pub(crate) fn minicore(&self) -> MiniCore<'_> {
+        match &self.minicore.minicore_text {
+            Some(minicore) => MiniCore::new(minicore),
+            None => MiniCore::default(),
+        }
+    }
 }
 
 pub(crate) fn file_id_to_url(vfs: &vfs::Vfs, id: FileId) -> Url {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
index aea116e647db..10bbb0bb31d9 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/dispatch.rs
@@ -141,7 +141,7 @@ impl RequestDispatcher<'_> {
                 Result: Serialize,
             > + 'static,
     {
-        if !self.global_state.vfs_done {
+        if !self.global_state.vfs_done || self.global_state.incomplete_crate_graph {
             if let Some(lsp_server::Request { id, .. }) =
                 self.req.take_if(|it| it.method == R::METHOD)
             {
@@ -433,10 +433,10 @@ impl NotificationDispatcher<'_> {
     }
 
     pub(crate) fn finish(&mut self) {
-        if let Some(not) = &self.not {
-            if !not.method.starts_with("$/") {
-                tracing::error!("unhandled notification: {:?}", not);
-            }
+        if let Some(not) = &self.not
+            && !not.method.starts_with("$/")
+        {
+            tracing::error!("unhandled notification: {:?}", not);
         }
     }
 }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
index 200e972e4289..87be09dcbd27 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/notification.rs
@@ -1,9 +1,11 @@
 //! This module is responsible for implementing handlers for Language Server
 //! Protocol. This module specifically handles notifications.
 
-use std::ops::{Deref, Not as _};
+use std::{
+    ops::{Deref, Not as _},
+    panic::UnwindSafe,
+};
 
-use ide_db::base_db::salsa::Cancelled;
 use itertools::Itertools;
 use lsp_types::{
     CancelParams, DidChangeConfigurationParams, DidChangeTextDocumentParams,
@@ -16,7 +18,7 @@ use vfs::{AbsPathBuf, ChangeKind, VfsPath};
 
 use crate::{
     config::{Config, ConfigChange},
-    flycheck::Target,
+    flycheck::{InvocationStrategy, Target},
     global_state::{FetchWorkspaceRequest, GlobalState},
     lsp::{from_proto, utils::apply_document_changes},
     lsp_ext::{self, RunFlycheckParams},
@@ -39,14 +41,12 @@ pub(crate) fn handle_work_done_progress_cancel(
     state: &mut GlobalState,
     params: WorkDoneProgressCancelParams,
 ) -> anyhow::Result<()> {
-    if let lsp_types::NumberOrString::String(s) = ¶ms.token {
-        if let Some(id) = s.strip_prefix("rust-analyzer/flycheck/") {
-            if let Ok(id) = id.parse::() {
-                if let Some(flycheck) = state.flycheck.get(id as usize) {
-                    flycheck.cancel();
-                }
-            }
-        }
+    if let lsp_types::NumberOrString::String(s) = ¶ms.token
+        && let Some(id) = s.strip_prefix("rust-analyzer/flycheck/")
+        && let Ok(id) = id.parse::()
+        && let Some(flycheck) = state.flycheck.get(id as usize)
+    {
+        flycheck.cancel();
     }
 
     // Just ignore this. It is OK to continue sending progress
@@ -76,12 +76,12 @@ pub(crate) fn handle_did_open_text_document(
             tracing::error!("duplicate DidOpenTextDocument: {}", path);
         }
 
-        if let Some(abs_path) = path.as_path() {
-            if state.config.excluded().any(|excluded| abs_path.starts_with(&excluded)) {
-                tracing::trace!("opened excluded file {abs_path}");
-                state.vfs.write().0.insert_excluded_file(path);
-                return Ok(());
-            }
+        if let Some(abs_path) = path.as_path()
+            && state.config.excluded().any(|excluded| abs_path.starts_with(&excluded))
+        {
+            tracing::trace!("opened excluded file {abs_path}");
+            state.vfs.write().0.insert_excluded_file(path);
+            return Ok(());
         }
 
         let contents = params.text_document.text.into_bytes();
@@ -303,124 +303,171 @@ fn run_flycheck(state: &mut GlobalState, vfs_path: VfsPath) -> bool {
     let file_id = state.vfs.read().0.file_id(&vfs_path);
     if let Some((file_id, vfs::FileExcluded::No)) = file_id {
         let world = state.snapshot();
-        let invocation_strategy_once = state.config.flycheck(None).invocation_strategy_once();
+        let invocation_strategy = state.config.flycheck(None).invocation_strategy();
         let may_flycheck_workspace = state.config.flycheck_workspace(None);
-        let mut updated = false;
-        let task = move || -> std::result::Result<(), Cancelled> {
-            if invocation_strategy_once {
-                let saved_file = vfs_path.as_path().map(|p| p.to_owned());
-                world.flycheck[0].restart_workspace(saved_file);
-            }
 
-            let target = TargetSpec::for_file(&world, file_id)?.and_then(|it| {
-                let tgt_kind = it.target_kind();
-                let (tgt_name, root, package) = match it {
-                    TargetSpec::Cargo(c) => (c.target, c.workspace_root, c.package),
-                    _ => return None,
-                };
-
-                let tgt = match tgt_kind {
-                    project_model::TargetKind::Bin => Target::Bin(tgt_name),
-                    project_model::TargetKind::Example => Target::Example(tgt_name),
-                    project_model::TargetKind::Test => Target::Test(tgt_name),
-                    project_model::TargetKind::Bench => Target::Benchmark(tgt_name),
-                    _ => return Some((None, root, package)),
-                };
-
-                Some((Some(tgt), root, package))
-            });
-            tracing::debug!(?target, "flycheck target");
-            // we have a specific non-library target, attempt to only check that target, nothing
-            // else will be affected
-            if let Some((target, root, package)) = target {
-                // trigger a package check if we have a non-library target as that can't affect
-                // anything else in the workspace OR if we're not allowed to check the workspace as
-                // the user opted into package checks then
-                let package_check_allowed = target.is_some() || !may_flycheck_workspace;
-                if package_check_allowed {
-                    let workspace = world.workspaces.iter().position(|ws| match &ws.kind {
-                        project_model::ProjectWorkspaceKind::Cargo { cargo, .. }
-                        | project_model::ProjectWorkspaceKind::DetachedFile {
-                            cargo: Some((cargo, _, _)),
-                            ..
-                        } => *cargo.workspace_root() == root,
-                        _ => false,
-                    });
-                    if let Some(idx) = workspace {
-                        world.flycheck[idx].restart_for_package(package, target);
-                    }
+        let task: Box ide::Cancellable<()> + Send + UnwindSafe> =
+            match invocation_strategy {
+                InvocationStrategy::Once => {
+                    Box::new(move || {
+                        // FIXME: Because triomphe::Arc's auto UnwindSafe impl requires that the inner type
+                        // be UnwindSafe, and FlycheckHandle is not UnwindSafe, `word.flycheck` cannot
+                        // be captured directly. std::sync::Arc has an UnwindSafe impl that only requires
+                        // that the inner type be RefUnwindSafe, so if we were using that one we wouldn't
+                        // have this problem. Remove the line below when triomphe::Arc has an UnwindSafe impl
+                        // like std::sync::Arc's.
+                        let world = world;
+                        stdx::always!(
+                            world.flycheck.len() == 1,
+                            "should have exactly one flycheck handle when invocation strategy is once"
+                        );
+                        let saved_file = vfs_path.as_path().map(ToOwned::to_owned);
+                        world.flycheck[0].restart_workspace(saved_file);
+                        Ok(())
+                    })
                 }
-            }
+                InvocationStrategy::PerWorkspace => {
+                    Box::new(move || {
+                        let target = TargetSpec::for_file(&world, file_id)?.and_then(|it| {
+                            let tgt_kind = it.target_kind();
+                            let (tgt_name, root, package) = match it {
+                                TargetSpec::Cargo(c) => (c.target, c.workspace_root, c.package_id),
+                                _ => return None,
+                            };
 
-            if !may_flycheck_workspace {
-                return Ok(());
-            }
+                            let tgt = match tgt_kind {
+                                project_model::TargetKind::Bin => Target::Bin(tgt_name),
+                                project_model::TargetKind::Example => Target::Example(tgt_name),
+                                project_model::TargetKind::Test => Target::Test(tgt_name),
+                                project_model::TargetKind::Bench => Target::Benchmark(tgt_name),
+                                _ => return Some((None, root, package)),
+                            };
 
-            // Trigger flychecks for all workspaces that depend on the saved file
-            // Crates containing or depending on the saved file
-            let crate_ids = world
-                .analysis
-                .crates_for(file_id)?
-                .into_iter()
-                .flat_map(|id| world.analysis.transitive_rev_deps(id))
-                .flatten()
-                .unique()
-                .collect::>();
-            tracing::debug!(?crate_ids, "flycheck crate ids");
-            let crate_root_paths: Vec<_> = crate_ids
-                .iter()
-                .filter_map(|&crate_id| {
-                    world
-                        .analysis
-                        .crate_root(crate_id)
-                        .map(|file_id| {
-                            world.file_id_to_file_path(file_id).as_path().map(ToOwned::to_owned)
-                        })
-                        .transpose()
-                })
-                .collect::>()?;
-            let crate_root_paths: Vec<_> = crate_root_paths.iter().map(Deref::deref).collect();
-            tracing::debug!(?crate_root_paths, "flycheck crate roots");
+                            Some((Some(tgt), root, package))
+                        });
+                        tracing::debug!(?target, "flycheck target");
+                        // we have a specific non-library target, attempt to only check that target, nothing
+                        // else will be affected
+                        let mut package_workspace_idx = None;
+                        if let Some((target, root, package)) = target {
+                            // trigger a package check if we have a non-library target as that can't affect
+                            // anything else in the workspace OR if we're not allowed to check the workspace as
+                            // the user opted into package checks then
+                            let package_check_allowed = target.is_some() || !may_flycheck_workspace;
+                            if package_check_allowed {
+                                package_workspace_idx =
+                                    world.workspaces.iter().position(|ws| match &ws.kind {
+                                        project_model::ProjectWorkspaceKind::Cargo {
+                                            cargo,
+                                            ..
+                                        }
+                                        | project_model::ProjectWorkspaceKind::DetachedFile {
+                                            cargo: Some((cargo, _, _)),
+                                            ..
+                                        } => *cargo.workspace_root() == root,
+                                        _ => false,
+                                    });
+                                if let Some(idx) = package_workspace_idx {
+                                    let workspace_deps =
+                                        world.all_workspace_dependencies_for_package(&package);
+                                    world.flycheck[idx].restart_for_package(
+                                        package,
+                                        target,
+                                        workspace_deps,
+                                    );
+                                }
+                            }
+                        }
 
-            // Find all workspaces that have at least one target containing the saved file
-            let workspace_ids =
-                world.workspaces.iter().enumerate().filter(|(_, ws)| match &ws.kind {
-                    project_model::ProjectWorkspaceKind::Cargo { cargo, .. }
-                    | project_model::ProjectWorkspaceKind::DetachedFile {
-                        cargo: Some((cargo, _, _)),
-                        ..
-                    } => cargo.packages().any(|pkg| {
-                        cargo[pkg]
-                            .targets
+                        if !may_flycheck_workspace {
+                            return Ok(());
+                        }
+
+                        // Trigger flychecks for all workspaces that depend on the saved file
+                        // Crates containing or depending on the saved file
+                        let crate_ids: Vec<_> = world
+                            .analysis
+                            .crates_for(file_id)?
+                            .into_iter()
+                            .flat_map(|id| world.analysis.transitive_rev_deps(id))
+                            .flatten()
+                            .unique()
+                            .collect();
+                        tracing::debug!(?crate_ids, "flycheck crate ids");
+                        let crate_root_paths: Vec<_> = crate_ids
                             .iter()
-                            .any(|&it| crate_root_paths.contains(&cargo[it].root.as_path()))
-                    }),
-                    project_model::ProjectWorkspaceKind::Json(project) => project
-                        .crates()
-                        .any(|(_, krate)| crate_root_paths.contains(&krate.root_module.as_path())),
-                    project_model::ProjectWorkspaceKind::DetachedFile { .. } => false,
-                });
+                            .filter_map(|&crate_id| {
+                                world
+                                    .analysis
+                                    .crate_root(crate_id)
+                                    .map(|file_id| {
+                                        world
+                                            .file_id_to_file_path(file_id)
+                                            .as_path()
+                                            .map(ToOwned::to_owned)
+                                    })
+                                    .transpose()
+                            })
+                            .collect::>()?;
+                        let crate_root_paths: Vec<_> =
+                            crate_root_paths.iter().map(Deref::deref).collect();
+                        tracing::debug!(?crate_root_paths, "flycheck crate roots");
 
-            let saved_file = vfs_path.as_path().map(|p| p.to_owned());
+                        // Find all workspaces that have at least one target containing the saved file
+                        let workspace_ids =
+                            world.workspaces.iter().enumerate().filter(|&(idx, ws)| {
+                                let ws_contains_file = match &ws.kind {
+                                    project_model::ProjectWorkspaceKind::Cargo {
+                                        cargo, ..
+                                    }
+                                    | project_model::ProjectWorkspaceKind::DetachedFile {
+                                        cargo: Some((cargo, _, _)),
+                                        ..
+                                    } => cargo.packages().any(|pkg| {
+                                        cargo[pkg].targets.iter().any(|&it| {
+                                            crate_root_paths.contains(&cargo[it].root.as_path())
+                                        })
+                                    }),
+                                    project_model::ProjectWorkspaceKind::Json(project) => {
+                                        project.crates().any(|(_, krate)| {
+                                            crate_root_paths.contains(&krate.root_module.as_path())
+                                        })
+                                    }
+                                    project_model::ProjectWorkspaceKind::DetachedFile {
+                                        ..
+                                    } => false,
+                                };
+                                let is_pkg_ws = match package_workspace_idx {
+                                    Some(pkg_idx) => pkg_idx == idx,
+                                    None => false,
+                                };
+                                ws_contains_file && !is_pkg_ws
+                            });
 
-            // Find and trigger corresponding flychecks
-            'flychecks: for flycheck in world.flycheck.iter() {
-                for (id, _) in workspace_ids.clone() {
-                    if id == flycheck.id() {
-                        updated = true;
-                        flycheck.restart_workspace(saved_file.clone());
-                        continue 'flychecks;
-                    }
+                        let saved_file = vfs_path.as_path().map(ToOwned::to_owned);
+                        let mut workspace_check_triggered = false;
+                        // Find and trigger corresponding flychecks
+                        'flychecks: for flycheck in world.flycheck.iter() {
+                            for (id, _) in workspace_ids.clone() {
+                                if id == flycheck.id() {
+                                    workspace_check_triggered = true;
+                                    flycheck.restart_workspace(saved_file.clone());
+                                    continue 'flychecks;
+                                }
+                            }
+                        }
+
+                        // No specific flycheck was triggered, so let's trigger all of them.
+                        if !workspace_check_triggered && package_workspace_idx.is_none() {
+                            for flycheck in world.flycheck.iter() {
+                                flycheck.restart_workspace(saved_file.clone());
+                            }
+                        }
+                        Ok(())
+                    })
                 }
-            }
-            // No specific flycheck was triggered, so let's trigger all of them.
-            if !updated {
-                for flycheck in world.flycheck.iter() {
-                    flycheck.restart_workspace(saved_file.clone());
-                }
-            }
-            Ok(())
-        };
+            };
+
         state.task_pool.handle.spawn_with_sender(stdx::thread::ThreadIntent::Worker, move |_| {
             if let Err(e) = std::panic::catch_unwind(task) {
                 tracing::error!("flycheck task panicked: {e:?}")
@@ -449,12 +496,11 @@ pub(crate) fn handle_run_flycheck(
     params: RunFlycheckParams,
 ) -> anyhow::Result<()> {
     let _p = tracing::info_span!("handle_run_flycheck").entered();
-    if let Some(text_document) = params.text_document {
-        if let Ok(vfs_path) = from_proto::vfs_path(&text_document.uri) {
-            if run_flycheck(state, vfs_path) {
-                return Ok(());
-            }
-        }
+    if let Some(text_document) = params.text_document
+        && let Ok(vfs_path) = from_proto::vfs_path(&text_document.uri)
+        && run_flycheck(state, vfs_path)
+    {
+        return Ok(());
     }
     // No specific flycheck was triggered, so let's trigger all of them.
     if state.config.flycheck_workspace(None) {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/request.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/request.rs
index a76a65220d3b..55d092f30f6b 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/request.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/handlers/request.rs
@@ -7,9 +7,10 @@ use anyhow::Context;
 
 use base64::{Engine, prelude::BASE64_STANDARD};
 use ide::{
-    AnnotationConfig, AssistKind, AssistResolveStrategy, Cancellable, CompletionFieldsToResolve,
-    FilePosition, FileRange, HoverAction, HoverGotoTypeData, InlayFieldsToResolve, Query,
-    RangeInfo, ReferenceCategory, Runnable, RunnableKind, SingleResolve, SourceChange, TextEdit,
+    AssistKind, AssistResolveStrategy, Cancellable, CompletionFieldsToResolve, FilePosition,
+    FileRange, FileStructureConfig, FindAllRefsConfig, HoverAction, HoverGotoTypeData,
+    InlayFieldsToResolve, Query, RangeInfo, ReferenceCategory, Runnable, RunnableKind,
+    SingleResolve, SourceChange, TextEdit,
 };
 use ide_db::{FxHashMap, SymbolKind};
 use itertools::Itertools;
@@ -566,41 +567,47 @@ pub(crate) fn handle_document_symbol(
     let file_id = try_default!(from_proto::file_id(&snap, ¶ms.text_document.uri)?);
     let line_index = snap.file_line_index(file_id)?;
 
-    let mut parents: Vec<(lsp_types::DocumentSymbol, Option)> = Vec::new();
+    let mut symbols: Vec<(lsp_types::DocumentSymbol, Option)> = Vec::new();
 
-    for symbol in snap.analysis.file_structure(file_id)? {
+    let config = snap.config.document_symbol(None);
+
+    let structure_nodes = snap.analysis.file_structure(
+        &FileStructureConfig { exclude_locals: config.search_exclude_locals },
+        file_id,
+    )?;
+
+    for node in structure_nodes {
         let mut tags = Vec::new();
-        if symbol.deprecated {
+        if node.deprecated {
             tags.push(SymbolTag::DEPRECATED)
         };
 
         #[allow(deprecated)]
-        let doc_symbol = lsp_types::DocumentSymbol {
-            name: symbol.label,
-            detail: symbol.detail,
-            kind: to_proto::structure_node_kind(symbol.kind),
+        let symbol = lsp_types::DocumentSymbol {
+            name: node.label,
+            detail: node.detail,
+            kind: to_proto::structure_node_kind(node.kind),
             tags: Some(tags),
-            deprecated: Some(symbol.deprecated),
-            range: to_proto::range(&line_index, symbol.node_range),
-            selection_range: to_proto::range(&line_index, symbol.navigation_range),
+            deprecated: Some(node.deprecated),
+            range: to_proto::range(&line_index, node.node_range),
+            selection_range: to_proto::range(&line_index, node.navigation_range),
             children: None,
         };
-        parents.push((doc_symbol, symbol.parent));
+        symbols.push((symbol, node.parent));
     }
 
-    // Builds hierarchy from a flat list, in reverse order (so that indices
-    // makes sense)
+    // Builds hierarchy from a flat list, in reverse order (so that the indices make sense)
     let document_symbols = {
         let mut acc = Vec::new();
-        while let Some((mut node, parent_idx)) = parents.pop() {
-            if let Some(children) = &mut node.children {
+        while let Some((mut symbol, parent_idx)) = symbols.pop() {
+            if let Some(children) = &mut symbol.children {
                 children.reverse();
             }
             let parent = match parent_idx {
                 None => &mut acc,
-                Some(i) => parents[i].0.children.get_or_insert_with(Vec::new),
+                Some(i) => symbols[i].0.children.get_or_insert_with(Vec::new),
             };
-            parent.push(node);
+            parent.push(symbol);
         }
         acc.reverse();
         acc
@@ -610,7 +617,7 @@ pub(crate) fn handle_document_symbol(
         document_symbols.into()
     } else {
         let url = to_proto::url(&snap, file_id);
-        let mut symbol_information = Vec::::new();
+        let mut symbol_information = Vec::new();
         for symbol in document_symbols {
             flatten_document_symbol(&symbol, None, &url, &mut symbol_information);
         }
@@ -647,7 +654,7 @@ pub(crate) fn handle_workspace_symbol(
     let _p = tracing::info_span!("handle_workspace_symbol").entered();
 
     let config = snap.config.workspace_symbol(None);
-    let (all_symbols, libs) = decide_search_scope_and_kind(¶ms, &config);
+    let (all_symbols, libs) = decide_search_kind_and_scope(¶ms, &config);
 
     let query = {
         let query: String = params.query.chars().filter(|&c| c != '#' && c != '*').collect();
@@ -670,7 +677,7 @@ pub(crate) fn handle_workspace_symbol(
 
     return Ok(Some(lsp_types::WorkspaceSymbolResponse::Nested(res)));
 
-    fn decide_search_scope_and_kind(
+    fn decide_search_kind_and_scope(
         params: &WorkspaceSymbolParams,
         config: &WorkspaceSymbolConfig,
     ) -> (bool, bool) {
@@ -804,7 +811,8 @@ pub(crate) fn handle_goto_definition(
     let _p = tracing::info_span!("handle_goto_definition").entered();
     let position =
         try_default!(from_proto::file_position(&snap, params.text_document_position_params)?);
-    let nav_info = match snap.analysis.goto_definition(position)? {
+    let config = snap.config.goto_definition(snap.minicore());
+    let nav_info = match snap.analysis.goto_definition(position, &config)? {
         None => return Ok(None),
         Some(it) => it,
     };
@@ -822,7 +830,8 @@ pub(crate) fn handle_goto_declaration(
         &snap,
         params.text_document_position_params.clone()
     )?);
-    let nav_info = match snap.analysis.goto_declaration(position)? {
+    let config = snap.config.goto_definition(snap.minicore());
+    let nav_info = match snap.analysis.goto_declaration(position, &config)? {
         None => return handle_goto_definition(snap, params),
         Some(it) => it,
     };
@@ -973,14 +982,13 @@ pub(crate) fn handle_runnables(
                 res.push(runnable);
             }
 
-            if let lsp_ext::RunnableArgs::Cargo(r) = &mut runnable.args {
-                if let Some(TargetSpec::Cargo(CargoTargetSpec {
+            if let lsp_ext::RunnableArgs::Cargo(r) = &mut runnable.args
+                && let Some(TargetSpec::Cargo(CargoTargetSpec {
                     sysroot_root: Some(sysroot_root),
                     ..
                 })) = &target_spec
-                {
-                    r.environment.insert("RUSTC_TOOLCHAIN".to_owned(), sysroot_root.to_string());
-                }
+            {
+                r.environment.insert("RUSTC_TOOLCHAIN".to_owned(), sysroot_root.to_string());
             };
 
             res.push(runnable);
@@ -1034,25 +1042,25 @@ pub(crate) fn handle_runnables(
         }
         Some(TargetSpec::ProjectJson(_)) => {}
         None => {
-            if !snap.config.linked_or_discovered_projects().is_empty() {
-                if let Some(path) = snap.file_id_to_file_path(file_id).parent() {
-                    let mut cargo_args = vec!["check".to_owned(), "--workspace".to_owned()];
-                    cargo_args.extend(config.cargo_extra_args.iter().cloned());
-                    res.push(lsp_ext::Runnable {
-                        label: "cargo check --workspace".to_owned(),
-                        location: None,
-                        kind: lsp_ext::RunnableKind::Cargo,
-                        args: lsp_ext::RunnableArgs::Cargo(lsp_ext::CargoRunnableArgs {
-                            workspace_root: None,
-                            cwd: path.as_path().unwrap().to_path_buf().into(),
-                            override_cargo: config.override_cargo,
-                            cargo_args,
-                            executable_args: Vec::new(),
-                            environment: Default::default(),
-                        }),
-                    });
-                };
-            }
+            if !snap.config.linked_or_discovered_projects().is_empty()
+                && let Some(path) = snap.file_id_to_file_path(file_id).parent()
+            {
+                let mut cargo_args = vec!["check".to_owned(), "--workspace".to_owned()];
+                cargo_args.extend(config.cargo_extra_args.iter().cloned());
+                res.push(lsp_ext::Runnable {
+                    label: "cargo check --workspace".to_owned(),
+                    location: None,
+                    kind: lsp_ext::RunnableKind::Cargo,
+                    args: lsp_ext::RunnableArgs::Cargo(lsp_ext::CargoRunnableArgs {
+                        workspace_root: None,
+                        cwd: path.as_path().unwrap().to_path_buf().into(),
+                        override_cargo: config.override_cargo,
+                        cargo_args,
+                        executable_args: Vec::new(),
+                        environment: Default::default(),
+                    }),
+                });
+            };
         }
     }
     Ok(res)
@@ -1100,7 +1108,7 @@ pub(crate) fn handle_completion(
         context.and_then(|ctx| ctx.trigger_character).and_then(|s| s.chars().next());
 
     let source_root = snap.analysis.source_root_id(position.file_id)?;
-    let completion_config = &snap.config.completion(Some(source_root));
+    let completion_config = &snap.config.completion(Some(source_root), snap.minicore());
     // FIXME: We should fix up the position when retrying the cancelled request instead
     position.offset = position.offset.min(line_index.index.len());
     let items = match snap.analysis.completions(
@@ -1154,7 +1162,8 @@ pub(crate) fn handle_completion_resolve(
     };
     let source_root = snap.analysis.source_root_id(file_id)?;
 
-    let mut forced_resolve_completions_config = snap.config.completion(Some(source_root));
+    let mut forced_resolve_completions_config =
+        snap.config.completion(Some(source_root), snap.minicore());
     forced_resolve_completions_config.fields_to_resolve = CompletionFieldsToResolve::empty();
 
     let position = FilePosition { file_id, offset };
@@ -1268,7 +1277,7 @@ pub(crate) fn handle_hover(
     };
     let file_range = try_default!(from_proto::file_range(&snap, ¶ms.text_document, range)?);
 
-    let hover = snap.config.hover();
+    let hover = snap.config.hover(snap.minicore());
     let info = match snap.analysis.hover(&hover, file_range)? {
         None => return Ok(None),
         Some(info) => info,
@@ -1354,7 +1363,11 @@ pub(crate) fn handle_references(
     let exclude_imports = snap.config.find_all_refs_exclude_imports();
     let exclude_tests = snap.config.find_all_refs_exclude_tests();
 
-    let Some(refs) = snap.analysis.find_all_refs(position, None)? else {
+    let Some(refs) = snap.analysis.find_all_refs(
+        position,
+        &FindAllRefsConfig { search_scope: None, minicore: snap.minicore() },
+    )?
+    else {
         return Ok(None);
     };
 
@@ -1557,12 +1570,12 @@ pub(crate) fn handle_code_action_resolve(
     code_action.edit = ca.edit;
     code_action.command = ca.command;
 
-    if let Some(edit) = code_action.edit.as_ref() {
-        if let Some(changes) = edit.document_changes.as_ref() {
-            for change in changes {
-                if let lsp_ext::SnippetDocumentChangeOperation::Op(res_op) = change {
-                    resource_ops_supported(&snap.config, resolve_resource_op(res_op))?
-                }
+    if let Some(edit) = code_action.edit.as_ref()
+        && let Some(changes) = edit.document_changes.as_ref()
+    {
+        for change in changes {
+            if let lsp_ext::SnippetDocumentChangeOperation::Op(res_op) = change {
+                resource_ops_supported(&snap.config, resolve_resource_op(res_op))?
             }
         }
     }
@@ -1609,8 +1622,8 @@ pub(crate) fn handle_code_lens(
     let target_spec = TargetSpec::for_file(&snap, file_id)?;
 
     let annotations = snap.analysis.annotations(
-        &AnnotationConfig {
-            binary_target: target_spec
+        &lens_config.into_annotation_config(
+            target_spec
                 .map(|spec| {
                     matches!(
                         spec.target_kind(),
@@ -1618,13 +1631,8 @@ pub(crate) fn handle_code_lens(
                     )
                 })
                 .unwrap_or(false),
-            annotate_runnables: lens_config.runnable(),
-            annotate_impls: lens_config.implementations,
-            annotate_references: lens_config.refs_adt,
-            annotate_method_references: lens_config.method_refs,
-            annotate_enum_variant_references: lens_config.enum_variant_refs,
-            location: lens_config.location.into(),
-        },
+            snap.minicore(),
+        ),
         file_id,
     )?;
 
@@ -1647,7 +1655,8 @@ pub(crate) fn handle_code_lens_resolve(
     let Some(annotation) = from_proto::annotation(&snap, code_lens.range, resolve)? else {
         return Ok(code_lens);
     };
-    let annotation = snap.analysis.resolve_annotation(annotation)?;
+    let config = snap.config.lens().into_annotation_config(false, snap.minicore());
+    let annotation = snap.analysis.resolve_annotation(&config, annotation)?;
 
     let mut acc = Vec::new();
     to_proto::code_lens(&mut acc, &snap, annotation)?;
@@ -1730,7 +1739,7 @@ pub(crate) fn handle_inlay_hints(
         range.end().min(line_index.index.len()),
     );
 
-    let inlay_hints_config = snap.config.inlay_hints();
+    let inlay_hints_config = snap.config.inlay_hints(snap.minicore());
     Ok(Some(
         snap.analysis
             .inlay_hints(&inlay_hints_config, file_id, Some(range))?
@@ -1771,7 +1780,7 @@ pub(crate) fn handle_inlay_hints_resolve(
     let line_index = snap.file_line_index(file_id)?;
     let range = from_proto::text_range(&line_index, resolve_data.resolve_range)?;
 
-    let mut forced_resolve_inlay_hints_config = snap.config.inlay_hints();
+    let mut forced_resolve_inlay_hints_config = snap.config.inlay_hints(snap.minicore());
     forced_resolve_inlay_hints_config.fields_to_resolve = InlayFieldsToResolve::empty();
     let resolve_hints = snap.analysis.inlay_hints_resolve(
         &forced_resolve_inlay_hints_config,
@@ -1810,7 +1819,8 @@ pub(crate) fn handle_call_hierarchy_prepare(
     let position =
         try_default!(from_proto::file_position(&snap, params.text_document_position_params)?);
 
-    let nav_info = match snap.analysis.call_hierarchy(position)? {
+    let config = snap.config.call_hierarchy(snap.minicore());
+    let nav_info = match snap.analysis.call_hierarchy(position, &config)? {
         None => return Ok(None),
         Some(it) => it,
     };
@@ -1836,8 +1846,8 @@ pub(crate) fn handle_call_hierarchy_incoming(
     let frange = try_default!(from_proto::file_range(&snap, &doc, item.selection_range)?);
     let fpos = FilePosition { file_id: frange.file_id, offset: frange.range.start() };
 
-    let config = snap.config.call_hierarchy();
-    let call_items = match snap.analysis.incoming_calls(config, fpos)? {
+    let config = snap.config.call_hierarchy(snap.minicore());
+    let call_items = match snap.analysis.incoming_calls(&config, fpos)? {
         None => return Ok(None),
         Some(it) => it,
     };
@@ -1875,8 +1885,8 @@ pub(crate) fn handle_call_hierarchy_outgoing(
     let fpos = FilePosition { file_id: frange.file_id, offset: frange.range.start() };
     let line_index = snap.file_line_index(fpos.file_id)?;
 
-    let config = snap.config.call_hierarchy();
-    let call_items = match snap.analysis.outgoing_calls(config, fpos)? {
+    let config = snap.config.call_hierarchy(snap.minicore());
+    let call_items = match snap.analysis.outgoing_calls(&config, fpos)? {
         None => return Ok(None),
         Some(it) => it,
     };
@@ -1910,7 +1920,7 @@ pub(crate) fn handle_semantic_tokens_full(
     let text = snap.analysis.file_text(file_id)?;
     let line_index = snap.file_line_index(file_id)?;
 
-    let mut highlight_config = snap.config.highlighting_config();
+    let mut highlight_config = snap.config.highlighting_config(snap.minicore());
     // Avoid flashing a bunch of unresolved references when the proc-macro servers haven't been spawned yet.
     highlight_config.syntactic_name_ref_highlighting =
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
@@ -1940,7 +1950,7 @@ pub(crate) fn handle_semantic_tokens_full_delta(
     let text = snap.analysis.file_text(file_id)?;
     let line_index = snap.file_line_index(file_id)?;
 
-    let mut highlight_config = snap.config.highlighting_config();
+    let mut highlight_config = snap.config.highlighting_config(snap.minicore());
     // Avoid flashing a bunch of unresolved references when the proc-macro servers haven't been spawned yet.
     highlight_config.syntactic_name_ref_highlighting =
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
@@ -1958,12 +1968,11 @@ pub(crate) fn handle_semantic_tokens_full_delta(
 
     if let Some(cached_tokens @ lsp_types::SemanticTokens { result_id: Some(prev_id), .. }) =
         &cached_tokens
+        && *prev_id == params.previous_result_id
     {
-        if *prev_id == params.previous_result_id {
-            let delta = to_proto::semantic_token_delta(cached_tokens, &semantic_tokens);
-            snap.semantic_tokens_cache.lock().insert(params.text_document.uri, semantic_tokens);
-            return Ok(Some(delta.into()));
-        }
+        let delta = to_proto::semantic_token_delta(cached_tokens, &semantic_tokens);
+        snap.semantic_tokens_cache.lock().insert(params.text_document.uri, semantic_tokens);
+        return Ok(Some(delta.into()));
     }
 
     // Clone first to keep the lock short
@@ -1983,7 +1992,7 @@ pub(crate) fn handle_semantic_tokens_range(
     let text = snap.analysis.file_text(frange.file_id)?;
     let line_index = snap.file_line_index(frange.file_id)?;
 
-    let mut highlight_config = snap.config.highlighting_config();
+    let mut highlight_config = snap.config.highlighting_config(snap.minicore());
     // Avoid flashing a bunch of unresolved references when the proc-macro servers haven't been spawned yet.
     highlight_config.syntactic_name_ref_highlighting =
         snap.workspaces.is_empty() || !snap.proc_macros_loaded;
@@ -2122,24 +2131,25 @@ fn show_impl_command_link(
     snap: &GlobalStateSnapshot,
     position: &FilePosition,
 ) -> Option {
-    if snap.config.hover_actions().implementations && snap.config.client_commands().show_reference {
-        if let Some(nav_data) = snap.analysis.goto_implementation(*position).unwrap_or(None) {
-            let uri = to_proto::url(snap, position.file_id);
-            let line_index = snap.file_line_index(position.file_id).ok()?;
-            let position = to_proto::position(&line_index, position.offset);
-            let locations: Vec<_> = nav_data
-                .info
-                .into_iter()
-                .filter_map(|nav| to_proto::location_from_nav(snap, nav).ok())
-                .collect();
-            let title = to_proto::implementation_title(locations.len());
-            let command = to_proto::command::show_references(title, &uri, position, locations);
+    if snap.config.hover_actions().implementations
+        && snap.config.client_commands().show_reference
+        && let Some(nav_data) = snap.analysis.goto_implementation(*position).unwrap_or(None)
+    {
+        let uri = to_proto::url(snap, position.file_id);
+        let line_index = snap.file_line_index(position.file_id).ok()?;
+        let position = to_proto::position(&line_index, position.offset);
+        let locations: Vec<_> = nav_data
+            .info
+            .into_iter()
+            .filter_map(|nav| to_proto::location_from_nav(snap, nav).ok())
+            .collect();
+        let title = to_proto::implementation_title(locations.len());
+        let command = to_proto::command::show_references(title, &uri, position, locations);
 
-            return Some(lsp_ext::CommandLinkGroup {
-                commands: vec![to_command_link(command, "Go to implementations".into())],
-                ..Default::default()
-            });
-        }
+        return Some(lsp_ext::CommandLinkGroup {
+            commands: vec![to_command_link(command, "Go to implementations".into())],
+            ..Default::default()
+        });
     }
     None
 }
@@ -2148,28 +2158,35 @@ fn show_ref_command_link(
     snap: &GlobalStateSnapshot,
     position: &FilePosition,
 ) -> Option {
-    if snap.config.hover_actions().references && snap.config.client_commands().show_reference {
-        if let Some(ref_search_res) = snap.analysis.find_all_refs(*position, None).unwrap_or(None) {
-            let uri = to_proto::url(snap, position.file_id);
-            let line_index = snap.file_line_index(position.file_id).ok()?;
-            let position = to_proto::position(&line_index, position.offset);
-            let locations: Vec<_> = ref_search_res
-                .into_iter()
-                .flat_map(|res| res.references)
-                .flat_map(|(file_id, ranges)| {
-                    ranges.into_iter().map(move |(range, _)| FileRange { file_id, range })
-                })
-                .unique()
-                .filter_map(|range| to_proto::location(snap, range).ok())
-                .collect();
-            let title = to_proto::reference_title(locations.len());
-            let command = to_proto::command::show_references(title, &uri, position, locations);
+    if snap.config.hover_actions().references
+        && snap.config.client_commands().show_reference
+        && let Some(ref_search_res) = snap
+            .analysis
+            .find_all_refs(
+                *position,
+                &FindAllRefsConfig { search_scope: None, minicore: snap.minicore() },
+            )
+            .unwrap_or(None)
+    {
+        let uri = to_proto::url(snap, position.file_id);
+        let line_index = snap.file_line_index(position.file_id).ok()?;
+        let position = to_proto::position(&line_index, position.offset);
+        let locations: Vec<_> = ref_search_res
+            .into_iter()
+            .flat_map(|res| res.references)
+            .flat_map(|(file_id, ranges)| {
+                ranges.into_iter().map(move |(range, _)| FileRange { file_id, range })
+            })
+            .unique()
+            .filter_map(|range| to_proto::location(snap, range).ok())
+            .collect();
+        let title = to_proto::reference_title(locations.len());
+        let command = to_proto::command::show_references(title, &uri, position, locations);
 
-            return Some(lsp_ext::CommandLinkGroup {
-                commands: vec![to_command_link(command, "Go to references".into())],
-                ..Default::default()
-            });
-        }
+        return Some(lsp_ext::CommandLinkGroup {
+            commands: vec![to_command_link(command, "Go to references".into())],
+            ..Default::default()
+        });
     }
     None
 }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
index 84b7888258f8..38ee9cbe7fc8 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/integrated_benchmarks.rs
@@ -16,7 +16,7 @@ use ide::{
     FilePosition, TextSize,
 };
 use ide_db::{
-    SnippetCap,
+    MiniCore, SnippetCap,
     imports::insert_use::{ImportGranularity, InsertUseConfig},
 };
 use project_model::CargoConfig;
@@ -186,6 +186,7 @@ fn integrated_completion_benchmark() {
             exclude_traits: &[],
             enable_auto_await: true,
             enable_auto_iter: true,
+            minicore: MiniCore::default(),
         };
         let position =
             FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
@@ -240,6 +241,7 @@ fn integrated_completion_benchmark() {
             exclude_traits: &[],
             enable_auto_await: true,
             enable_auto_iter: true,
+            minicore: MiniCore::default(),
         };
         let position =
             FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
@@ -292,6 +294,7 @@ fn integrated_completion_benchmark() {
             exclude_traits: &[],
             enable_auto_await: true,
             enable_auto_iter: true,
+            minicore: MiniCore::default(),
         };
         let position =
             FilePosition { file_id, offset: TextSize::try_from(completion_offset).unwrap() };
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/lib.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/lib.rs
index 0dea285e97bd..44af8fbddf30 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/lib.rs
@@ -9,6 +9,8 @@
 //! The `cli` submodule implements some batch-processing analysis, primarily as
 //! a debugging aid.
 
+extern crate ra_ap_rustc_type_ir as rustc_type_ir;
+
 /// Any toolchain less than this version will likely not work with rust-analyzer built from this revision.
 pub const MINIMUM_SUPPORTED_TOOLCHAIN_VERSION: semver::Version = semver::Version {
     major: 1,
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/from_proto.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/from_proto.rs
index 02757616d4ff..333826a1790e 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/from_proto.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/from_proto.rs
@@ -40,12 +40,13 @@ pub(crate) fn offset(
     })?;
     let col = TextSize::from(line_col.col);
     let clamped_len = col.min(line_range.len());
-    if clamped_len < col {
-        tracing::error!(
-            "Position {line_col:?} column exceeds line length {}, clamping it",
-            u32::from(line_range.len()),
-        );
-    }
+    // FIXME: The cause for this is likely our request retrying. Commented out as this log is just too chatty and very easy to trigger.
+    // if clamped_len < col {
+    //     tracing::error!(
+    //         "Position {line_col:?} column exceeds line length {}, clamping it",
+    //         u32::from(line_range.len()),
+    //     );
+    // }
     Ok(line_range.start() + clamped_len)
 }
 
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/semantic_tokens.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/semantic_tokens.rs
index 3c21e1992525..828118a0866d 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/semantic_tokens.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/semantic_tokens.rs
@@ -91,6 +91,7 @@ define_semantic_token_types![
         (LIFETIME, "lifetime"),
         (LOGICAL, "logical") => OPERATOR,
         (MACRO_BANG, "macroBang") => MACRO,
+        (NEGATION, "negation") => OPERATOR,
         (PARENTHESIS, "parenthesis"),
         (PROC_MACRO, "procMacro") => MACRO,
         (PUNCTUATION, "punctuation"),
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
index 292be1d5315d..024c13e1918d 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/lsp/to_proto.rs
@@ -16,7 +16,9 @@ use ide::{
     SnippetEdit, SourceChange, StructureNodeKind, SymbolKind, TextEdit, TextRange, TextSize,
     UpdateTest,
 };
-use ide_db::{FxHasher, assists, rust_doc::format_docs, source_change::ChangeAnnotationId};
+use ide_db::{
+    FxHasher, MiniCore, assists, rust_doc::format_docs, source_change::ChangeAnnotationId,
+};
 use itertools::Itertools;
 use paths::{Utf8Component, Utf8Prefix};
 use semver::VersionReq;
@@ -61,7 +63,7 @@ pub(crate) fn symbol_kind(symbol_kind: SymbolKind) -> lsp_types::SymbolKind {
         SymbolKind::Struct => lsp_types::SymbolKind::STRUCT,
         SymbolKind::Enum => lsp_types::SymbolKind::ENUM,
         SymbolKind::Variant => lsp_types::SymbolKind::ENUM_MEMBER,
-        SymbolKind::Trait | SymbolKind::TraitAlias => lsp_types::SymbolKind::INTERFACE,
+        SymbolKind::Trait => lsp_types::SymbolKind::INTERFACE,
         SymbolKind::Macro
         | SymbolKind::ProcMacro
         | SymbolKind::BuiltinAttr
@@ -156,7 +158,6 @@ pub(crate) fn completion_item_kind(
             SymbolKind::Static => lsp_types::CompletionItemKind::VALUE,
             SymbolKind::Struct => lsp_types::CompletionItemKind::STRUCT,
             SymbolKind::Trait => lsp_types::CompletionItemKind::INTERFACE,
-            SymbolKind::TraitAlias => lsp_types::CompletionItemKind::INTERFACE,
             SymbolKind::TypeAlias => lsp_types::CompletionItemKind::STRUCT,
             SymbolKind::TypeParam => lsp_types::CompletionItemKind::TYPE_PARAMETER,
             SymbolKind::Union => lsp_types::CompletionItemKind::STRUCT,
@@ -271,7 +272,7 @@ pub(crate) fn completion_items(
         );
     }
 
-    if let Some(limit) = config.completion(None).limit {
+    if let Some(limit) = config.completion(None, MiniCore::default()).limit {
         res.sort_by(|item1, item2| item1.sort_text.cmp(&item2.sort_text));
         res.truncate(limit);
     }
@@ -401,16 +402,17 @@ fn completion_item(
 
     set_score(&mut lsp_item, max_relevance, item.relevance);
 
-    let imports =
-        if config.completion(None).enable_imports_on_the_fly && !item.import_to_add.is_empty() {
-            item.import_to_add
-                .clone()
-                .into_iter()
-                .map(|import_path| lsp_ext::CompletionImport { full_import_path: import_path })
-                .collect()
-        } else {
-            Vec::new()
-        };
+    let imports = if config.completion(None, MiniCore::default()).enable_imports_on_the_fly
+        && !item.import_to_add.is_empty()
+    {
+        item.import_to_add
+            .clone()
+            .into_iter()
+            .map(|import_path| lsp_ext::CompletionImport { full_import_path: import_path })
+            .collect()
+    } else {
+        Vec::new()
+    };
     let (ref_resolve_data, resolve_data) = if something_to_resolve || !imports.is_empty() {
         let ref_resolve_data = if ref_match.is_some() {
             let ref_resolve_data = lsp_ext::CompletionResolveData {
@@ -494,8 +496,15 @@ pub(crate) fn signature_help(
                 .parameter_ranges()
                 .iter()
                 .map(|it| {
-                    let start = call_info.signature[..it.start().into()].chars().count() as u32;
-                    let end = call_info.signature[..it.end().into()].chars().count() as u32;
+                    let start = call_info.signature[..it.start().into()]
+                        .chars()
+                        .map(|c| c.len_utf16())
+                        .sum::() as u32;
+                    let end = start
+                        + call_info.signature[it.start().into()..it.end().into()]
+                            .chars()
+                            .map(|c| c.len_utf16())
+                            .sum::() as u32;
                     [start, end]
                 })
                 .map(|label_offsets| lsp_types::ParameterInformation {
@@ -514,9 +523,9 @@ pub(crate) fn signature_help(
                     label.push_str(", ");
                 }
                 first = false;
-                let start = label.chars().count() as u32;
+                let start = label.len() as u32;
                 label.push_str(param);
-                let end = label.chars().count() as u32;
+                let end = label.len() as u32;
                 params.push(lsp_types::ParameterInformation {
                     label: lsp_types::ParameterLabel::LabelOffsets([start, end]),
                     documentation: None,
@@ -817,7 +826,6 @@ fn semantic_token_type_and_modifiers(
             SymbolKind::Union => types::UNION,
             SymbolKind::TypeAlias => types::TYPE_ALIAS,
             SymbolKind::Trait => types::INTERFACE,
-            SymbolKind::TraitAlias => types::INTERFACE,
             SymbolKind::Macro => types::MACRO,
             SymbolKind::ProcMacro => types::PROC_MACRO,
             SymbolKind::BuiltinAttr => types::BUILTIN_ATTRIBUTE,
@@ -839,6 +847,7 @@ fn semantic_token_type_and_modifiers(
             HlOperator::Bitwise => types::BITWISE,
             HlOperator::Arithmetic => types::ARITHMETIC,
             HlOperator::Logical => types::LOGICAL,
+            HlOperator::Negation => types::NEGATION,
             HlOperator::Comparison => types::COMPARISON,
             HlOperator::Other => types::OPERATOR,
         },
@@ -909,7 +918,6 @@ pub(crate) fn folding_range(
         | FoldKind::WhereClause
         | FoldKind::ReturnType
         | FoldKind::Array
-        | FoldKind::TraitAliases
         | FoldKind::ExternCrates
         | FoldKind::MatchArm
         | FoldKind::Function => None,
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs
index 00cf890510d4..c0947b2a291e 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/main_loop.rs
@@ -20,7 +20,7 @@ use crate::{
     config::Config,
     diagnostics::{DiagnosticsGeneration, NativeDiagnosticsFetchKind, fetch_native_diagnostics},
     discover::{DiscoverArgument, DiscoverCommand, DiscoverProjectMessage},
-    flycheck::{self, FlycheckMessage},
+    flycheck::{self, ClearDiagnosticsKind, ClearScope, FlycheckMessage},
     global_state::{
         FetchBuildDataResponse, FetchWorkspaceRequest, FetchWorkspaceResponse, GlobalState,
         file_id_to_url, url_to_file_id,
@@ -501,14 +501,12 @@ impl GlobalState {
             }
         }
 
-        if self.config.cargo_autoreload_config(None)
-            || self.config.discover_workspace_config().is_some()
-        {
-            if let Some((cause, FetchWorkspaceRequest { path, force_crate_graph_reload })) =
+        if (self.config.cargo_autoreload_config(None)
+            || self.config.discover_workspace_config().is_some())
+            && let Some((cause, FetchWorkspaceRequest { path, force_crate_graph_reload })) =
                 self.fetch_workspaces_queue.should_start_op()
-            {
-                self.fetch_workspaces(cause, path, force_crate_graph_reload);
-            }
+        {
+            self.fetch_workspaces(cause, path, force_crate_graph_reload);
         }
 
         if !self.fetch_workspaces_queue.op_in_progress() {
@@ -765,33 +763,33 @@ impl GlobalState {
                 self.report_progress("Fetching", state, msg, None, None);
             }
             Task::DiscoverLinkedProjects(arg) => {
-                if let Some(cfg) = self.config.discover_workspace_config() {
-                    if !self.discover_workspace_queue.op_in_progress() {
-                        // the clone is unfortunately necessary to avoid a borrowck error when
-                        // `self.report_progress` is called later
-                        let title = &cfg.progress_label.clone();
-                        let command = cfg.command.clone();
-                        let discover = DiscoverCommand::new(self.discover_sender.clone(), command);
+                if let Some(cfg) = self.config.discover_workspace_config()
+                    && !self.discover_workspace_queue.op_in_progress()
+                {
+                    // the clone is unfortunately necessary to avoid a borrowck error when
+                    // `self.report_progress` is called later
+                    let title = &cfg.progress_label.clone();
+                    let command = cfg.command.clone();
+                    let discover = DiscoverCommand::new(self.discover_sender.clone(), command);
 
-                        self.report_progress(title, Progress::Begin, None, None, None);
-                        self.discover_workspace_queue
-                            .request_op("Discovering workspace".to_owned(), ());
-                        let _ = self.discover_workspace_queue.should_start_op();
+                    self.report_progress(title, Progress::Begin, None, None, None);
+                    self.discover_workspace_queue
+                        .request_op("Discovering workspace".to_owned(), ());
+                    let _ = self.discover_workspace_queue.should_start_op();
 
-                        let arg = match arg {
-                            DiscoverProjectParam::Buildfile(it) => DiscoverArgument::Buildfile(it),
-                            DiscoverProjectParam::Path(it) => DiscoverArgument::Path(it),
-                        };
+                    let arg = match arg {
+                        DiscoverProjectParam::Buildfile(it) => DiscoverArgument::Buildfile(it),
+                        DiscoverProjectParam::Path(it) => DiscoverArgument::Path(it),
+                    };
 
-                        let handle = discover.spawn(
-                            arg,
-                            &std::env::current_dir()
-                                .expect("Failed to get cwd during project discovery"),
-                        );
-                        self.discover_handle = Some(handle.unwrap_or_else(|e| {
-                            panic!("Failed to spawn project discovery command: {e}")
-                        }));
-                    }
+                    let handle = discover.spawn(
+                        arg,
+                        &std::env::current_dir()
+                            .expect("Failed to get cwd during project discovery"),
+                    );
+                    self.discover_handle = Some(handle.unwrap_or_else(|e| {
+                        panic!("Failed to spawn project discovery command: {e}")
+                    }));
                 }
             }
             Task::FetchBuildData(progress) => {
@@ -814,7 +812,7 @@ impl GlobalState {
                 };
 
                 if let Some(state) = state {
-                    self.report_progress("Building build-artifacts", state, msg, None, None);
+                    self.report_progress("Building compile-time-deps", state, msg, None, None);
                 }
             }
             Task::LoadProcMacros(progress) => {
@@ -849,6 +847,13 @@ impl GlobalState {
                 self.debounce_workspace_fetch();
                 let vfs = &mut self.vfs.write().0;
                 for (path, contents) in files {
+                    if matches!(path.name_and_extension(), Some(("minicore", Some("rs")))) {
+                        // Not a lot of bad can happen from mistakenly identifying `minicore`, so proceed with that.
+                        self.minicore.minicore_text = contents
+                            .as_ref()
+                            .and_then(|contents| String::from_utf8(contents.clone()).ok());
+                    }
+
                     let path = VfsPath::from(path);
                     // if the file is in mem docs, it's managed by the client via notifications
                     // so only set it if its not in there
@@ -1010,7 +1015,13 @@ impl GlobalState {
 
     fn handle_flycheck_msg(&mut self, message: FlycheckMessage) {
         match message {
-            FlycheckMessage::AddDiagnostic { id, workspace_root, diagnostic, package_id } => {
+            FlycheckMessage::AddDiagnostic {
+                id,
+                generation,
+                workspace_root,
+                diagnostic,
+                package_id,
+            } => {
                 let snap = self.snapshot();
                 let diagnostics = crate::diagnostics::to_proto::map_rust_diagnostic_to_lsp(
                     &self.config.diagnostics_map(None),
@@ -1022,6 +1033,7 @@ impl GlobalState {
                     match url_to_file_id(&self.vfs.read().0, &diag.url) {
                         Ok(Some(file_id)) => self.diagnostics.add_check_diagnostic(
                             id,
+                            generation,
                             &package_id,
                             file_id,
                             diag.diagnostic,
@@ -1037,12 +1049,22 @@ impl GlobalState {
                     };
                 }
             }
-            FlycheckMessage::ClearDiagnostics { id, package_id: None } => {
-                self.diagnostics.clear_check(id)
-            }
-            FlycheckMessage::ClearDiagnostics { id, package_id: Some(package_id) } => {
-                self.diagnostics.clear_check_for_package(id, package_id)
-            }
+            FlycheckMessage::ClearDiagnostics {
+                id,
+                kind: ClearDiagnosticsKind::All(ClearScope::Workspace),
+            } => self.diagnostics.clear_check(id),
+            FlycheckMessage::ClearDiagnostics {
+                id,
+                kind: ClearDiagnosticsKind::All(ClearScope::Package(package_id)),
+            } => self.diagnostics.clear_check_for_package(id, package_id),
+            FlycheckMessage::ClearDiagnostics {
+                id,
+                kind: ClearDiagnosticsKind::OlderThan(generation, ClearScope::Workspace),
+            } => self.diagnostics.clear_check_older_than(id, generation),
+            FlycheckMessage::ClearDiagnostics {
+                id,
+                kind: ClearDiagnosticsKind::OlderThan(generation, ClearScope::Package(package_id)),
+            } => self.diagnostics.clear_check_older_than_for_package(id, package_id, generation),
             FlycheckMessage::Progress { id, progress } => {
                 let (state, message) = match progress {
                     flycheck::Progress::DidStart => (Progress::Begin, None),
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
index e798aa6a8a60..1475f02447d2 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/reload.rs
@@ -152,7 +152,9 @@ impl GlobalState {
         if self.fetch_build_data_error().is_err() {
             status.health |= lsp_ext::Health::Warning;
             message.push_str("Failed to run build scripts of some packages.\n\n");
-            message.push_str("Please refer to the logs for more details on the errors.");
+            message.push_str(
+                "Please refer to the language server logs for more details on the errors.",
+            );
         }
         if let Some(err) = &self.config_errors {
             status.health |= lsp_ext::Health::Warning;
@@ -220,6 +222,16 @@ impl GlobalState {
                     message.push_str(err);
                     message.push_str("\n\n");
                 }
+                if let Some(err) = ws.sysroot.metadata_error() {
+                    status.health |= lsp_ext::Health::Warning;
+                    format_to!(
+                        message,
+                        "Failed to read Cargo metadata with dependencies for sysroot of `{}`: ",
+                        ws.manifest_or_root()
+                    );
+                    message.push_str(err);
+                    message.push_str("\n\n");
+                }
                 if let ProjectWorkspaceKind::Cargo { rustc: Err(Some(err)), .. } = &ws.kind {
                     status.health |= lsp_ext::Health::Warning;
                     format_to!(
@@ -306,17 +318,17 @@ impl GlobalState {
                         _ => None,
                     });
 
-                    if let Some(build) = build {
-                        if is_quiescent {
-                            let path = AbsPathBuf::try_from(build.build_file)
-                                .expect("Unable to convert to an AbsPath");
-                            let arg = DiscoverProjectParam::Buildfile(path);
-                            sender.send(Task::DiscoverLinkedProjects(arg)).unwrap();
-                        }
+                    if let Some(build) = build
+                        && is_quiescent
+                    {
+                        let path = AbsPathBuf::try_from(build.build_file)
+                            .expect("Unable to convert to an AbsPath");
+                        let arg = DiscoverProjectParam::Buildfile(path);
+                        sender.send(Task::DiscoverLinkedProjects(arg)).unwrap();
                     }
                 }
 
-                let mut workspaces = linked_projects
+                let mut workspaces: Vec<_> = linked_projects
                     .iter()
                     .map(|project| match project {
                         LinkedProject::ProjectManifest(manifest) => {
@@ -337,7 +349,7 @@ impl GlobalState {
                             Ok(workspace)
                         }
                     })
-                    .collect::>();
+                    .collect();
 
                 let mut i = 0;
                 while i < workspaces.len() {
@@ -739,13 +751,16 @@ impl GlobalState {
             })
             .collect();
 
+        self.incomplete_crate_graph = false;
         let (crate_graph, proc_macro_paths) = {
             // Create crate graph from all the workspaces
             let vfs = &self.vfs.read().0;
             let load = |path: &AbsPath| {
                 let vfs_path = vfs::VfsPath::from(path.to_path_buf());
                 self.crate_graph_file_dependencies.insert(vfs_path.clone());
-                vfs.file_id(&vfs_path).and_then(|(file_id, excluded)| {
+                let file_id = vfs.file_id(&vfs_path);
+                self.incomplete_crate_graph |= file_id.is_none();
+                file_id.and_then(|(file_id, excluded)| {
                     (excluded == vfs::FileExcluded::No).then_some(file_id)
                 })
             };
@@ -846,21 +861,17 @@ impl GlobalState {
     fn reload_flycheck(&mut self) {
         let _p = tracing::info_span!("GlobalState::reload_flycheck").entered();
         let config = self.config.flycheck(None);
-        let sender = self.flycheck_sender.clone();
-        let invocation_strategy = match config {
-            FlycheckConfig::CargoCommand { .. } => {
-                crate::flycheck::InvocationStrategy::PerWorkspace
-            }
-            FlycheckConfig::CustomCommand { ref invocation_strategy, .. } => {
-                invocation_strategy.clone()
-            }
-        };
+        let sender = &self.flycheck_sender;
+        let invocation_strategy = config.invocation_strategy();
+        let next_gen =
+            self.flycheck.iter().map(FlycheckHandle::generation).max().unwrap_or_default() + 1;
 
         self.flycheck = match invocation_strategy {
             crate::flycheck::InvocationStrategy::Once => {
                 vec![FlycheckHandle::spawn(
                     0,
-                    sender,
+                    next_gen,
+                    sender.clone(),
                     config,
                     None,
                     self.config.root_path().clone(),
@@ -898,6 +909,7 @@ impl GlobalState {
                     .map(|(id, (root, manifest_path), sysroot_root)| {
                         FlycheckHandle::spawn(
                             id,
+                            next_gen,
                             sender.clone(),
                             config.clone(),
                             sysroot_root,
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs
index 7132e09146eb..e532d155536c 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/target_spec.rs
@@ -2,12 +2,14 @@
 
 use std::mem;
 
+use cargo_metadata::PackageId;
 use cfg::{CfgAtom, CfgExpr};
 use hir::sym;
 use ide::{Cancellable, Crate, FileId, RunnableKind, TestId};
 use project_model::project_json::Runnable;
 use project_model::{CargoFeatures, ManifestPath, TargetKind};
 use rustc_hash::FxHashSet;
+use triomphe::Arc;
 use vfs::AbsPathBuf;
 
 use crate::global_state::GlobalStateSnapshot;
@@ -52,6 +54,7 @@ pub(crate) struct CargoTargetSpec {
     pub(crate) workspace_root: AbsPathBuf,
     pub(crate) cargo_toml: ManifestPath,
     pub(crate) package: String,
+    pub(crate) package_id: Arc,
     pub(crate) target: String,
     pub(crate) target_kind: TargetKind,
     pub(crate) crate_id: Crate,
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs b/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs
index e7528dbc9396..0c8658c75df0 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/src/test_runner.rs
@@ -136,7 +136,12 @@ impl CargoTestHandle {
         }
 
         Ok(Self {
-            _handle: CommandHandle::spawn(cmd, CargoTestOutputParser::new(&test_target), sender)?,
+            _handle: CommandHandle::spawn(
+                cmd,
+                CargoTestOutputParser::new(&test_target),
+                sender,
+                None,
+            )?,
         })
     }
 }
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
index 1b940c70da66..5a4ad6f380f9 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/main.rs
@@ -256,7 +256,7 @@ fn main() {}
           {
             "args": {
               "cargoArgs": ["test", "--package", "foo", "--test", "spam"],
-              "executableArgs": ["test_eggs", "--exact", "--show-output"],
+              "executableArgs": ["test_eggs", "--exact", "--nocapture"],
               "overrideCargo": null,
               "cwd": server.path().join("foo"),
               "workspaceRoot": server.path().join("foo")
@@ -289,7 +289,7 @@ fn main() {}
               ],
               "executableArgs": [
                 "",
-                "--show-output"
+                "--nocapture"
               ]
             },
             "kind": "cargo",
@@ -1061,7 +1061,7 @@ fn main() {
         ),
         work_done_progress_params: Default::default(),
     });
-    assert!(res.to_string().contains("&'static str"));
+    assert!(res.to_string().contains("&str"));
 
     let res = server.send_request::(HoverParams {
         text_document_position_params: TextDocumentPositionParams::new(
@@ -1070,7 +1070,7 @@ fn main() {
         ),
         work_done_progress_params: Default::default(),
     });
-    assert!(res.to_string().contains("&'static str"));
+    assert!(res.to_string().contains("&str"));
 
     server.request::(
         GotoDefinitionParams {
diff --git a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/support.rs b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/support.rs
index 2bebb0c1b970..3464a9644b19 100644
--- a/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/support.rs
+++ b/src/tools/rust-analyzer/crates/rust-analyzer/tests/slow-tests/support.rs
@@ -97,6 +97,7 @@ impl Project<'_> {
             proc_macro_names,
             toolchain,
             target_data_layout: _,
+            target_arch: _,
         } = FixtureWithProjectMeta::parse(self.fixture);
         assert!(proc_macro_names.is_empty());
         assert!(mini_core.is_none());
@@ -177,6 +178,7 @@ impl Project<'_> {
             proc_macro_names,
             toolchain,
             target_data_layout: _,
+            target_arch: _,
         } = FixtureWithProjectMeta::parse(self.fixture);
         assert!(proc_macro_names.is_empty());
         assert!(mini_core.is_none());
diff --git a/src/tools/rust-analyzer/crates/span/src/ast_id.rs b/src/tools/rust-analyzer/crates/span/src/ast_id.rs
index a9288ecd6fa1..e803747998b5 100644
--- a/src/tools/rust-analyzer/crates/span/src/ast_id.rs
+++ b/src/tools/rust-analyzer/crates/span/src/ast_id.rs
@@ -485,8 +485,7 @@ register_has_name_ast_id! {
         MacroRules = name,
         Module = name,
         Static = name,
-        Trait = name,
-        TraitAlias = name
+        Trait = name
 }
 
 macro_rules! register_assoc_item_ast_id {
diff --git a/src/tools/rust-analyzer/crates/span/src/hygiene.rs b/src/tools/rust-analyzer/crates/span/src/hygiene.rs
index aef3fbf05177..6c363825aa18 100644
--- a/src/tools/rust-analyzer/crates/span/src/hygiene.rs
+++ b/src/tools/rust-analyzer/crates/span/src/hygiene.rs
@@ -67,6 +67,16 @@ const _: () = {
             self.parent.hash(state);
         }
     }
+
+    impl zalsa_::HasJar for SyntaxContext {
+        type Jar = zalsa_struct_::JarImpl;
+        const KIND: zalsa_::JarKind = zalsa_::JarKind::Struct;
+    }
+
+    zalsa_::register_jar! {
+        zalsa_::ErasedJar::erase::()
+    }
+
     /// Key to use during hash lookups. Each field is some type that implements `Lookup`
     /// for the owned type. This permits interning with an `&str` when a `String` is required and so forth.
     #[derive(Hash)]
@@ -98,21 +108,38 @@ const _: () = {
             salsa::plumbing::Location { file: file!(), line: line!() };
         const DEBUG_NAME: &'static str = "SyntaxContextData";
         const REVISIONS: std::num::NonZeroUsize = std::num::NonZeroUsize::MAX;
+        const PERSIST: bool = false;
+
         type Fields<'a> = SyntaxContextData;
         type Struct<'a> = SyntaxContext;
-    }
-    impl SyntaxContext {
-        pub fn ingredient(db: &Db) -> &zalsa_struct_::IngredientImpl
+
+        fn serialize(_: &Self::Fields<'_>, _: S) -> Result
         where
-            Db: ?Sized + zalsa_::Database,
+            S: zalsa_::serde::Serializer,
         {
+            unimplemented!("attempted to serialize value that set `PERSIST` to false")
+        }
+
+        fn deserialize<'de, D>(_: D) -> Result, D::Error>
+        where
+            D: zalsa_::serde::Deserializer<'de>,
+        {
+            unimplemented!("attempted to deserialize value that cannot set `PERSIST` to false");
+        }
+    }
+
+    impl SyntaxContext {
+        pub fn ingredient(zalsa: &zalsa_::Zalsa) -> &zalsa_struct_::IngredientImpl {
             static CACHE: zalsa_::IngredientCache> =
                 zalsa_::IngredientCache::new();
-            CACHE.get_or_create(db.zalsa(), || {
-                db.zalsa()
-                    .lookup_jar_by_type::>()
-                    .get_or_create()
-            })
+
+            // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the only
+            // ingredient created by our jar is the struct ingredient.
+            unsafe {
+                CACHE.get_or_create(zalsa, || {
+                    zalsa.lookup_jar_by_type::>()
+                })
+            }
         }
     }
     impl zalsa_::AsId for SyntaxContext {
@@ -132,13 +159,14 @@ const _: () = {
     impl zalsa_::SalsaStructInDb for SyntaxContext {
         type MemoIngredientMap = salsa::plumbing::MemoIngredientSingletonIndex;
 
-        fn lookup_or_create_ingredient_index(
-            zalsa: &salsa::plumbing::Zalsa,
-        ) -> salsa::plumbing::IngredientIndices {
-            zalsa
-                .lookup_jar_by_type::>()
-                .get_or_create()
-                .into()
+        fn lookup_ingredient_index(aux: &zalsa_::Zalsa) -> salsa::plumbing::IngredientIndices {
+            aux.lookup_jar_by_type::>().into()
+        }
+
+        fn entries(zalsa: &zalsa_::Zalsa) -> impl Iterator + '_ {
+            let _ingredient_index =
+                zalsa.lookup_jar_by_type::>();
+            ::ingredient(zalsa).entries(zalsa).map(|entry| entry.key())
         }
 
         #[inline]
@@ -149,6 +177,18 @@ const _: () = {
                 None
             }
         }
+
+        #[inline]
+        unsafe fn memo_table(
+            zalsa: &zalsa_::Zalsa,
+            id: zalsa_::Id,
+            current_revision: zalsa_::Revision,
+        ) -> zalsa_::MemoTableWithTypes<'_> {
+            // SAFETY: Guaranteed by caller.
+            unsafe {
+                zalsa.table().memos::>(id, current_revision)
+            }
+        }
     }
 
     unsafe impl salsa::plumbing::Update for SyntaxContext {
@@ -184,8 +224,11 @@ const _: () = {
             Edition: zalsa_::interned::HashEqLike,
             SyntaxContext: zalsa_::interned::HashEqLike,
         {
-            SyntaxContext::ingredient(db).intern(
-                db.as_dyn_database(),
+            let (zalsa, zalsa_local) = db.zalsas();
+
+            SyntaxContext::ingredient(zalsa).intern(
+                zalsa,
+                zalsa_local,
                 StructKey::<'db>(
                     outer_expn,
                     outer_transparency,
@@ -216,7 +259,8 @@ const _: () = {
             Db: ?Sized + zalsa_::Database,
         {
             let id = self.as_salsa_id()?;
-            let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+            let zalsa = db.zalsa();
+            let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
             fields.outer_expn
         }
 
@@ -225,7 +269,8 @@ const _: () = {
             Db: ?Sized + zalsa_::Database,
         {
             let Some(id) = self.as_salsa_id() else { return Transparency::Opaque };
-            let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+            let zalsa = db.zalsa();
+            let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
             fields.outer_transparency
         }
 
@@ -235,7 +280,8 @@ const _: () = {
         {
             match self.as_salsa_id() {
                 Some(id) => {
-                    let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+                    let zalsa = db.zalsa();
+                    let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
                     fields.edition
                 }
                 None => Edition::from_u32(SyntaxContext::MAX_ID - self.into_u32()),
@@ -248,7 +294,8 @@ const _: () = {
         {
             match self.as_salsa_id() {
                 Some(id) => {
-                    let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+                    let zalsa = db.zalsa();
+                    let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
                     fields.parent
                 }
                 None => self,
@@ -262,7 +309,8 @@ const _: () = {
         {
             match self.as_salsa_id() {
                 Some(id) => {
-                    let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+                    let zalsa = db.zalsa();
+                    let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
                     fields.opaque
                 }
                 None => self,
@@ -276,7 +324,8 @@ const _: () = {
         {
             match self.as_salsa_id() {
                 Some(id) => {
-                    let fields = SyntaxContext::ingredient(db).data(db.as_dyn_database(), id);
+                    let zalsa = db.zalsa();
+                    let fields = SyntaxContext::ingredient(zalsa).data(zalsa, id);
                     fields.opaque_and_semitransparent
                 }
                 None => self,
diff --git a/src/tools/rust-analyzer/crates/span/src/lib.rs b/src/tools/rust-analyzer/crates/span/src/lib.rs
index b81d08eed6d8..ae9e038459e5 100644
--- a/src/tools/rust-analyzer/crates/span/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/span/src/lib.rs
@@ -203,15 +203,3 @@ pub struct HirFileId(pub salsa::Id);
 /// `println!("Hello, {}", world)`.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct MacroCallId(pub salsa::Id);
-
-/// Legacy span type, only defined here as it is still used by the proc-macro server.
-/// While rust-analyzer doesn't use this anymore at all, RustRover relies on the legacy type for
-/// proc-macro expansion.
-#[derive(Clone, Copy, PartialEq, Eq, Hash)]
-pub struct TokenId(pub u32);
-
-impl std::fmt::Debug for TokenId {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.0.fmt(f)
-    }
-}
diff --git a/src/tools/rust-analyzer/crates/span/src/map.rs b/src/tools/rust-analyzer/crates/span/src/map.rs
index f58201793da2..bb09933536e7 100644
--- a/src/tools/rust-analyzer/crates/span/src/map.rs
+++ b/src/tools/rust-analyzer/crates/span/src/map.rs
@@ -41,13 +41,13 @@ where
 
     /// Pushes a new span onto the [`SpanMap`].
     pub fn push(&mut self, offset: TextSize, span: SpanData) {
-        if cfg!(debug_assertions) {
-            if let Some(&(last_offset, _)) = self.spans.last() {
-                assert!(
-                    last_offset < offset,
-                    "last_offset({last_offset:?}) must be smaller than offset({offset:?})"
-                );
-            }
+        if cfg!(debug_assertions)
+            && let Some(&(last_offset, _)) = self.spans.last()
+        {
+            assert!(
+                last_offset < offset,
+                "last_offset({last_offset:?}) must be smaller than offset({offset:?})"
+            );
         }
         self.spans.push((offset, span));
     }
diff --git a/src/tools/rust-analyzer/crates/stdx/src/lib.rs b/src/tools/rust-analyzer/crates/stdx/src/lib.rs
index 978c50d807bc..5fa007416371 100644
--- a/src/tools/rust-analyzer/crates/stdx/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/stdx/src/lib.rs
@@ -187,11 +187,19 @@ pub fn is_upper_snake_case(s: &str) -> bool {
 }
 
 pub fn replace(buf: &mut String, from: char, to: &str) {
-    if !buf.contains(from) {
+    let replace_count = buf.chars().filter(|&ch| ch == from).count();
+    if replace_count == 0 {
         return;
     }
-    // FIXME: do this in place.
-    *buf = buf.replace(from, to);
+    let from_len = from.len_utf8();
+    let additional = to.len().saturating_sub(from_len);
+    buf.reserve(additional * replace_count);
+
+    let mut end = buf.len();
+    while let Some(i) = buf[..end].rfind(from) {
+        buf.replace_range(i..i + from_len, to);
+        end = i;
+    }
 }
 
 #[must_use]
@@ -343,4 +351,34 @@ mod tests {
             "fn main() {\n    return 92;\n}\n"
         );
     }
+
+    #[test]
+    fn test_replace() {
+        #[track_caller]
+        fn test_replace(src: &str, from: char, to: &str, expected: &str) {
+            let mut s = src.to_owned();
+            replace(&mut s, from, to);
+            assert_eq!(s, expected, "from: {from:?}, to: {to:?}");
+        }
+
+        test_replace("", 'a', "b", "");
+        test_replace("", 'a', "😀", "");
+        test_replace("", '😀', "a", "");
+        test_replace("a", 'a', "b", "b");
+        test_replace("aa", 'a', "b", "bb");
+        test_replace("ada", 'a', "b", "bdb");
+        test_replace("a", 'a', "😀", "😀");
+        test_replace("😀", '😀', "a", "a");
+        test_replace("😀x", '😀', "a", "ax");
+        test_replace("y😀x", '😀', "a", "yax");
+        test_replace("a,b,c", ',', ".", "a.b.c");
+        test_replace("a,b,c", ',', "..", "a..b..c");
+        test_replace("a.b.c", '.', "..", "a..b..c");
+        test_replace("a.b.c", '.', "..", "a..b..c");
+        test_replace("a😀b😀c", '😀', ".", "a.b.c");
+        test_replace("a.b.c", '.', "😀", "a😀b😀c");
+        test_replace("a.b.c", '.', "😀😀", "a😀😀b😀😀c");
+        test_replace(".a.b.c.", '.', "()", "()a()b()c()");
+        test_replace(".a.b.c.", '.', "", "abc");
+    }
 }
diff --git a/src/tools/rust-analyzer/crates/stdx/src/thread.rs b/src/tools/rust-analyzer/crates/stdx/src/thread.rs
index a34e9e4a6551..37b7a9f5edfa 100644
--- a/src/tools/rust-analyzer/crates/stdx/src/thread.rs
+++ b/src/tools/rust-analyzer/crates/stdx/src/thread.rs
@@ -101,7 +101,6 @@ impl Drop for JoinHandle {
     }
 }
 
-#[expect(clippy::min_ident_chars, reason = "trait impl")]
 impl fmt::Debug for JoinHandle {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.pad("JoinHandle { .. }")
diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs
index d59229952f52..4e525be3fe3c 100644
--- a/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/lib.rs
@@ -127,11 +127,11 @@ where
 
 // The following items are what `rustc` macro can be parsed into :
 // link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
-// * Expr(P)                     -> token_tree_to_expr
-// * Pat(P)                       -> token_tree_to_pat
-// * Ty(P)                         -> token_tree_to_ty
+// * Expr(Box)                     -> token_tree_to_expr
+// * Pat(Box)                       -> token_tree_to_pat
+// * Ty(Box)                         -> token_tree_to_ty
 // * Stmts(SmallVec<[ast::Stmt; 1]>)        -> token_tree_to_stmts
-// * Items(SmallVec<[P; 1]>)     -> token_tree_to_items
+// * Items(SmallVec<[Box; 1]>)     -> token_tree_to_items
 //
 // * TraitItems(SmallVec<[ast::TraitItem; 1]>)
 // * AssocItems(SmallVec<[ast::AssocItem; 1]>)
@@ -768,17 +768,17 @@ where
     }
 
     fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
-        if let Some((punct, offset)) = self.punct_offset.clone() {
-            if usize::from(offset) + 1 < punct.text().len() {
-                let offset = offset + TextSize::of('.');
-                let range = punct.text_range();
-                self.punct_offset = Some((punct.clone(), offset));
-                let range = TextRange::at(range.start() + offset, TextSize::of('.'));
-                return Some((
-                    SynToken::Punct { token: punct, offset: u32::from(offset) as usize },
-                    range,
-                ));
-            }
+        if let Some((punct, offset)) = self.punct_offset.clone()
+            && usize::from(offset) + 1 < punct.text().len()
+        {
+            let offset = offset + TextSize::of('.');
+            let range = punct.text_range();
+            self.punct_offset = Some((punct.clone(), offset));
+            let range = TextRange::at(range.start() + offset, TextSize::of('.'));
+            return Some((
+                SynToken::Punct { token: punct, offset: u32::from(offset) as usize },
+                range,
+            ));
         }
 
         if let Some(leaf) = self.current_leaves.pop() {
diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/prettify_macro_expansion.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/prettify_macro_expansion.rs
index 0a5c8df0d0ae..2f932e045832 100644
--- a/src/tools/rust-analyzer/crates/syntax-bridge/src/prettify_macro_expansion.rs
+++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/prettify_macro_expansion.rs
@@ -61,10 +61,11 @@ pub fn prettify_macro_expansion(
             }
             _ => continue,
         };
-        if token.kind() == SyntaxKind::IDENT && token.text() == "$crate" {
-            if let Some(replacement) = dollar_crate_replacement(&token) {
-                dollar_crate_replacements.push((token.clone(), replacement));
-            }
+        if token.kind() == SyntaxKind::IDENT
+            && token.text() == "$crate"
+            && let Some(replacement) = dollar_crate_replacement(&token)
+        {
+            dollar_crate_replacements.push((token.clone(), replacement));
         }
         let tok = &token;
 
diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs
index 8871bf56a5df..c8dc3131b59c 100644
--- a/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs
+++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/tests.rs
@@ -34,14 +34,11 @@ fn check_punct_spacing(fixture: &str) {
     while !cursor.eof() {
         while let Some(token_tree) = cursor.token_tree() {
             if let tt::TokenTree::Leaf(Leaf::Punct(Punct {
-                spacing,
-                span: Span { range, .. },
-                ..
+                spacing, span: Span { range, .. }, ..
             })) = token_tree
+                && let Some(expected) = annotations.remove(range)
             {
-                if let Some(expected) = annotations.remove(range) {
-                    assert_eq!(expected, *spacing);
-                }
+                assert_eq!(expected, *spacing);
             }
             cursor.bump();
         }
diff --git a/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs b/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs
index 021dc6595f9b..c0ff8e1db2c2 100644
--- a/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs
+++ b/src/tools/rust-analyzer/crates/syntax-bridge/src/to_parser_input.rs
@@ -21,17 +21,17 @@ pub fn to_parser_input(
         let tt = current.token_tree();
 
         // Check if it is lifetime
-        if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tt {
-            if punct.char == '\'' {
-                current.bump();
-                match current.token_tree() {
-                    Some(tt::TokenTree::Leaf(tt::Leaf::Ident(_ident))) => {
-                        res.push(LIFETIME_IDENT);
-                        current.bump();
-                        continue;
-                    }
-                    _ => panic!("Next token must be ident"),
+        if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tt
+            && punct.char == '\''
+        {
+            current.bump();
+            match current.token_tree() {
+                Some(tt::TokenTree::Leaf(tt::Leaf::Ident(_ident))) => {
+                    res.push(LIFETIME_IDENT);
+                    current.bump();
+                    continue;
                 }
+                _ => panic!("Next token must be ident"),
             }
         }
 
diff --git a/src/tools/rust-analyzer/crates/syntax/rust.ungram b/src/tools/rust-analyzer/crates/syntax/rust.ungram
index 6d8a360d715b..d73d60c51f0c 100644
--- a/src/tools/rust-analyzer/crates/syntax/rust.ungram
+++ b/src/tools/rust-analyzer/crates/syntax/rust.ungram
@@ -154,7 +154,6 @@ Item =
 | Static
 | Struct
 | Trait
-| TraitAlias
 | TypeAlias
 | Union
 | Use
@@ -306,11 +305,8 @@ Trait =
   Attr* Visibility?
   'unsafe'? 'auto'?
   'trait' Name GenericParamList?
-  (':' TypeBoundList?)? WhereClause? AssocItemList
-
-TraitAlias =
-  Attr* Visibility?
-  'trait' Name GenericParamList? '=' TypeBoundList? WhereClause? ';'
+  (((':' TypeBoundList?)? WhereClause? AssocItemList) |
+  ('=' TypeBoundList? WhereClause? ';'))
 
 AssocItemList =
   '{' Attr* AssocItem* '}'
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast.rs b/src/tools/rust-analyzer/crates/syntax/src/ast.rs
index a9aeeedb6542..aea99a4389b9 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast.rs
@@ -26,11 +26,12 @@ pub use self::{
     generated::{nodes::*, tokens::*},
     node_ext::{
         AttrKind, FieldKind, Macro, NameLike, NameOrNameRef, PathSegmentKind, SelfParamKind,
-        SlicePatComponents, StructKind, TraitOrAlias, TypeBoundKind, TypeOrConstParam,
-        VisibilityKind,
+        SlicePatComponents, StructKind, TypeBoundKind, TypeOrConstParam, VisibilityKind,
     },
     operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp},
-    token_ext::{CommentKind, CommentPlacement, CommentShape, IsString, QuoteOffsets, Radix},
+    token_ext::{
+        AnyString, CommentKind, CommentPlacement, CommentShape, IsString, QuoteOffsets, Radix,
+    },
     traits::{
         AttrDocCommentIter, DocCommentIter, HasArgList, HasAttrs, HasDocComments, HasGenericArgs,
         HasGenericParams, HasLoopBody, HasModuleItem, HasName, HasTypeBounds, HasVisibility,
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/edit.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/edit.rs
index d97fdec524fb..9b30642fe4b0 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/edit.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/edit.rs
@@ -89,11 +89,11 @@ impl IndentLevel {
             _ => None,
         });
         for token in tokens {
-            if let Some(ws) = ast::Whitespace::cast(token) {
-                if ws.text().contains('\n') {
-                    let new_ws = make::tokens::whitespace(&format!("{}{self}", ws.syntax()));
-                    ted::replace(ws.syntax(), &new_ws);
-                }
+            if let Some(ws) = ast::Whitespace::cast(token)
+                && ws.text().contains('\n')
+            {
+                let new_ws = make::tokens::whitespace(&format!("{}{self}", ws.syntax()));
+                ted::replace(ws.syntax(), &new_ws);
             }
         }
     }
@@ -122,13 +122,13 @@ impl IndentLevel {
             _ => None,
         });
         for token in tokens {
-            if let Some(ws) = ast::Whitespace::cast(token) {
-                if ws.text().contains('\n') {
-                    let new_ws = make::tokens::whitespace(
-                        &ws.syntax().text().replace(&format!("\n{self}"), "\n"),
-                    );
-                    ted::replace(ws.syntax(), &new_ws);
-                }
+            if let Some(ws) = ast::Whitespace::cast(token)
+                && ws.text().contains('\n')
+            {
+                let new_ws = make::tokens::whitespace(
+                    &ws.syntax().text().replace(&format!("\n{self}"), "\n"),
+                );
+                ted::replace(ws.syntax(), &new_ws);
             }
         }
     }
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/edit_in_place.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/edit_in_place.rs
index 28b543ea7064..1cd8146f6863 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/edit_in_place.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/edit_in_place.rs
@@ -9,11 +9,11 @@ use crate::{
     SyntaxKind::{ATTR, COMMENT, WHITESPACE},
     SyntaxNode, SyntaxToken,
     algo::{self, neighbor},
-    ast::{self, HasGenericArgs, HasGenericParams, edit::IndentLevel, make},
+    ast::{self, HasGenericParams, edit::IndentLevel, make},
     ted::{self, Position},
 };
 
-use super::{GenericParam, HasArgList, HasName};
+use super::{GenericParam, HasName};
 
 pub trait GenericParamsOwnerEdit: ast::HasGenericParams {
     fn get_or_create_generic_param_list(&self) -> ast::GenericParamList;
@@ -99,38 +99,10 @@ impl GenericParamsOwnerEdit for ast::Trait {
 
     fn get_or_create_where_clause(&self) -> ast::WhereClause {
         if self.where_clause().is_none() {
-            let position = match self.assoc_item_list() {
-                Some(items) => Position::before(items.syntax()),
-                None => Position::last_child_of(self.syntax()),
-            };
-            create_where_clause(position);
-        }
-        self.where_clause().unwrap()
-    }
-}
-
-impl GenericParamsOwnerEdit for ast::TraitAlias {
-    fn get_or_create_generic_param_list(&self) -> ast::GenericParamList {
-        match self.generic_param_list() {
-            Some(it) => it,
-            None => {
-                let position = if let Some(name) = self.name() {
-                    Position::after(name.syntax)
-                } else if let Some(trait_token) = self.trait_token() {
-                    Position::after(trait_token)
-                } else {
-                    Position::last_child_of(self.syntax())
-                };
-                create_generic_param_list(position)
-            }
-        }
-    }
-
-    fn get_or_create_where_clause(&self) -> ast::WhereClause {
-        if self.where_clause().is_none() {
-            let position = match self.semicolon_token() {
-                Some(tok) => Position::before(tok),
-                None => Position::last_child_of(self.syntax()),
+            let position = match (self.assoc_item_list(), self.semicolon_token()) {
+                (Some(items), _) => Position::before(items.syntax()),
+                (_, Some(tok)) => Position::before(tok),
+                (None, None) => Position::last_child_of(self.syntax()),
             };
             create_where_clause(position);
         }
@@ -273,28 +245,6 @@ pub trait AttrsOwnerEdit: ast::HasAttrs {
             }
         }
     }
-
-    fn add_attr(&self, attr: ast::Attr) {
-        add_attr(self.syntax(), attr);
-
-        fn add_attr(node: &SyntaxNode, attr: ast::Attr) {
-            let indent = IndentLevel::from_node(node);
-            attr.reindent_to(indent);
-
-            let after_attrs_and_comments = node
-                .children_with_tokens()
-                .find(|it| !matches!(it.kind(), WHITESPACE | COMMENT | ATTR))
-                .map_or(Position::first_child_of(node), Position::before);
-
-            ted::insert_all(
-                after_attrs_and_comments,
-                vec![
-                    attr.syntax().clone().into(),
-                    make::tokens::whitespace(&format!("\n{indent}")).into(),
-                ],
-            )
-        }
-    }
 }
 
 impl AttrsOwnerEdit for T {}
@@ -383,10 +333,10 @@ impl ast::GenericParamList {
 
 impl ast::WhereClause {
     pub fn add_predicate(&self, predicate: ast::WherePred) {
-        if let Some(pred) = self.predicates().last() {
-            if !pred.syntax().siblings_with_tokens(Direction::Next).any(|it| it.kind() == T![,]) {
-                ted::append_child_raw(self.syntax(), make::token(T![,]));
-            }
+        if let Some(pred) = self.predicates().last()
+            && !pred.syntax().siblings_with_tokens(Direction::Next).any(|it| it.kind() == T![,])
+        {
+            ted::append_child_raw(self.syntax(), make::token(T![,]));
         }
         ted::append_child(self.syntax(), predicate.syntax());
     }
@@ -419,34 +369,6 @@ impl Removable for ast::TypeBoundList {
     }
 }
 
-impl ast::PathSegment {
-    pub fn get_or_create_generic_arg_list(&self) -> ast::GenericArgList {
-        if self.generic_arg_list().is_none() {
-            let arg_list = make::generic_arg_list(empty()).clone_for_update();
-            ted::append_child(self.syntax(), arg_list.syntax());
-        }
-        self.generic_arg_list().unwrap()
-    }
-}
-
-impl ast::MethodCallExpr {
-    pub fn get_or_create_generic_arg_list(&self) -> ast::GenericArgList {
-        if self.generic_arg_list().is_none() {
-            let generic_arg_list = make::turbofish_generic_arg_list(empty()).clone_for_update();
-
-            if let Some(arg_list) = self.arg_list() {
-                ted::insert_raw(
-                    ted::Position::before(arg_list.syntax()),
-                    generic_arg_list.syntax(),
-                );
-            } else {
-                ted::append_child(self.syntax(), generic_arg_list.syntax());
-            }
-        }
-        self.generic_arg_list().unwrap()
-    }
-}
-
 impl Removable for ast::UseTree {
     fn remove(&self) {
         for dir in [Direction::Next, Direction::Prev] {
@@ -677,106 +599,6 @@ impl ast::AssocItemList {
         ];
         ted::insert_all(position, elements);
     }
-
-    /// Adds a new associated item at the start of the associated item list.
-    ///
-    /// Attention! This function does align the first line of `item` with respect to `self`,
-    /// but it does _not_ change indentation of other lines (if any).
-    pub fn add_item_at_start(&self, item: ast::AssocItem) {
-        match self.assoc_items().next() {
-            Some(first_item) => {
-                let indent = IndentLevel::from_node(first_item.syntax());
-                let before = Position::before(first_item.syntax());
-
-                ted::insert_all(
-                    before,
-                    vec![
-                        item.syntax().clone().into(),
-                        make::tokens::whitespace(&format!("\n\n{indent}")).into(),
-                    ],
-                )
-            }
-            None => {
-                let (indent, position, whitespace) = match self.l_curly_token() {
-                    Some(l_curly) => {
-                        normalize_ws_between_braces(self.syntax());
-                        (IndentLevel::from_token(&l_curly) + 1, Position::after(&l_curly), "\n")
-                    }
-                    None => (IndentLevel::single(), Position::first_child_of(self.syntax()), ""),
-                };
-
-                let mut elements = vec![];
-
-                // Avoid pushing an empty whitespace token
-                if !indent.is_zero() || !whitespace.is_empty() {
-                    elements.push(make::tokens::whitespace(&format!("{whitespace}{indent}")).into())
-                }
-                elements.push(item.syntax().clone().into());
-
-                ted::insert_all(position, elements)
-            }
-        };
-    }
-}
-
-impl ast::Fn {
-    pub fn get_or_create_body(&self) -> ast::BlockExpr {
-        if self.body().is_none() {
-            let body = make::ext::empty_block_expr().clone_for_update();
-            match self.semicolon_token() {
-                Some(semi) => {
-                    ted::replace(semi, body.syntax());
-                    ted::insert(Position::before(body.syntax), make::tokens::single_space());
-                }
-                None => ted::append_child(self.syntax(), body.syntax()),
-            }
-        }
-        self.body().unwrap()
-    }
-}
-
-impl ast::LetStmt {
-    pub fn set_ty(&self, ty: Option) {
-        match ty {
-            None => {
-                if let Some(colon_token) = self.colon_token() {
-                    ted::remove(colon_token);
-                }
-
-                if let Some(existing_ty) = self.ty() {
-                    if let Some(sibling) = existing_ty.syntax().prev_sibling_or_token() {
-                        if sibling.kind() == SyntaxKind::WHITESPACE {
-                            ted::remove(sibling);
-                        }
-                    }
-
-                    ted::remove(existing_ty.syntax());
-                }
-
-                // Remove any trailing ws
-                if let Some(last) = self.syntax().last_token().filter(|it| it.kind() == WHITESPACE)
-                {
-                    last.detach();
-                }
-            }
-            Some(new_ty) => {
-                if self.colon_token().is_none() {
-                    ted::insert_raw(
-                        Position::after(
-                            self.pat().expect("let stmt should have a pattern").syntax(),
-                        ),
-                        make::token(T![:]),
-                    );
-                }
-
-                if let Some(old_ty) = self.ty() {
-                    ted::replace(old_ty.syntax(), new_ty.syntax());
-                } else {
-                    ted::insert(Position::after(self.colon_token().unwrap()), new_ty.syntax());
-                }
-            }
-        }
-    }
 }
 
 impl ast::RecordExprFieldList {
@@ -823,19 +645,18 @@ impl ast::RecordExprField {
             return;
         }
         // this is a shorthand
-        if let Some(ast::Expr::PathExpr(path_expr)) = self.expr() {
-            if let Some(path) = path_expr.path() {
-                if let Some(name_ref) = path.as_single_name_ref() {
-                    path_expr.syntax().detach();
-                    let children = vec![
-                        name_ref.syntax().clone().into(),
-                        ast::make::token(T![:]).into(),
-                        ast::make::tokens::single_space().into(),
-                        expr.syntax().clone().into(),
-                    ];
-                    ted::insert_all_raw(Position::last_child_of(self.syntax()), children);
-                }
-            }
+        if let Some(ast::Expr::PathExpr(path_expr)) = self.expr()
+            && let Some(path) = path_expr.path()
+            && let Some(name_ref) = path.as_single_name_ref()
+        {
+            path_expr.syntax().detach();
+            let children = vec![
+                name_ref.syntax().clone().into(),
+                ast::make::token(T![:]).into(),
+                ast::make::tokens::single_space().into(),
+                expr.syntax().clone().into(),
+            ];
+            ted::insert_all_raw(Position::last_child_of(self.syntax()), children);
         }
     }
 }
@@ -1092,35 +913,4 @@ mod tests {
         check("let a @ ()", "let a", None);
         check("let a @ ", "let a", None);
     }
-
-    #[test]
-    fn test_let_stmt_set_ty() {
-        #[track_caller]
-        fn check(before: &str, expected: &str, ty: Option) {
-            let ty = ty.map(|it| it.clone_for_update());
-
-            let let_stmt = ast_mut_from_text::(&format!("fn f() {{ {before} }}"));
-            let_stmt.set_ty(ty);
-
-            let after = ast_mut_from_text::(&format!("fn f() {{ {expected} }}"));
-            assert_eq!(let_stmt.to_string(), after.to_string(), "{let_stmt:#?}\n!=\n{after:#?}");
-        }
-
-        // adding
-        check("let a;", "let a: ();", Some(make::ty_tuple([])));
-        // no semicolon due to it being eaten during error recovery
-        check("let a:", "let a: ()", Some(make::ty_tuple([])));
-
-        // replacing
-        check("let a: u8;", "let a: ();", Some(make::ty_tuple([])));
-        check("let a: u8 = 3;", "let a: () = 3;", Some(make::ty_tuple([])));
-        check("let a: = 3;", "let a: () = 3;", Some(make::ty_tuple([])));
-
-        // removing
-        check("let a: u8;", "let a;", None);
-        check("let a:;", "let a;", None);
-
-        check("let a: u8 = 3;", "let a = 3;", None);
-        check("let a: = 3;", "let a = 3;", None);
-    }
 }
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
index ceb2866ebcdf..6c1dcf336ac5 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/nodes.rs
@@ -2,9 +2,9 @@
 
 #![allow(non_snake_case)]
 use crate::{
-    ast::{self, support, AstChildren, AstNode},
     SyntaxKind::{self, *},
     SyntaxNode, SyntaxToken, T,
+    ast::{self, AstChildren, AstNode, support},
 };
 use std::{fmt, hash};
 pub struct Abi {
@@ -1614,30 +1614,16 @@ impl Trait {
     #[inline]
     pub fn assoc_item_list(&self) -> Option { support::child(&self.syntax) }
     #[inline]
+    pub fn semicolon_token(&self) -> Option { support::token(&self.syntax, T![;]) }
+    #[inline]
+    pub fn eq_token(&self) -> Option { support::token(&self.syntax, T![=]) }
+    #[inline]
     pub fn auto_token(&self) -> Option { support::token(&self.syntax, T![auto]) }
     #[inline]
     pub fn trait_token(&self) -> Option { support::token(&self.syntax, T![trait]) }
     #[inline]
     pub fn unsafe_token(&self) -> Option { support::token(&self.syntax, T![unsafe]) }
 }
-pub struct TraitAlias {
-    pub(crate) syntax: SyntaxNode,
-}
-impl ast::HasAttrs for TraitAlias {}
-impl ast::HasDocComments for TraitAlias {}
-impl ast::HasGenericParams for TraitAlias {}
-impl ast::HasName for TraitAlias {}
-impl ast::HasVisibility for TraitAlias {}
-impl TraitAlias {
-    #[inline]
-    pub fn type_bound_list(&self) -> Option { support::child(&self.syntax) }
-    #[inline]
-    pub fn semicolon_token(&self) -> Option { support::token(&self.syntax, T![;]) }
-    #[inline]
-    pub fn eq_token(&self) -> Option { support::token(&self.syntax, T![=]) }
-    #[inline]
-    pub fn trait_token(&self) -> Option { support::token(&self.syntax, T![trait]) }
-}
 pub struct TryExpr {
     pub(crate) syntax: SyntaxNode,
 }
@@ -2107,7 +2093,6 @@ pub enum Item {
     Static(Static),
     Struct(Struct),
     Trait(Trait),
-    TraitAlias(TraitAlias),
     TypeAlias(TypeAlias),
     Union(Union),
     Use(Use),
@@ -2277,11 +2262,7 @@ impl AstNode for Abi {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ABI }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2313,11 +2294,7 @@ impl AstNode for ArgList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ARG_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2349,11 +2326,7 @@ impl AstNode for ArrayExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ARRAY_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2385,11 +2358,7 @@ impl AstNode for ArrayType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ARRAY_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2421,11 +2390,7 @@ impl AstNode for AsmClobberAbi {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_CLOBBER_ABI }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2457,11 +2422,7 @@ impl AstNode for AsmConst {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_CONST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2493,11 +2454,7 @@ impl AstNode for AsmDirSpec {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_DIR_SPEC }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2529,11 +2486,7 @@ impl AstNode for AsmExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2565,11 +2518,7 @@ impl AstNode for AsmLabel {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_LABEL }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2601,11 +2550,7 @@ impl AstNode for AsmOperandExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_OPERAND_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2637,11 +2582,7 @@ impl AstNode for AsmOperandNamed {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_OPERAND_NAMED }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2673,11 +2614,7 @@ impl AstNode for AsmOption {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_OPTION }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2709,11 +2646,7 @@ impl AstNode for AsmOptions {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_OPTIONS }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2745,11 +2678,7 @@ impl AstNode for AsmRegOperand {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_REG_OPERAND }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2781,11 +2710,7 @@ impl AstNode for AsmRegSpec {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_REG_SPEC }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2817,11 +2742,7 @@ impl AstNode for AsmSym {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASM_SYM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2853,11 +2774,7 @@ impl AstNode for AssocItemList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASSOC_ITEM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2889,11 +2806,7 @@ impl AstNode for AssocTypeArg {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ASSOC_TYPE_ARG }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2925,11 +2838,7 @@ impl AstNode for Attr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ATTR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2961,11 +2870,7 @@ impl AstNode for AwaitExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == AWAIT_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -2997,11 +2902,7 @@ impl AstNode for BecomeExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BECOME_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3033,11 +2934,7 @@ impl AstNode for BinExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BIN_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3069,11 +2966,7 @@ impl AstNode for BlockExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BLOCK_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3105,11 +2998,7 @@ impl AstNode for BoxPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BOX_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3141,11 +3030,7 @@ impl AstNode for BreakExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BREAK_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3177,11 +3062,7 @@ impl AstNode for CallExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CALL_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3213,11 +3094,7 @@ impl AstNode for CastExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CAST_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3249,11 +3126,7 @@ impl AstNode for ClosureExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CLOSURE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3285,11 +3158,7 @@ impl AstNode for Const {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CONST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3321,11 +3190,7 @@ impl AstNode for ConstArg {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CONST_ARG }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3357,11 +3222,7 @@ impl AstNode for ConstBlockPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CONST_BLOCK_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3393,11 +3254,7 @@ impl AstNode for ConstParam {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CONST_PARAM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3429,11 +3286,7 @@ impl AstNode for ContinueExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CONTINUE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3465,11 +3318,7 @@ impl AstNode for DynTraitType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == DYN_TRAIT_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3501,11 +3350,7 @@ impl AstNode for Enum {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ENUM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3537,11 +3382,7 @@ impl AstNode for ExprStmt {
     fn can_cast(kind: SyntaxKind) -> bool { kind == EXPR_STMT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3573,11 +3414,7 @@ impl AstNode for ExternBlock {
     fn can_cast(kind: SyntaxKind) -> bool { kind == EXTERN_BLOCK }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3609,11 +3446,7 @@ impl AstNode for ExternCrate {
     fn can_cast(kind: SyntaxKind) -> bool { kind == EXTERN_CRATE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3645,11 +3478,7 @@ impl AstNode for ExternItemList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == EXTERN_ITEM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3681,11 +3510,7 @@ impl AstNode for FieldExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FIELD_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3717,11 +3542,7 @@ impl AstNode for Fn {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FN }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3753,11 +3574,7 @@ impl AstNode for FnPtrType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FN_PTR_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3789,11 +3606,7 @@ impl AstNode for ForBinder {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FOR_BINDER }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3825,11 +3638,7 @@ impl AstNode for ForExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FOR_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3861,11 +3670,7 @@ impl AstNode for ForType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FOR_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3897,11 +3702,7 @@ impl AstNode for FormatArgsArg {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FORMAT_ARGS_ARG }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3933,11 +3734,7 @@ impl AstNode for FormatArgsExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FORMAT_ARGS_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -3969,11 +3766,7 @@ impl AstNode for GenericArgList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == GENERIC_ARG_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4005,11 +3798,7 @@ impl AstNode for GenericParamList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == GENERIC_PARAM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4041,11 +3830,7 @@ impl AstNode for IdentPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == IDENT_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4077,11 +3862,7 @@ impl AstNode for IfExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == IF_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4113,11 +3894,7 @@ impl AstNode for Impl {
     fn can_cast(kind: SyntaxKind) -> bool { kind == IMPL }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4149,11 +3926,7 @@ impl AstNode for ImplTraitType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == IMPL_TRAIT_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4185,11 +3958,7 @@ impl AstNode for IndexExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == INDEX_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4221,11 +3990,7 @@ impl AstNode for InferType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == INFER_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4257,11 +4022,7 @@ impl AstNode for ItemList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == ITEM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4293,11 +4054,7 @@ impl AstNode for Label {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LABEL }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4329,11 +4086,7 @@ impl AstNode for LetElse {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LET_ELSE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4365,11 +4118,7 @@ impl AstNode for LetExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LET_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4401,11 +4150,7 @@ impl AstNode for LetStmt {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LET_STMT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4437,11 +4182,7 @@ impl AstNode for Lifetime {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LIFETIME }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4473,11 +4214,7 @@ impl AstNode for LifetimeArg {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LIFETIME_ARG }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4509,11 +4246,7 @@ impl AstNode for LifetimeParam {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LIFETIME_PARAM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4545,11 +4278,7 @@ impl AstNode for Literal {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LITERAL }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4581,11 +4310,7 @@ impl AstNode for LiteralPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LITERAL_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4617,11 +4342,7 @@ impl AstNode for LoopExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == LOOP_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4653,11 +4374,7 @@ impl AstNode for MacroCall {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_CALL }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4689,11 +4406,7 @@ impl AstNode for MacroDef {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_DEF }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4725,11 +4438,7 @@ impl AstNode for MacroExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4761,11 +4470,7 @@ impl AstNode for MacroItems {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_ITEMS }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4797,11 +4502,7 @@ impl AstNode for MacroPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4833,11 +4534,7 @@ impl AstNode for MacroRules {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_RULES }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4869,11 +4566,7 @@ impl AstNode for MacroStmts {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_STMTS }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4905,11 +4598,7 @@ impl AstNode for MacroType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MACRO_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4941,11 +4630,7 @@ impl AstNode for MatchArm {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MATCH_ARM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -4977,11 +4662,7 @@ impl AstNode for MatchArmList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MATCH_ARM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5013,11 +4694,7 @@ impl AstNode for MatchExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MATCH_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5049,11 +4726,7 @@ impl AstNode for MatchGuard {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MATCH_GUARD }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5085,11 +4758,7 @@ impl AstNode for Meta {
     fn can_cast(kind: SyntaxKind) -> bool { kind == META }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5121,11 +4790,7 @@ impl AstNode for MethodCallExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == METHOD_CALL_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5157,11 +4822,7 @@ impl AstNode for Module {
     fn can_cast(kind: SyntaxKind) -> bool { kind == MODULE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5193,11 +4854,7 @@ impl AstNode for Name {
     fn can_cast(kind: SyntaxKind) -> bool { kind == NAME }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5229,11 +4886,7 @@ impl AstNode for NameRef {
     fn can_cast(kind: SyntaxKind) -> bool { kind == NAME_REF }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5265,11 +4918,7 @@ impl AstNode for NeverType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == NEVER_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5301,11 +4950,7 @@ impl AstNode for OffsetOfExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == OFFSET_OF_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5337,11 +4982,7 @@ impl AstNode for OrPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == OR_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5373,11 +5014,7 @@ impl AstNode for Param {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PARAM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5409,11 +5046,7 @@ impl AstNode for ParamList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PARAM_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5445,11 +5078,7 @@ impl AstNode for ParenExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PAREN_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5481,11 +5110,7 @@ impl AstNode for ParenPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PAREN_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5517,11 +5142,7 @@ impl AstNode for ParenType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PAREN_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5553,11 +5174,7 @@ impl AstNode for ParenthesizedArgList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PARENTHESIZED_ARG_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5589,11 +5206,7 @@ impl AstNode for Path {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PATH }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5625,11 +5238,7 @@ impl AstNode for PathExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PATH_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5661,11 +5270,7 @@ impl AstNode for PathPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PATH_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5697,11 +5302,7 @@ impl AstNode for PathSegment {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PATH_SEGMENT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5733,11 +5334,7 @@ impl AstNode for PathType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PATH_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5769,11 +5366,7 @@ impl AstNode for PrefixExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PREFIX_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5805,11 +5398,7 @@ impl AstNode for PtrType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == PTR_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5841,11 +5430,7 @@ impl AstNode for RangeExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RANGE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5877,11 +5462,7 @@ impl AstNode for RangePat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RANGE_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5913,11 +5494,7 @@ impl AstNode for RecordExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5949,11 +5526,7 @@ impl AstNode for RecordExprField {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_EXPR_FIELD }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -5985,11 +5558,7 @@ impl AstNode for RecordExprFieldList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_EXPR_FIELD_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6021,11 +5590,7 @@ impl AstNode for RecordField {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_FIELD }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6057,11 +5622,7 @@ impl AstNode for RecordFieldList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_FIELD_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6093,11 +5654,7 @@ impl AstNode for RecordPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6129,11 +5686,7 @@ impl AstNode for RecordPatField {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_PAT_FIELD }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6165,11 +5718,7 @@ impl AstNode for RecordPatFieldList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RECORD_PAT_FIELD_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6201,11 +5750,7 @@ impl AstNode for RefExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == REF_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6237,11 +5782,7 @@ impl AstNode for RefPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == REF_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6273,11 +5814,7 @@ impl AstNode for RefType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == REF_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6309,11 +5846,7 @@ impl AstNode for Rename {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RENAME }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6345,11 +5878,7 @@ impl AstNode for RestPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == REST_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6381,11 +5910,7 @@ impl AstNode for RetType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RET_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6417,11 +5942,7 @@ impl AstNode for ReturnExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RETURN_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6453,11 +5974,7 @@ impl AstNode for ReturnTypeSyntax {
     fn can_cast(kind: SyntaxKind) -> bool { kind == RETURN_TYPE_SYNTAX }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6489,11 +6006,7 @@ impl AstNode for SelfParam {
     fn can_cast(kind: SyntaxKind) -> bool { kind == SELF_PARAM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6525,11 +6038,7 @@ impl AstNode for SlicePat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == SLICE_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6561,11 +6070,7 @@ impl AstNode for SliceType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == SLICE_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6597,11 +6102,7 @@ impl AstNode for SourceFile {
     fn can_cast(kind: SyntaxKind) -> bool { kind == SOURCE_FILE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6633,11 +6134,7 @@ impl AstNode for Static {
     fn can_cast(kind: SyntaxKind) -> bool { kind == STATIC }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6669,11 +6166,7 @@ impl AstNode for StmtList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6705,11 +6198,7 @@ impl AstNode for Struct {
     fn can_cast(kind: SyntaxKind) -> bool { kind == STRUCT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6741,11 +6230,7 @@ impl AstNode for TokenTree {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TOKEN_TREE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6777,11 +6262,7 @@ impl AstNode for Trait {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TRAIT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6801,42 +6282,6 @@ impl fmt::Debug for Trait {
         f.debug_struct("Trait").field("syntax", &self.syntax).finish()
     }
 }
-impl AstNode for TraitAlias {
-    #[inline]
-    fn kind() -> SyntaxKind
-    where
-        Self: Sized,
-    {
-        TRAIT_ALIAS
-    }
-    #[inline]
-    fn can_cast(kind: SyntaxKind) -> bool { kind == TRAIT_ALIAS }
-    #[inline]
-    fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
-    }
-    #[inline]
-    fn syntax(&self) -> &SyntaxNode { &self.syntax }
-}
-impl hash::Hash for TraitAlias {
-    fn hash(&self, state: &mut H) { self.syntax.hash(state); }
-}
-impl Eq for TraitAlias {}
-impl PartialEq for TraitAlias {
-    fn eq(&self, other: &Self) -> bool { self.syntax == other.syntax }
-}
-impl Clone for TraitAlias {
-    fn clone(&self) -> Self { Self { syntax: self.syntax.clone() } }
-}
-impl fmt::Debug for TraitAlias {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("TraitAlias").field("syntax", &self.syntax).finish()
-    }
-}
 impl AstNode for TryExpr {
     #[inline]
     fn kind() -> SyntaxKind
@@ -6849,11 +6294,7 @@ impl AstNode for TryExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TRY_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6885,11 +6326,7 @@ impl AstNode for TupleExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6921,11 +6358,7 @@ impl AstNode for TupleField {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_FIELD }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6957,11 +6390,7 @@ impl AstNode for TupleFieldList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_FIELD_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -6993,11 +6422,7 @@ impl AstNode for TuplePat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7029,11 +6454,7 @@ impl AstNode for TupleStructPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_STRUCT_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7065,11 +6486,7 @@ impl AstNode for TupleType {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TUPLE_TYPE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7101,11 +6518,7 @@ impl AstNode for TypeAlias {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_ALIAS }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7137,11 +6550,7 @@ impl AstNode for TypeAnchor {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_ANCHOR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7173,11 +6582,7 @@ impl AstNode for TypeArg {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_ARG }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7209,11 +6614,7 @@ impl AstNode for TypeBound {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_BOUND }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7245,11 +6646,7 @@ impl AstNode for TypeBoundList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_BOUND_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7281,11 +6678,7 @@ impl AstNode for TypeParam {
     fn can_cast(kind: SyntaxKind) -> bool { kind == TYPE_PARAM }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7317,11 +6710,7 @@ impl AstNode for UnderscoreExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == UNDERSCORE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7353,11 +6742,7 @@ impl AstNode for Union {
     fn can_cast(kind: SyntaxKind) -> bool { kind == UNION }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7389,11 +6774,7 @@ impl AstNode for Use {
     fn can_cast(kind: SyntaxKind) -> bool { kind == USE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7425,11 +6806,7 @@ impl AstNode for UseBoundGenericArgs {
     fn can_cast(kind: SyntaxKind) -> bool { kind == USE_BOUND_GENERIC_ARGS }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7461,11 +6838,7 @@ impl AstNode for UseTree {
     fn can_cast(kind: SyntaxKind) -> bool { kind == USE_TREE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7497,11 +6870,7 @@ impl AstNode for UseTreeList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == USE_TREE_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7533,11 +6902,7 @@ impl AstNode for Variant {
     fn can_cast(kind: SyntaxKind) -> bool { kind == VARIANT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7569,11 +6934,7 @@ impl AstNode for VariantList {
     fn can_cast(kind: SyntaxKind) -> bool { kind == VARIANT_LIST }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7605,11 +6966,7 @@ impl AstNode for Visibility {
     fn can_cast(kind: SyntaxKind) -> bool { kind == VISIBILITY }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7641,11 +6998,7 @@ impl AstNode for WhereClause {
     fn can_cast(kind: SyntaxKind) -> bool { kind == WHERE_CLAUSE }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7677,11 +7030,7 @@ impl AstNode for WherePred {
     fn can_cast(kind: SyntaxKind) -> bool { kind == WHERE_PRED }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7713,11 +7062,7 @@ impl AstNode for WhileExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == WHILE_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7749,11 +7094,7 @@ impl AstNode for WildcardPat {
     fn can_cast(kind: SyntaxKind) -> bool { kind == WILDCARD_PAT }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7785,11 +7126,7 @@ impl AstNode for YeetExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == YEET_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -7821,11 +7158,7 @@ impl AstNode for YieldExpr {
     fn can_cast(kind: SyntaxKind) -> bool { kind == YIELD_EXPR }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     #[inline]
     fn syntax(&self) -> &SyntaxNode { &self.syntax }
@@ -8471,10 +7804,6 @@ impl From for Item {
     #[inline]
     fn from(node: Trait) -> Item { Item::Trait(node) }
 }
-impl From for Item {
-    #[inline]
-    fn from(node: TraitAlias) -> Item { Item::TraitAlias(node) }
-}
 impl From for Item {
     #[inline]
     fn from(node: TypeAlias) -> Item { Item::TypeAlias(node) }
@@ -8506,7 +7835,6 @@ impl AstNode for Item {
                 | STATIC
                 | STRUCT
                 | TRAIT
-                | TRAIT_ALIAS
                 | TYPE_ALIAS
                 | UNION
                 | USE
@@ -8529,7 +7857,6 @@ impl AstNode for Item {
             STATIC => Item::Static(Static { syntax }),
             STRUCT => Item::Struct(Struct { syntax }),
             TRAIT => Item::Trait(Trait { syntax }),
-            TRAIT_ALIAS => Item::TraitAlias(TraitAlias { syntax }),
             TYPE_ALIAS => Item::TypeAlias(TypeAlias { syntax }),
             UNION => Item::Union(Union { syntax }),
             USE => Item::Use(Use { syntax }),
@@ -8554,7 +7881,6 @@ impl AstNode for Item {
             Item::Static(it) => &it.syntax,
             Item::Struct(it) => &it.syntax,
             Item::Trait(it) => &it.syntax,
-            Item::TraitAlias(it) => &it.syntax,
             Item::TypeAlias(it) => &it.syntax,
             Item::Union(it) => &it.syntax,
             Item::Use(it) => &it.syntax,
@@ -8984,7 +8310,6 @@ impl AstNode for AnyHasAttrs {
                 | STMT_LIST
                 | STRUCT
                 | TRAIT
-                | TRAIT_ALIAS
                 | TRY_EXPR
                 | TUPLE_EXPR
                 | TUPLE_FIELD
@@ -9257,10 +8582,6 @@ impl From for AnyHasAttrs {
     #[inline]
     fn from(node: Trait) -> AnyHasAttrs { AnyHasAttrs { syntax: node.syntax } }
 }
-impl From for AnyHasAttrs {
-    #[inline]
-    fn from(node: TraitAlias) -> AnyHasAttrs { AnyHasAttrs { syntax: node.syntax } }
-}
 impl From for AnyHasAttrs {
     #[inline]
     fn from(node: TryExpr) -> AnyHasAttrs { AnyHasAttrs { syntax: node.syntax } }
@@ -9330,7 +8651,6 @@ impl AstNode for AnyHasDocComments {
                 | STATIC
                 | STRUCT
                 | TRAIT
-                | TRAIT_ALIAS
                 | TUPLE_FIELD
                 | TYPE_ALIAS
                 | UNION
@@ -9420,10 +8740,6 @@ impl From for AnyHasDocComments {
     #[inline]
     fn from(node: Trait) -> AnyHasDocComments { AnyHasDocComments { syntax: node.syntax } }
 }
-impl From for AnyHasDocComments {
-    #[inline]
-    fn from(node: TraitAlias) -> AnyHasDocComments { AnyHasDocComments { syntax: node.syntax } }
-}
 impl From for AnyHasDocComments {
     #[inline]
     fn from(node: TupleField) -> AnyHasDocComments { AnyHasDocComments { syntax: node.syntax } }
@@ -9488,7 +8804,7 @@ impl ast::HasGenericParams for AnyHasGenericParams {}
 impl AstNode for AnyHasGenericParams {
     #[inline]
     fn can_cast(kind: SyntaxKind) -> bool {
-        matches!(kind, CONST | ENUM | FN | IMPL | STRUCT | TRAIT | TRAIT_ALIAS | TYPE_ALIAS | UNION)
+        matches!(kind, CONST | ENUM | FN | IMPL | STRUCT | TRAIT | TYPE_ALIAS | UNION)
     }
     #[inline]
     fn cast(syntax: SyntaxNode) -> Option {
@@ -9536,10 +8852,6 @@ impl From for AnyHasGenericParams {
     #[inline]
     fn from(node: Trait) -> AnyHasGenericParams { AnyHasGenericParams { syntax: node.syntax } }
 }
-impl From for AnyHasGenericParams {
-    #[inline]
-    fn from(node: TraitAlias) -> AnyHasGenericParams { AnyHasGenericParams { syntax: node.syntax } }
-}
 impl From for AnyHasGenericParams {
     #[inline]
     fn from(node: TypeAlias) -> AnyHasGenericParams { AnyHasGenericParams { syntax: node.syntax } }
@@ -9646,7 +8958,6 @@ impl AstNode for AnyHasName {
                 | STATIC
                 | STRUCT
                 | TRAIT
-                | TRAIT_ALIAS
                 | TYPE_ALIAS
                 | TYPE_PARAM
                 | UNION
@@ -9739,10 +9050,6 @@ impl From for AnyHasName {
     #[inline]
     fn from(node: Trait) -> AnyHasName { AnyHasName { syntax: node.syntax } }
 }
-impl From for AnyHasName {
-    #[inline]
-    fn from(node: TraitAlias) -> AnyHasName { AnyHasName { syntax: node.syntax } }
-}
 impl From for AnyHasName {
     #[inline]
     fn from(node: TypeAlias) -> AnyHasName { AnyHasName { syntax: node.syntax } }
@@ -9832,7 +9139,6 @@ impl AstNode for AnyHasVisibility {
                 | STATIC
                 | STRUCT
                 | TRAIT
-                | TRAIT_ALIAS
                 | TUPLE_FIELD
                 | TYPE_ALIAS
                 | UNION
@@ -9910,10 +9216,6 @@ impl From for AnyHasVisibility {
     #[inline]
     fn from(node: Trait) -> AnyHasVisibility { AnyHasVisibility { syntax: node.syntax } }
 }
-impl From for AnyHasVisibility {
-    #[inline]
-    fn from(node: TraitAlias) -> AnyHasVisibility { AnyHasVisibility { syntax: node.syntax } }
-}
 impl From for AnyHasVisibility {
     #[inline]
     fn from(node: TupleField) -> AnyHasVisibility { AnyHasVisibility { syntax: node.syntax } }
@@ -10639,11 +9941,6 @@ impl std::fmt::Display for Trait {
         std::fmt::Display::fmt(self.syntax(), f)
     }
 }
-impl std::fmt::Display for TraitAlias {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(self.syntax(), f)
-    }
-}
 impl std::fmt::Display for TryExpr {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         std::fmt::Display::fmt(self.syntax(), f)
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/tokens.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/tokens.rs
index b2f56c0b1dbf..3dca0db82630 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/generated/tokens.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/generated/tokens.rs
@@ -1,9 +1,9 @@
 //! Generated by `cargo xtask codegen grammar`, do not edit by hand.
 
 use crate::{
-    ast::AstToken,
     SyntaxKind::{self, *},
     SyntaxToken,
+    ast::AstToken,
 };
 use std::{fmt, hash};
 pub struct Byte {
@@ -17,11 +17,7 @@ impl std::fmt::Display for Byte {
 impl AstToken for Byte {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -51,11 +47,7 @@ impl std::fmt::Display for ByteString {
 impl AstToken for ByteString {
     fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE_STRING }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -85,11 +77,7 @@ impl std::fmt::Display for CString {
 impl AstToken for CString {
     fn can_cast(kind: SyntaxKind) -> bool { kind == C_STRING }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -119,11 +107,7 @@ impl std::fmt::Display for Char {
 impl AstToken for Char {
     fn can_cast(kind: SyntaxKind) -> bool { kind == CHAR }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -153,11 +137,7 @@ impl std::fmt::Display for Comment {
 impl AstToken for Comment {
     fn can_cast(kind: SyntaxKind) -> bool { kind == COMMENT }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -187,11 +167,7 @@ impl std::fmt::Display for FloatNumber {
 impl AstToken for FloatNumber {
     fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -221,11 +197,7 @@ impl std::fmt::Display for Ident {
 impl AstToken for Ident {
     fn can_cast(kind: SyntaxKind) -> bool { kind == IDENT }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -255,11 +227,7 @@ impl std::fmt::Display for IntNumber {
 impl AstToken for IntNumber {
     fn can_cast(kind: SyntaxKind) -> bool { kind == INT_NUMBER }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -289,11 +257,7 @@ impl std::fmt::Display for String {
 impl AstToken for String {
     fn can_cast(kind: SyntaxKind) -> bool { kind == STRING }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
@@ -323,11 +287,7 @@ impl std::fmt::Display for Whitespace {
 impl AstToken for Whitespace {
     fn can_cast(kind: SyntaxKind) -> bool { kind == WHITESPACE }
     fn cast(syntax: SyntaxToken) -> Option {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
+        if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/make.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/make.rs
index 2a7b51c3c248..051c5835571b 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/make.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/make.rs
@@ -190,6 +190,7 @@ fn ty_from_text(text: &str) -> ast::Type {
 }
 
 pub fn ty_alias(
+    attrs: impl IntoIterator,
     ident: &str,
     generic_param_list: Option,
     type_param_bounds: Option,
@@ -200,6 +201,7 @@ pub fn ty_alias(
     let assignment_where = assignment_where.flatten();
     quote! {
         TypeAlias {
+            #(#attrs "\n")*
             [type] " "
                 Name { [IDENT ident] }
                 #generic_param_list
@@ -229,9 +231,24 @@ pub fn ty_fn_ptr>(
     }
 }
 
-pub fn assoc_item_list(
-    body: Option>>,
-) -> ast::AssocItemList {
+pub fn item_list(body: Option>) -> ast::ItemList {
+    let is_break_braces = body.is_some();
+    let body_newline = if is_break_braces { "\n" } else { "" };
+    let body_indent = if is_break_braces { "    " } else { "" };
+
+    let body = match body {
+        Some(bd) => bd.iter().map(|elem| elem.to_string()).join("\n\n    "),
+        None => String::new(),
+    };
+    ast_from_text(&format!("mod C {{{body_newline}{body_indent}{body}{body_newline}}}"))
+}
+
+pub fn mod_(name: ast::Name, body: Option) -> ast::Module {
+    let body = body.map_or(";".to_owned(), |body| format!(" {body}"));
+    ast_from_text(&format!("mod {name}{body}"))
+}
+
+pub fn assoc_item_list(body: Option>) -> ast::AssocItemList {
     let is_break_braces = body.is_some();
     let body_newline = if is_break_braces { "\n".to_owned() } else { String::new() };
     let body_indent = if is_break_braces { "    ".to_owned() } else { String::new() };
@@ -279,12 +296,16 @@ fn merge_where_clause(
 }
 
 pub fn impl_(
+    attrs: impl IntoIterator,
     generic_params: Option,
     generic_args: Option,
     path_type: ast::Type,
     where_clause: Option,
     body: Option,
 ) -> ast::Impl {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
+
     let gen_args = generic_args.map_or_else(String::new, |it| it.to_string());
 
     let gen_params = generic_params.map_or_else(String::new, |it| it.to_string());
@@ -297,10 +318,11 @@ pub fn impl_(
     };
 
     let body = body.map_or_else(|| format!("{{{body_newline}}}"), |it| it.to_string());
-    ast_from_text(&format!("impl{gen_params} {path_type}{gen_args}{where_clause}{body}"))
+    ast_from_text(&format!("{attrs}impl{gen_params} {path_type}{gen_args}{where_clause}{body}"))
 }
 
 pub fn impl_trait(
+    attrs: impl IntoIterator,
     is_unsafe: bool,
     trait_gen_params: Option,
     trait_gen_args: Option,
@@ -313,6 +335,8 @@ pub fn impl_trait(
     ty_where_clause: Option,
     body: Option,
 ) -> ast::Impl {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
     let is_unsafe = if is_unsafe { "unsafe " } else { "" };
 
     let trait_gen_args = trait_gen_args.map(|args| args.to_string()).unwrap_or_default();
@@ -336,7 +360,7 @@ pub fn impl_trait(
     let body = body.map_or_else(|| format!("{{{body_newline}}}"), |it| it.to_string());
 
     ast_from_text(&format!(
-        "{is_unsafe}impl{gen_params} {is_negative}{path_type}{trait_gen_args} for {ty}{type_gen_args}{where_clause}{body}"
+        "{attrs}{is_unsafe}impl{gen_params} {is_negative}{path_type}{trait_gen_args} for {ty}{type_gen_args}{where_clause}{body}"
     ))
 }
 
@@ -454,12 +478,18 @@ pub fn use_tree_list(use_trees: impl IntoIterator) -> ast::
     ast_from_text(&format!("use {{{use_trees}}};"))
 }
 
-pub fn use_(visibility: Option, use_tree: ast::UseTree) -> ast::Use {
+pub fn use_(
+    attrs: impl IntoIterator,
+    visibility: Option,
+    use_tree: ast::UseTree,
+) -> ast::Use {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
     let visibility = match visibility {
         None => String::new(),
         Some(it) => format!("{it} "),
     };
-    ast_from_text(&format!("{visibility}use {use_tree};"))
+    ast_from_text(&format!("{attrs}{visibility}use {use_tree};"))
 }
 
 pub fn record_expr(path: ast::Path, fields: ast::RecordExprFieldList) -> ast::RecordExpr {
@@ -948,16 +978,19 @@ pub fn expr_stmt(expr: ast::Expr) -> ast::ExprStmt {
 }
 
 pub fn item_const(
+    attrs: impl IntoIterator,
     visibility: Option,
     name: ast::Name,
     ty: ast::Type,
     expr: ast::Expr,
 ) -> ast::Const {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
     let visibility = match visibility {
         None => String::new(),
         Some(it) => format!("{it} "),
     };
-    ast_from_text(&format!("{visibility}const {name}: {ty} = {expr};"))
+    ast_from_text(&format!("{attrs}{visibility}const {name}: {ty} = {expr};"))
 }
 
 pub fn item_static(
@@ -1164,6 +1197,7 @@ pub fn variant(
 }
 
 pub fn fn_(
+    attrs: impl IntoIterator,
     visibility: Option,
     fn_name: ast::Name,
     type_params: Option,
@@ -1176,6 +1210,8 @@ pub fn fn_(
     is_unsafe: bool,
     is_gen: bool,
 ) -> ast::Fn {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
     let type_params = match type_params {
         Some(type_params) => format!("{type_params}"),
         None => "".into(),
@@ -1199,7 +1235,7 @@ pub fn fn_(
     let gen_literal = if is_gen { "gen " } else { "" };
 
     ast_from_text(&format!(
-        "{visibility}{const_literal}{async_literal}{gen_literal}{unsafe_literal}fn {fn_name}{type_params}{params} {ret_type}{where_clause}{body}",
+        "{attrs}{visibility}{const_literal}{async_literal}{gen_literal}{unsafe_literal}fn {fn_name}{type_params}{params} {ret_type}{where_clause}{body}",
     ))
 }
 pub fn struct_(
@@ -1208,23 +1244,29 @@ pub fn struct_(
     generic_param_list: Option,
     field_list: ast::FieldList,
 ) -> ast::Struct {
-    let semicolon = if matches!(field_list, ast::FieldList::TupleFieldList(_)) { ";" } else { "" };
+    let (semicolon, ws) =
+        if matches!(field_list, ast::FieldList::TupleFieldList(_)) { (";", "") } else { ("", " ") };
     let type_params = generic_param_list.map_or_else(String::new, |it| it.to_string());
     let visibility = match visibility {
         None => String::new(),
         Some(it) => format!("{it} "),
     };
 
-    ast_from_text(&format!("{visibility}struct {strukt_name}{type_params}{field_list}{semicolon}",))
+    ast_from_text(&format!(
+        "{visibility}struct {strukt_name}{type_params}{ws}{field_list}{semicolon}"
+    ))
 }
 
 pub fn enum_(
+    attrs: impl IntoIterator,
     visibility: Option,
     enum_name: ast::Name,
     generic_param_list: Option,
     where_clause: Option,
     variant_list: ast::VariantList,
 ) -> ast::Enum {
+    let attrs =
+        attrs.into_iter().fold(String::new(), |mut acc, attr| format_to_acc!(acc, "{}\n", attr));
     let visibility = match visibility {
         None => String::new(),
         Some(it) => format!("{it} "),
@@ -1234,7 +1276,7 @@ pub fn enum_(
     let where_clause = where_clause.map(|it| format!(" {it}")).unwrap_or_default();
 
     ast_from_text(&format!(
-        "{visibility}enum {enum_name}{generic_params}{where_clause} {variant_list}"
+        "{attrs}{visibility}enum {enum_name}{generic_params}{where_clause} {variant_list}"
     ))
 }
 
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/node_ext.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/node_ext.rs
index 62a7d4df2cf6..af741d100f68 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/node_ext.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/node_ext.rs
@@ -12,8 +12,8 @@ use rowan::{GreenNodeData, GreenTokenData};
 use crate::{
     NodeOrToken, SmolStr, SyntaxElement, SyntaxToken, T, TokenText,
     ast::{
-        self, AstNode, AstToken, HasAttrs, HasGenericArgs, HasGenericParams, HasName, SyntaxNode,
-        support,
+        self, AstNode, AstToken, HasAttrs, HasGenericArgs, HasGenericParams, HasName,
+        HasTypeBounds, SyntaxNode, support,
     },
     ted,
 };
@@ -880,51 +880,6 @@ impl AstNode for TypeOrConstParam {
 
 impl HasAttrs for TypeOrConstParam {}
 
-#[derive(Debug, Clone)]
-pub enum TraitOrAlias {
-    Trait(ast::Trait),
-    TraitAlias(ast::TraitAlias),
-}
-
-impl TraitOrAlias {
-    pub fn name(&self) -> Option {
-        match self {
-            TraitOrAlias::Trait(x) => x.name(),
-            TraitOrAlias::TraitAlias(x) => x.name(),
-        }
-    }
-}
-
-impl AstNode for TraitOrAlias {
-    fn can_cast(kind: SyntaxKind) -> bool
-    where
-        Self: Sized,
-    {
-        matches!(kind, SyntaxKind::TRAIT | SyntaxKind::TRAIT_ALIAS)
-    }
-
-    fn cast(syntax: SyntaxNode) -> Option
-    where
-        Self: Sized,
-    {
-        let res = match syntax.kind() {
-            SyntaxKind::TRAIT => TraitOrAlias::Trait(ast::Trait { syntax }),
-            SyntaxKind::TRAIT_ALIAS => TraitOrAlias::TraitAlias(ast::TraitAlias { syntax }),
-            _ => return None,
-        };
-        Some(res)
-    }
-
-    fn syntax(&self) -> &SyntaxNode {
-        match self {
-            TraitOrAlias::Trait(it) => it.syntax(),
-            TraitOrAlias::TraitAlias(it) => it.syntax(),
-        }
-    }
-}
-
-impl HasAttrs for TraitOrAlias {}
-
 pub enum VisibilityKind {
     In(ast::Path),
     PubCrate,
@@ -957,11 +912,10 @@ impl ast::Visibility {
 
 impl ast::LifetimeParam {
     pub fn lifetime_bounds(&self) -> impl Iterator {
-        self.syntax()
-            .children_with_tokens()
-            .filter_map(|it| it.into_token())
-            .skip_while(|x| x.kind() != T![:])
-            .filter(|it| it.kind() == T![lifetime_ident])
+        self.type_bound_list()
+            .into_iter()
+            .flat_map(|it| it.bounds())
+            .filter_map(|it| it.lifetime()?.lifetime_ident_token())
     }
 }
 
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
index 00750bff0ba2..1364adb187fc 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/prec.rs
@@ -276,19 +276,19 @@ impl Expr {
         }
 
         // Not every expression can be followed by `else` in the `let-else`
-        if let Some(ast::Stmt::LetStmt(e)) = stmt {
-            if e.let_else().is_some() {
-                match self {
-                    BinExpr(e)
-                        if e.op_kind()
-                            .map(|op| matches!(op, BinaryOp::LogicOp(_)))
-                            .unwrap_or(false) =>
-                    {
-                        return true;
-                    }
-                    _ if self.clone().trailing_brace().is_some() => return true,
-                    _ => {}
+        if let Some(ast::Stmt::LetStmt(e)) = stmt
+            && e.let_else().is_some()
+        {
+            match self {
+                BinExpr(e)
+                    if e.op_kind()
+                        .map(|op| matches!(op, BinaryOp::LogicOp(_)))
+                        .unwrap_or(false) =>
+                {
+                    return true;
                 }
+                _ if self.clone().trailing_brace().is_some() => return true,
+                _ => {}
             }
         }
 
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
index 1ba610731512..8bf27f967482 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/syntax_factory/constructors.rs
@@ -2,8 +2,8 @@
 use crate::{
     AstNode, NodeOrToken, SyntaxKind, SyntaxNode, SyntaxToken,
     ast::{
-        self, HasArgList, HasGenericArgs, HasGenericParams, HasLoopBody, HasName, HasTypeBounds,
-        HasVisibility, RangeItem, make,
+        self, HasArgList, HasAttrs, HasGenericArgs, HasGenericParams, HasLoopBody, HasName,
+        HasTypeBounds, HasVisibility, RangeItem, make,
     },
     syntax_editor::SyntaxMappingBuilder,
 };
@@ -107,8 +107,13 @@ impl SyntaxFactory {
         ast
     }
 
-    pub fn use_(&self, visibility: Option, use_tree: ast::UseTree) -> ast::Use {
-        make::use_(visibility, use_tree).clone_for_update()
+    pub fn use_(
+        &self,
+        attrs: impl IntoIterator,
+        visibility: Option,
+        use_tree: ast::UseTree,
+    ) -> ast::Use {
+        make::use_(attrs, visibility, use_tree).clone_for_update()
     }
 
     pub fn use_tree(
@@ -840,16 +845,20 @@ impl SyntaxFactory {
 
     pub fn item_const(
         &self,
+        attrs: impl IntoIterator,
         visibility: Option,
         name: ast::Name,
         ty: ast::Type,
         expr: ast::Expr,
     ) -> ast::Const {
-        let ast = make::item_const(visibility.clone(), name.clone(), ty.clone(), expr.clone())
-            .clone_for_update();
+        let (attrs, attrs_input) = iterator_input(attrs);
+        let ast =
+            make::item_const(attrs, visibility.clone(), name.clone(), ty.clone(), expr.clone())
+                .clone_for_update();
 
         if let Some(mut mapping) = self.mappings() {
             let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone());
+            builder.map_children(attrs_input, ast.attrs().map(|attr| attr.syntax().clone()));
             if let Some(visibility) = visibility {
                 builder.map_node(
                     visibility.syntax().clone(),
@@ -939,6 +948,24 @@ impl SyntaxFactory {
         ast
     }
 
+    pub fn record_expr(
+        &self,
+        path: ast::Path,
+        fields: ast::RecordExprFieldList,
+    ) -> ast::RecordExpr {
+        let ast = make::record_expr(path.clone(), fields.clone()).clone_for_update();
+        if let Some(mut mapping) = self.mappings() {
+            let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone());
+            builder.map_node(path.syntax().clone(), ast.path().unwrap().syntax().clone());
+            builder.map_node(
+                fields.syntax().clone(),
+                ast.record_expr_field_list().unwrap().syntax().clone(),
+            );
+            builder.finish(&mut mapping);
+        }
+        ast
+    }
+
     pub fn record_expr_field(
         &self,
         name: ast::NameRef,
@@ -1049,6 +1076,7 @@ impl SyntaxFactory {
 
     pub fn item_enum(
         &self,
+        attrs: impl IntoIterator,
         visibility: Option,
         name: ast::Name,
         generic_param_list: Option,
@@ -1056,6 +1084,7 @@ impl SyntaxFactory {
         variant_list: ast::VariantList,
     ) -> ast::Enum {
         let ast = make::enum_(
+            attrs,
             visibility.clone(),
             name.clone(),
             generic_param_list.clone(),
@@ -1164,6 +1193,7 @@ impl SyntaxFactory {
 
     pub fn fn_(
         &self,
+        attrs: impl IntoIterator,
         visibility: Option,
         fn_name: ast::Name,
         type_params: Option,
@@ -1176,7 +1206,9 @@ impl SyntaxFactory {
         is_unsafe: bool,
         is_gen: bool,
     ) -> ast::Fn {
+        let (attrs, input) = iterator_input(attrs);
         let ast = make::fn_(
+            attrs,
             visibility.clone(),
             fn_name.clone(),
             type_params.clone(),
@@ -1192,6 +1224,7 @@ impl SyntaxFactory {
 
         if let Some(mut mapping) = self.mappings() {
             let mut builder = SyntaxMappingBuilder::new(ast.syntax().clone());
+            builder.map_children(input, ast.attrs().map(|attr| attr.syntax().clone()));
 
             if let Some(visibility) = visibility {
                 builder.map_node(
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ast/token_ext.rs b/src/tools/rust-analyzer/crates/syntax/src/ast/token_ext.rs
index 4afdda78a0e7..e1a9f3ac0341 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ast/token_ext.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ast/token_ext.rs
@@ -151,10 +151,10 @@ impl QuoteOffsets {
 }
 
 pub trait IsString: AstToken {
-    const RAW_PREFIX: &'static str;
-    fn unescape(s: &str, callback: impl FnMut(Range, Result));
+    fn raw_prefix(&self) -> &'static str;
+    fn unescape(&self, s: &str, callback: impl FnMut(Range, Result));
     fn is_raw(&self) -> bool {
-        self.text().starts_with(Self::RAW_PREFIX)
+        self.text().starts_with(self.raw_prefix())
     }
     fn quote_offsets(&self) -> Option {
         let text = self.text();
@@ -187,7 +187,7 @@ pub trait IsString: AstToken {
         let text = &self.text()[text_range_no_quotes - start];
         let offset = text_range_no_quotes.start() - start;
 
-        Self::unescape(text, &mut |range: Range, unescaped_char| {
+        self.unescape(text, &mut |range: Range, unescaped_char| {
             if let Some((s, e)) = range.start.try_into().ok().zip(range.end.try_into().ok()) {
                 cb(TextRange::new(s, e) + offset, unescaped_char);
             }
@@ -201,11 +201,17 @@ pub trait IsString: AstToken {
             None
         }
     }
+    fn map_offset_down(&self, offset: TextSize) -> Option {
+        let contents_range = self.text_range_between_quotes()?;
+        offset.checked_sub(contents_range.start())
+    }
 }
 
 impl IsString for ast::String {
-    const RAW_PREFIX: &'static str = "r";
-    fn unescape(s: &str, cb: impl FnMut(Range, Result)) {
+    fn raw_prefix(&self) -> &'static str {
+        "r"
+    }
+    fn unescape(&self, s: &str, cb: impl FnMut(Range, Result)) {
         unescape_str(s, cb)
     }
 }
@@ -246,8 +252,10 @@ impl ast::String {
 }
 
 impl IsString for ast::ByteString {
-    const RAW_PREFIX: &'static str = "br";
-    fn unescape(s: &str, mut callback: impl FnMut(Range, Result)) {
+    fn raw_prefix(&self) -> &'static str {
+        "br"
+    }
+    fn unescape(&self, s: &str, mut callback: impl FnMut(Range, Result)) {
         unescape_byte_str(s, |range, res| callback(range, res.map(char::from)))
     }
 }
@@ -288,10 +296,12 @@ impl ast::ByteString {
 }
 
 impl IsString for ast::CString {
-    const RAW_PREFIX: &'static str = "cr";
+    fn raw_prefix(&self) -> &'static str {
+        "cr"
+    }
     // NOTE: This method should only be used for highlighting ranges. The unescaped
     // char/byte is not used. For simplicity, we return an arbitrary placeholder char.
-    fn unescape(s: &str, mut callback: impl FnMut(Range, Result)) {
+    fn unescape(&self, s: &str, mut callback: impl FnMut(Range, Result)) {
         unescape_c_str(s, |range, _res| callback(range, Ok('_')))
     }
 }
@@ -465,6 +475,74 @@ impl ast::Byte {
     }
 }
 
+pub enum AnyString {
+    ByteString(ast::ByteString),
+    CString(ast::CString),
+    String(ast::String),
+}
+
+impl AnyString {
+    pub fn value(&self) -> Result, EscapeError> {
+        fn from_utf8(s: Cow<'_, [u8]>) -> Result, EscapeError> {
+            match s {
+                Cow::Borrowed(s) => str::from_utf8(s)
+                    .map_err(|_| EscapeError::NonAsciiCharInByte)
+                    .map(Cow::Borrowed),
+                Cow::Owned(s) => String::from_utf8(s)
+                    .map_err(|_| EscapeError::NonAsciiCharInByte)
+                    .map(Cow::Owned),
+            }
+        }
+
+        match self {
+            AnyString::String(s) => s.value(),
+            AnyString::ByteString(s) => s.value().and_then(from_utf8),
+            AnyString::CString(s) => s.value().and_then(from_utf8),
+        }
+    }
+}
+
+impl ast::AstToken for AnyString {
+    fn can_cast(kind: crate::SyntaxKind) -> bool {
+        ast::String::can_cast(kind)
+            || ast::ByteString::can_cast(kind)
+            || ast::CString::can_cast(kind)
+    }
+
+    fn cast(syntax: crate::SyntaxToken) -> Option {
+        ast::String::cast(syntax.clone())
+            .map(Self::String)
+            .or_else(|| ast::ByteString::cast(syntax.clone()).map(Self::ByteString))
+            .or_else(|| ast::CString::cast(syntax).map(Self::CString))
+    }
+
+    fn syntax(&self) -> &crate::SyntaxToken {
+        match self {
+            Self::ByteString(it) => it.syntax(),
+            Self::CString(it) => it.syntax(),
+            Self::String(it) => it.syntax(),
+        }
+    }
+}
+
+impl IsString for AnyString {
+    fn raw_prefix(&self) -> &'static str {
+        match self {
+            AnyString::ByteString(s) => s.raw_prefix(),
+            AnyString::CString(s) => s.raw_prefix(),
+            AnyString::String(s) => s.raw_prefix(),
+        }
+    }
+
+    fn unescape(&self, s: &str, callback: impl FnMut(Range, Result)) {
+        match self {
+            AnyString::ByteString(it) => it.unescape(s, callback),
+            AnyString::CString(it) => it.unescape(s, callback),
+            AnyString::String(it) => it.unescape(s, callback),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use rustc_apfloat::ieee::Quad as f128;
diff --git a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor.rs b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor.rs
index 5107754b1825..0b358878fcf2 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor.rs
@@ -83,6 +83,16 @@ impl SyntaxEditor {
         self.changes.push(Change::Replace(element.syntax_element(), None));
     }
 
+    pub fn delete_all(&mut self, range: RangeInclusive) {
+        if range.start() == range.end() {
+            self.delete(range.start());
+            return;
+        }
+
+        debug_assert!(is_ancestor_or_self_of_element(range.start(), &self.root));
+        self.changes.push(Change::ReplaceAll(range, Vec::new()))
+    }
+
     pub fn replace(&mut self, old: impl Element, new: impl Element) {
         let old = old.syntax_element();
         debug_assert!(is_ancestor_or_self_of_element(&old, &self.root));
@@ -608,6 +618,7 @@ mod tests {
     #[test]
     fn test_replace_token_in_parent() {
         let parent_fn = make::fn_(
+            None,
             None,
             make::name("it"),
             None,
@@ -626,10 +637,10 @@ mod tests {
         if let Some(ret_ty) = parent_fn.ret_type() {
             editor.delete(ret_ty.syntax().clone());
 
-            if let Some(SyntaxElement::Token(token)) = ret_ty.syntax().next_sibling_or_token() {
-                if token.kind().is_trivia() {
-                    editor.delete(token);
-                }
+            if let Some(SyntaxElement::Token(token)) = ret_ty.syntax().next_sibling_or_token()
+                && token.kind().is_trivia()
+            {
+                editor.delete(token);
             }
         }
 
diff --git a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/edits.rs b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/edits.rs
index 840e76979792..9090f7c9eb14 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/edits.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/syntax_editor/edits.rs
@@ -153,6 +153,23 @@ impl ast::VariantList {
     }
 }
 
+impl ast::Fn {
+    pub fn replace_or_insert_body(&self, editor: &mut SyntaxEditor, body: ast::BlockExpr) {
+        if let Some(old_body) = self.body() {
+            editor.replace(old_body.syntax(), body.syntax());
+        } else {
+            let single_space = make::tokens::single_space();
+            let elements = vec![single_space.into(), body.syntax().clone().into()];
+
+            if let Some(semicolon) = self.semicolon_token() {
+                editor.replace_with_many(semicolon, elements);
+            } else {
+                editor.insert_all(Position::last_child_of(self.syntax()), elements);
+            }
+        }
+    }
+}
+
 fn normalize_ws_between_braces(editor: &mut SyntaxEditor, node: &SyntaxNode) -> Option<()> {
     let make = SyntaxFactory::without_mappings();
     let l = node
@@ -184,6 +201,15 @@ pub trait Removable: AstNode {
     fn remove(&self, editor: &mut SyntaxEditor);
 }
 
+impl Removable for ast::TypeBoundList {
+    fn remove(&self, editor: &mut SyntaxEditor) {
+        match self.syntax().siblings_with_tokens(Direction::Prev).find(|it| it.kind() == T![:]) {
+            Some(colon) => editor.delete_all(colon..=self.syntax().clone().into()),
+            None => editor.delete(self.syntax()),
+        }
+    }
+}
+
 impl Removable for ast::Use {
     fn remove(&self, editor: &mut SyntaxEditor) {
         let make = SyntaxFactory::without_mappings();
diff --git a/src/tools/rust-analyzer/crates/syntax/src/syntax_error.rs b/src/tools/rust-analyzer/crates/syntax/src/syntax_error.rs
index dc6130bd6415..1c902893abc6 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/syntax_error.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/syntax_error.rs
@@ -42,3 +42,5 @@ impl fmt::Display for SyntaxError {
         self.0.fmt(f)
     }
 }
+
+impl std::error::Error for SyntaxError {}
diff --git a/src/tools/rust-analyzer/crates/syntax/src/ted.rs b/src/tools/rust-analyzer/crates/syntax/src/ted.rs
index 6fcbdd006c24..5c286479c4e3 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/ted.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/ted.rs
@@ -90,15 +90,15 @@ pub fn insert_raw(position: Position, elem: impl Element) {
     insert_all_raw(position, vec![elem.syntax_element()]);
 }
 pub fn insert_all(position: Position, mut elements: Vec) {
-    if let Some(first) = elements.first() {
-        if let Some(ws) = ws_before(&position, first) {
-            elements.insert(0, ws.into());
-        }
+    if let Some(first) = elements.first()
+        && let Some(ws) = ws_before(&position, first)
+    {
+        elements.insert(0, ws.into());
     }
-    if let Some(last) = elements.last() {
-        if let Some(ws) = ws_after(&position, last) {
-            elements.push(ws.into());
-        }
+    if let Some(last) = elements.last()
+        && let Some(ws) = ws_after(&position, last)
+    {
+        elements.push(ws.into());
     }
     insert_all_raw(position, elements);
 }
@@ -165,20 +165,22 @@ fn ws_before(position: &Position, new: &SyntaxElement) -> Option {
         PositionRepr::After(it) => it,
     };
 
-    if prev.kind() == T!['{'] && new.kind() == SyntaxKind::USE {
-        if let Some(item_list) = prev.parent().and_then(ast::ItemList::cast) {
-            let mut indent = IndentLevel::from_element(&item_list.syntax().clone().into());
-            indent.0 += 1;
-            return Some(make::tokens::whitespace(&format!("\n{indent}")));
-        }
+    if prev.kind() == T!['{']
+        && new.kind() == SyntaxKind::USE
+        && let Some(item_list) = prev.parent().and_then(ast::ItemList::cast)
+    {
+        let mut indent = IndentLevel::from_element(&item_list.syntax().clone().into());
+        indent.0 += 1;
+        return Some(make::tokens::whitespace(&format!("\n{indent}")));
     }
 
-    if prev.kind() == T!['{'] && ast::Stmt::can_cast(new.kind()) {
-        if let Some(stmt_list) = prev.parent().and_then(ast::StmtList::cast) {
-            let mut indent = IndentLevel::from_element(&stmt_list.syntax().clone().into());
-            indent.0 += 1;
-            return Some(make::tokens::whitespace(&format!("\n{indent}")));
-        }
+    if prev.kind() == T!['{']
+        && ast::Stmt::can_cast(new.kind())
+        && let Some(stmt_list) = prev.parent().and_then(ast::StmtList::cast)
+    {
+        let mut indent = IndentLevel::from_element(&stmt_list.syntax().clone().into());
+        indent.0 += 1;
+        return Some(make::tokens::whitespace(&format!("\n{indent}")));
     }
 
     ws_between(prev, new)
diff --git a/src/tools/rust-analyzer/crates/syntax/src/validation.rs b/src/tools/rust-analyzer/crates/syntax/src/validation.rs
index 4180f9cd1855..485140be8f69 100644
--- a/src/tools/rust-analyzer/crates/syntax/src/validation.rs
+++ b/src/tools/rust-analyzer/crates/syntax/src/validation.rs
@@ -142,50 +142,50 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) {
 
     match literal.kind() {
         ast::LiteralKind::String(s) => {
-            if !s.is_raw() {
-                if let Some(without_quotes) = unquote(text, 1, '"') {
-                    unescape_str(without_quotes, |range, char| {
-                        if let Err(err) = char {
-                            push_err(1, range.start, err);
-                        }
-                    });
-                }
+            if !s.is_raw()
+                && let Some(without_quotes) = unquote(text, 1, '"')
+            {
+                unescape_str(without_quotes, |range, char| {
+                    if let Err(err) = char {
+                        push_err(1, range.start, err);
+                    }
+                });
             }
         }
         ast::LiteralKind::ByteString(s) => {
-            if !s.is_raw() {
-                if let Some(without_quotes) = unquote(text, 2, '"') {
-                    unescape_byte_str(without_quotes, |range, char| {
-                        if let Err(err) = char {
-                            push_err(1, range.start, err);
-                        }
-                    });
-                }
+            if !s.is_raw()
+                && let Some(without_quotes) = unquote(text, 2, '"')
+            {
+                unescape_byte_str(without_quotes, |range, char| {
+                    if let Err(err) = char {
+                        push_err(1, range.start, err);
+                    }
+                });
             }
         }
         ast::LiteralKind::CString(s) => {
-            if !s.is_raw() {
-                if let Some(without_quotes) = unquote(text, 2, '"') {
-                    unescape_c_str(without_quotes, |range, char| {
-                        if let Err(err) = char {
-                            push_err(1, range.start, err);
-                        }
-                    });
-                }
+            if !s.is_raw()
+                && let Some(without_quotes) = unquote(text, 2, '"')
+            {
+                unescape_c_str(without_quotes, |range, char| {
+                    if let Err(err) = char {
+                        push_err(1, range.start, err);
+                    }
+                });
             }
         }
         ast::LiteralKind::Char(_) => {
-            if let Some(without_quotes) = unquote(text, 1, '\'') {
-                if let Err(err) = unescape_char(without_quotes) {
-                    push_err(1, 0, err);
-                }
+            if let Some(without_quotes) = unquote(text, 1, '\'')
+                && let Err(err) = unescape_char(without_quotes)
+            {
+                push_err(1, 0, err);
             }
         }
         ast::LiteralKind::Byte(_) => {
-            if let Some(without_quotes) = unquote(text, 2, '\'') {
-                if let Err(err) = unescape_byte(without_quotes) {
-                    push_err(2, 0, err);
-                }
+            if let Some(without_quotes) = unquote(text, 2, '\'')
+                && let Err(err) = unescape_byte(without_quotes)
+            {
+                push_err(2, 0, err);
             }
         }
         ast::LiteralKind::IntNumber(_)
@@ -224,14 +224,14 @@ pub(crate) fn validate_block_structure(root: &SyntaxNode) {
 }
 
 fn validate_numeric_name(name_ref: Option, errors: &mut Vec) {
-    if let Some(int_token) = int_token(name_ref) {
-        if int_token.text().chars().any(|c| !c.is_ascii_digit()) {
-            errors.push(SyntaxError::new(
-                "Tuple (struct) field access is only allowed through \
+    if let Some(int_token) = int_token(name_ref)
+        && int_token.text().chars().any(|c| !c.is_ascii_digit())
+    {
+        errors.push(SyntaxError::new(
+            "Tuple (struct) field access is only allowed through \
                 decimal integers with no underscores or suffix",
-                int_token.text_range(),
-            ));
-        }
+            int_token.text_range(),
+        ));
     }
 
     fn int_token(name_ref: Option) -> Option {
@@ -285,13 +285,13 @@ fn validate_path_keywords(segment: ast::PathSegment, errors: &mut Vec Option {
diff --git a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs
index 8937e53175ab..a718b96a8252 100644
--- a/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/test-fixture/src/lib.rs
@@ -1,10 +1,11 @@
 //! A set of high-level utility fixture methods to use in tests.
 use std::{any::TypeId, mem, str::FromStr, sync};
 
+use base_db::target::TargetData;
 use base_db::{
     Crate, CrateDisplayName, CrateGraphBuilder, CrateName, CrateOrigin, CrateWorkspaceData,
-    DependencyBuilder, Env, FileChange, FileSet, LangCrateOrigin, SourceDatabase, SourceRoot,
-    Version, VfsPath, salsa,
+    DependencyBuilder, Env, FileChange, FileSet, FxIndexMap, LangCrateOrigin, SourceDatabase,
+    SourceRoot, Version, VfsPath, salsa,
 };
 use cfg::CfgOptions;
 use hir_expand::{
@@ -20,11 +21,10 @@ use hir_expand::{
 };
 use intern::{Symbol, sym};
 use paths::AbsPathBuf;
-use rustc_hash::FxHashMap;
 use span::{Edition, FileId, Span};
 use stdx::itertools::Itertools;
 use test_utils::{
-    CURSOR_MARKER, ESCAPED_CURSOR_MARKER, Fixture, FixtureWithProjectMeta, RangeOrOffset,
+    CURSOR_MARKER, ESCAPED_CURSOR_MARKER, Fixture, FixtureWithProjectMeta, MiniCore, RangeOrOffset,
     extract_range_or_offset,
 };
 use triomphe::Arc;
@@ -69,7 +69,12 @@ pub trait WithFixture: Default + ExpandDatabase + SourceDatabase + 'static {
         proc_macros: Vec<(String, ProcMacro)>,
     ) -> Self {
         let mut db = Self::default();
-        let fixture = ChangeFixture::parse_with_proc_macros(&db, ra_fixture, proc_macros);
+        let fixture = ChangeFixture::parse_with_proc_macros(
+            &db,
+            ra_fixture,
+            MiniCore::RAW_SOURCE,
+            proc_macros,
+        );
         fixture.change.apply(&mut db);
         assert!(fixture.file_position.is_none());
         db
@@ -112,8 +117,10 @@ impl WithFixture for DB
 
 pub struct ChangeFixture {
     pub file_position: Option<(EditionedFileId, RangeOrOffset)>,
+    pub file_lines: Vec,
     pub files: Vec,
     pub change: ChangeWithProcMacros,
+    pub sysroot_files: Vec,
 }
 
 const SOURCE_ROOT_PREFIX: &str = "/";
@@ -123,12 +130,13 @@ impl ChangeFixture {
         db: &dyn salsa::Database,
         #[rust_analyzer::rust_fixture] ra_fixture: &str,
     ) -> ChangeFixture {
-        Self::parse_with_proc_macros(db, ra_fixture, Vec::new())
+        Self::parse_with_proc_macros(db, ra_fixture, MiniCore::RAW_SOURCE, Vec::new())
     }
 
     pub fn parse_with_proc_macros(
         db: &dyn salsa::Database,
         #[rust_analyzer::rust_fixture] ra_fixture: &str,
+        minicore_raw: &str,
         mut proc_macro_defs: Vec<(String, ProcMacro)>,
     ) -> ChangeFixture {
         let FixtureWithProjectMeta {
@@ -137,8 +145,11 @@ impl ChangeFixture {
             proc_macro_names,
             toolchain,
             target_data_layout,
+            target_arch,
         } = FixtureWithProjectMeta::parse(ra_fixture);
-        let target_data_layout = Ok(target_data_layout.into());
+        let target_data_layout = target_data_layout.into();
+        let target_arch = parse_target_arch(&target_arch);
+        let target = Ok(TargetData { arch: target_arch, data_layout: target_data_layout });
         let toolchain = Some({
             let channel = toolchain.as_deref().unwrap_or("stable");
             Version::parse(&format!("1.76.0-{channel}")).unwrap()
@@ -146,8 +157,10 @@ impl ChangeFixture {
         let mut source_change = FileChange::default();
 
         let mut files = Vec::new();
+        let mut sysroot_files = Vec::new();
+        let mut file_lines = Vec::new();
         let mut crate_graph = CrateGraphBuilder::default();
-        let mut crates = FxHashMap::default();
+        let mut crates = FxIndexMap::default();
         let mut crate_deps = Vec::new();
         let mut default_crate_root: Option = None;
         let mut default_edition = Edition::CURRENT;
@@ -164,13 +177,14 @@ impl ChangeFixture {
 
         let mut file_position = None;
 
-        let crate_ws_data =
-            Arc::new(CrateWorkspaceData { data_layout: target_data_layout, toolchain });
+        let crate_ws_data = Arc::new(CrateWorkspaceData { target, toolchain });
 
         // FIXME: This is less than ideal
         let proc_macro_cwd = Arc::new(AbsPathBuf::assert_utf8(std::env::current_dir().unwrap()));
 
         for entry in fixture {
+            file_lines.push(entry.line);
+
             let mut range_or_offset = None;
             let text = if entry.text.contains(CURSOR_MARKER) {
                 if entry.text.contains(ESCAPED_CURSOR_MARKER) {
@@ -238,7 +252,7 @@ impl ChangeFixture {
                 assert!(default_crate_root.is_none());
                 default_crate_root = Some(file_id);
                 default_edition = meta.edition;
-                default_cfg.extend(meta.cfg.into_iter());
+                default_cfg.append(meta.cfg);
                 default_env.extend_from_other(&meta.env);
             }
 
@@ -249,37 +263,7 @@ impl ChangeFixture {
             file_id = FileId::from_raw(file_id.index() + 1);
         }
 
-        if crates.is_empty() {
-            let crate_root = default_crate_root
-                .expect("missing default crate root, specify a main.rs or lib.rs");
-            crate_graph.add_crate_root(
-                crate_root,
-                default_edition,
-                Some(CrateName::new("ra_test_fixture").unwrap().into()),
-                None,
-                default_cfg.clone(),
-                Some(default_cfg),
-                default_env,
-                CrateOrigin::Local { repo: None, name: None },
-                false,
-                proc_macro_cwd.clone(),
-                crate_ws_data.clone(),
-            );
-        } else {
-            for (from, to, prelude) in crate_deps {
-                let from_id = crates[&from];
-                let to_id = crates[&to];
-                let sysroot = crate_graph[to_id].basic.origin.is_lang();
-                crate_graph
-                    .add_dep(
-                        from_id,
-                        DependencyBuilder::with_prelude(to.clone(), to_id, prelude, sysroot),
-                    )
-                    .unwrap();
-            }
-        }
-
-        if let Some(mini_core) = mini_core {
+        let mini_core = mini_core.map(|mini_core| {
             let core_file = file_id;
             file_id = FileId::from_raw(file_id.index() + 1);
 
@@ -287,9 +271,9 @@ impl ChangeFixture {
             fs.insert(core_file, VfsPath::new_virtual_path("/sysroot/core/lib.rs".to_owned()));
             roots.push(SourceRoot::new_library(fs));
 
-            source_change.change_file(core_file, Some(mini_core.source_code()));
+            sysroot_files.push(core_file);
 
-            let all_crates = crate_graph.iter().collect::>();
+            source_change.change_file(core_file, Some(mini_core.source_code(minicore_raw)));
 
             let core_crate = crate_graph.add_crate_root(
                 core_file,
@@ -308,16 +292,58 @@ impl ChangeFixture {
                 crate_ws_data.clone(),
             );
 
-            for krate in all_crates {
+            (
+                move || {
+                    DependencyBuilder::with_prelude(
+                        CrateName::new("core").unwrap(),
+                        core_crate,
+                        true,
+                        true,
+                    )
+                },
+                core_crate,
+            )
+        });
+
+        if crates.is_empty() {
+            let crate_root = default_crate_root
+                .expect("missing default crate root, specify a main.rs or lib.rs");
+            let root = crate_graph.add_crate_root(
+                crate_root,
+                default_edition,
+                Some(CrateName::new("ra_test_fixture").unwrap().into()),
+                None,
+                default_cfg.clone(),
+                Some(default_cfg),
+                default_env,
+                CrateOrigin::Local { repo: None, name: None },
+                false,
+                proc_macro_cwd.clone(),
+                crate_ws_data.clone(),
+            );
+            if let Some((mini_core, _)) = mini_core {
+                crate_graph.add_dep(root, mini_core()).unwrap();
+            }
+        } else {
+            // Insert minicore first to match with `project-model::workspace`
+            if let Some((mini_core, core_crate)) = mini_core {
+                let all_crates = crate_graph.iter().collect::>();
+                for krate in all_crates {
+                    if krate == core_crate {
+                        continue;
+                    }
+                    crate_graph.add_dep(krate, mini_core()).unwrap();
+                }
+            }
+
+            for (from, to, prelude) in crate_deps {
+                let from_id = crates[&from];
+                let to_id = crates[&to];
+                let sysroot = crate_graph[to_id].basic.origin.is_lang();
                 crate_graph
                     .add_dep(
-                        krate,
-                        DependencyBuilder::with_prelude(
-                            CrateName::new("core").unwrap(),
-                            core_crate,
-                            true,
-                            true,
-                        ),
+                        from_id,
+                        DependencyBuilder::with_prelude(to.clone(), to_id, prelude, sysroot),
                     )
                     .unwrap();
             }
@@ -336,6 +362,8 @@ impl ChangeFixture {
             );
             roots.push(SourceRoot::new_library(fs));
 
+            sysroot_files.push(proc_lib_file);
+
             source_change.change_file(proc_lib_file, Some(source));
 
             let all_crates = crate_graph.iter().collect::>();
@@ -371,6 +399,8 @@ impl ChangeFixture {
             }
         }
 
+        let _ = file_id;
+
         let root = match current_source_root_kind {
             SourceRootKind::Local => SourceRoot::new_local(mem::take(&mut file_set)),
             SourceRootKind::Library => SourceRoot::new_library(mem::take(&mut file_set)),
@@ -382,7 +412,16 @@ impl ChangeFixture {
         change.source_change.set_roots(roots);
         change.source_change.set_crate_graph(crate_graph);
 
-        ChangeFixture { file_position, files, change }
+        ChangeFixture { file_position, file_lines, files, change, sysroot_files }
+    }
+}
+
+fn parse_target_arch(arch: &str) -> base_db::target::Arch {
+    use base_db::target::Arch::*;
+    match arch {
+        "wasm32" => Wasm32,
+        "wasm64" => Wasm64,
+        _ => Other,
     }
 }
 
@@ -627,11 +666,23 @@ impl FileMeta {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ForceNoneLangOrigin {
+    Yes,
+    No,
+}
+
 fn parse_crate(
     crate_str: String,
     current_source_root_kind: SourceRootKind,
     explicit_non_workspace_member: bool,
 ) -> (String, CrateOrigin, Option) {
+    let (crate_str, force_non_lang_origin) = if let Some(s) = crate_str.strip_prefix("r#") {
+        (s.to_owned(), ForceNoneLangOrigin::Yes)
+    } else {
+        (crate_str, ForceNoneLangOrigin::No)
+    };
+
     // syntax:
     //   "my_awesome_crate"
     //   "my_awesome_crate@0.0.1,http://example.com"
@@ -646,16 +697,25 @@ fn parse_crate(
     let non_workspace_member = explicit_non_workspace_member
         || matches!(current_source_root_kind, SourceRootKind::Library);
 
-    let origin = match LangCrateOrigin::from(&*name) {
-        LangCrateOrigin::Other => {
-            let name = Symbol::intern(&name);
-            if non_workspace_member {
-                CrateOrigin::Library { repo, name }
-            } else {
-                CrateOrigin::Local { repo, name: Some(name) }
-            }
+    let origin = if force_non_lang_origin == ForceNoneLangOrigin::Yes {
+        let name = Symbol::intern(&name);
+        if non_workspace_member {
+            CrateOrigin::Library { repo, name }
+        } else {
+            CrateOrigin::Local { repo, name: Some(name) }
+        }
+    } else {
+        match LangCrateOrigin::from(&*name) {
+            LangCrateOrigin::Other => {
+                let name = Symbol::intern(&name);
+                if non_workspace_member {
+                    CrateOrigin::Library { repo, name }
+                } else {
+                    CrateOrigin::Local { repo, name: Some(name) }
+                }
+            }
+            origin => CrateOrigin::Lang(origin),
         }
-        origin => CrateOrigin::Lang(origin),
     };
 
     (name, origin, version)
@@ -955,12 +1015,12 @@ impl ProcMacroExpander for DisallowCfgProcMacroExpander {
         _: String,
     ) -> Result {
         for tt in subtree.token_trees().flat_tokens() {
-            if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt {
-                if ident.sym == sym::cfg || ident.sym == sym::cfg_attr {
-                    return Err(ProcMacroExpansionError::Panic(
-                        "cfg or cfg_attr found in DisallowCfgProcMacroExpander".to_owned(),
-                    ));
-                }
+            if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt
+                && (ident.sym == sym::cfg || ident.sym == sym::cfg_attr)
+            {
+                return Err(ProcMacroExpansionError::Panic(
+                    "cfg or cfg_attr found in DisallowCfgProcMacroExpander".to_owned(),
+                ));
             }
         }
         Ok(subtree.clone())
diff --git a/src/tools/rust-analyzer/crates/test-utils/src/fixture.rs b/src/tools/rust-analyzer/crates/test-utils/src/fixture.rs
index e830c6a7cf68..559894ee6205 100644
--- a/src/tools/rust-analyzer/crates/test-utils/src/fixture.rs
+++ b/src/tools/rust-analyzer/crates/test-utils/src/fixture.rs
@@ -132,13 +132,17 @@ pub struct Fixture {
     pub library: bool,
     /// Actual file contents. All meta comments are stripped.
     pub text: String,
+    /// The line number in the original fixture of the beginning of this fixture.
+    pub line: usize,
 }
 
+#[derive(Debug)]
 pub struct MiniCore {
     activated_flags: Vec,
     valid_flags: Vec,
 }
 
+#[derive(Debug)]
 pub struct FixtureWithProjectMeta {
     pub fixture: Vec,
     pub mini_core: Option,
@@ -149,6 +153,8 @@ pub struct FixtureWithProjectMeta {
     /// You probably don't want to manually specify this. See LLVM manual for the
     /// syntax, if you must: 
     pub target_data_layout: String,
+    /// Specifies the target architecture.
+    pub target_arch: String,
 }
 
 impl FixtureWithProjectMeta {
@@ -178,37 +184,53 @@ impl FixtureWithProjectMeta {
         let mut toolchain = None;
         let mut target_data_layout =
             "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128".to_owned();
+        let mut target_arch = "x86_64".to_owned();
         let mut mini_core = None;
         let mut res: Vec = Vec::new();
         let mut proc_macro_names = vec![];
+        let mut first_row = 0;
 
         if let Some(meta) = fixture.strip_prefix("//- toolchain:") {
+            first_row += 1;
             let (meta, remain) = meta.split_once('\n').unwrap();
             toolchain = Some(meta.trim().to_owned());
             fixture = remain;
         }
 
         if let Some(meta) = fixture.strip_prefix("//- target_data_layout:") {
+            first_row += 1;
             let (meta, remain) = meta.split_once('\n').unwrap();
             meta.trim().clone_into(&mut target_data_layout);
             fixture = remain;
         }
 
+        if let Some(meta) = fixture.strip_prefix("//- target_arch:") {
+            first_row += 1;
+            let (meta, remain) = meta.split_once('\n').unwrap();
+            meta.trim().clone_into(&mut target_arch);
+            fixture = remain;
+        }
+
         if let Some(meta) = fixture.strip_prefix("//- proc_macros:") {
+            first_row += 1;
             let (meta, remain) = meta.split_once('\n').unwrap();
             proc_macro_names = meta.split(',').map(|it| it.trim().to_owned()).collect();
             fixture = remain;
         }
 
         if let Some(meta) = fixture.strip_prefix("//- minicore:") {
+            first_row += 1;
             let (meta, remain) = meta.split_once('\n').unwrap();
             mini_core = Some(MiniCore::parse(meta));
             fixture = remain;
         }
 
-        let default = if fixture.contains("//-") { None } else { Some("//- /main.rs") };
+        let default =
+            if fixture.contains("//- /") { None } else { Some((first_row - 1, "//- /main.rs")) };
 
-        for (ix, line) in default.into_iter().chain(fixture.split_inclusive('\n')).enumerate() {
+        for (ix, line) in
+            default.into_iter().chain((first_row..).zip(fixture.split_inclusive('\n')))
+        {
             if line.contains("//-") {
                 assert!(
                     line.starts_with("//-"),
@@ -219,7 +241,7 @@ impl FixtureWithProjectMeta {
             }
 
             if let Some(line) = line.strip_prefix("//-") {
-                let meta = Self::parse_meta_line(line);
+                let meta = Self::parse_meta_line(line, (ix + 1).try_into().unwrap());
                 res.push(meta);
             } else {
                 if matches!(line.strip_prefix("// "), Some(l) if l.trim().starts_with('/')) {
@@ -232,11 +254,18 @@ impl FixtureWithProjectMeta {
             }
         }
 
-        Self { fixture: res, mini_core, proc_macro_names, toolchain, target_data_layout }
+        Self {
+            fixture: res,
+            mini_core,
+            proc_macro_names,
+            toolchain,
+            target_data_layout,
+            target_arch,
+        }
     }
 
     //- /lib.rs crate:foo deps:bar,baz cfg:foo=a,bar=b env:OUTDIR=path/to,OTHER=foo
-    fn parse_meta_line(meta: &str) -> Fixture {
+    fn parse_meta_line(meta: &str, line: usize) -> Fixture {
         let meta = meta.trim();
         let mut components = meta.split_ascii_whitespace();
 
@@ -301,6 +330,7 @@ impl FixtureWithProjectMeta {
         Fixture {
             path,
             text: String::new(),
+            line,
             krate,
             deps,
             extern_prelude,
@@ -314,7 +344,7 @@ impl FixtureWithProjectMeta {
 }
 
 impl MiniCore {
-    const RAW_SOURCE: &'static str = include_str!("./minicore.rs");
+    pub const RAW_SOURCE: &'static str = include_str!("./minicore.rs");
 
     fn has_flag(&self, flag: &str) -> bool {
         self.activated_flags.iter().any(|it| it == flag)
@@ -347,8 +377,8 @@ impl MiniCore {
         res
     }
 
-    pub fn available_flags() -> impl Iterator {
-        let lines = MiniCore::RAW_SOURCE.split_inclusive('\n');
+    pub fn available_flags(raw_source: &str) -> impl Iterator {
+        let lines = raw_source.split_inclusive('\n');
         lines
             .map_while(|x| x.strip_prefix("//!"))
             .skip_while(|line| !line.contains("Available flags:"))
@@ -359,9 +389,9 @@ impl MiniCore {
     /// Strips parts of minicore.rs which are flagged by inactive flags.
     ///
     /// This is probably over-engineered to support flags dependencies.
-    pub fn source_code(mut self) -> String {
+    pub fn source_code(mut self, raw_source: &str) -> String {
         let mut buf = String::new();
-        let mut lines = MiniCore::RAW_SOURCE.split_inclusive('\n');
+        let mut lines = raw_source.split_inclusive('\n');
 
         let mut implications = Vec::new();
 
@@ -511,6 +541,7 @@ fn parse_fixture_gets_full_meta() {
         proc_macro_names,
         toolchain,
         target_data_layout: _,
+        target_arch: _,
     } = FixtureWithProjectMeta::parse(
         r#"
 //- toolchain: nightly
diff --git a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
index 7b719b5dec75..696928b522f9 100644
--- a/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
+++ b/src/tools/rust-analyzer/crates/test-utils/src/minicore.rs
@@ -35,7 +35,7 @@
 //!     error: fmt
 //!     fmt: option, result, transmute, coerce_unsized, copy, clone, derive
 //!     fmt_before_1_89_0: fmt
-//!     fn: tuple
+//!     fn: sized, tuple
 //!     from: sized, result
 //!     future: pin
 //!     coroutine: pin
@@ -55,7 +55,7 @@
 //!     panic: fmt
 //!     phantom_data:
 //!     pin:
-//!     pointee: copy, send, sync, ord, hash, unpin
+//!     pointee: copy, send, sync, ord, hash, unpin, phantom_data
 //!     range:
 //!     receiver: deref
 //!     result:
@@ -70,6 +70,7 @@
 //!     tuple:
 //!     unpin: sized
 //!     unsize: sized
+//!     write: fmt
 //!     todo: panic
 //!     unimplemented: panic
 //!     column:
@@ -168,6 +169,17 @@ pub mod marker {
     // region:phantom_data
     #[lang = "phantom_data"]
     pub struct PhantomData;
+
+    // region:clone
+    impl Clone for PhantomData {
+        fn clone(&self) -> Self { Self }
+    }
+    // endregion:clone
+
+    // region:copy
+    impl Copy for PhantomData {}
+    // endregion:copy
+
     // endregion:phantom_data
 
     // region:discriminant
@@ -325,6 +337,18 @@ pub mod clone {
             *self
         }
     }
+
+    impl Clone for [T; 0] {
+        fn clone(&self) -> Self {
+            []
+        }
+    }
+
+    impl Clone for [T; 1] {
+        fn clone(&self) -> Self {
+            [self[0].clone()]
+        }
+    }
     // endregion:builtin_impls
 
     // region:derive
@@ -491,6 +515,16 @@ pub mod ptr {
         #[lang = "metadata_type"]
         type Metadata: Copy + Send + Sync + Ord + Hash + Unpin;
     }
+
+    #[lang = "dyn_metadata"]
+    pub struct DynMetadata {
+        _phantom: crate::marker::PhantomData,
+    }
+
+    pub const fn metadata(ptr: *const T) -> ::Metadata {
+        loop {}
+    }
+
     // endregion:pointee
     // region:non_null
     #[rustc_layout_scalar_valid_range_start(1)]
@@ -1035,6 +1069,7 @@ pub mod ops {
 
         #[lang = "coroutine"]
         pub trait Coroutine {
+            #[lang = "coroutine_yield"]
             type Yield;
             #[lang = "coroutine_return"]
             type Return;
@@ -1123,7 +1158,7 @@ pub mod fmt {
 
     pub struct Error;
     pub type Result = crate::result::Result<(), Error>;
-    pub struct Formatter<'a>;
+    pub struct Formatter<'a>(&'a ());
     pub struct DebugTuple;
     pub struct DebugStruct;
     impl Formatter<'_> {
@@ -1596,6 +1631,12 @@ pub mod iter {
                 {
                     loop {}
                 }
+                fn collect>(self) -> B
+                where
+                    Self: Sized,
+                {
+                    loop {}
+                }
                 // endregion:iterators
             }
             impl Iterator for &mut I {
@@ -1665,10 +1706,13 @@ pub mod iter {
                     loop {}
                 }
             }
+            pub trait FromIterator: Sized {
+                fn from_iter>(iter: T) -> Self;
+            }
         }
-        pub use self::collect::IntoIterator;
+        pub use self::collect::{IntoIterator, FromIterator};
     }
-    pub use self::traits::{IntoIterator, Iterator};
+    pub use self::traits::{IntoIterator, FromIterator, Iterator};
 }
 // endregion:iterator
 
@@ -1769,6 +1813,26 @@ mod macros {
     }
     // endregion:panic
 
+    // region:write
+    #[macro_export]
+    macro_rules! write {
+        ($dst:expr, $($arg:tt)*) => {
+            $dst.write_fmt($crate::format_args!($($arg)*))
+        };
+    }
+
+    #[macro_export]
+    #[allow_internal_unstable(format_args_nl)]
+    macro_rules! writeln {
+        ($dst:expr $(,)?) => {
+            $crate::write!($dst, "\n")
+        };
+        ($dst:expr, $($arg:tt)*) => {
+            $dst.write_fmt($crate::format_args_nl!($($arg)*))
+        };
+    }
+    // endregion:write
+
     // region:assert
     #[macro_export]
     #[rustc_builtin_macro]
@@ -1944,7 +2008,7 @@ pub mod prelude {
             convert::AsRef,                          // :as_ref
             convert::{From, Into, TryFrom, TryInto}, // :from
             default::Default,                        // :default
-            iter::{IntoIterator, Iterator},          // :iterator
+            iter::{IntoIterator, Iterator, FromIterator}, // :iterator
             macros::builtin::{derive, derive_const}, // :derive
             marker::Copy,                            // :copy
             marker::Send,                            // :send
diff --git a/src/tools/rust-analyzer/crates/toolchain/src/lib.rs b/src/tools/rust-analyzer/crates/toolchain/src/lib.rs
index 8b7bf1a80641..39319886cfe4 100644
--- a/src/tools/rust-analyzer/crates/toolchain/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/toolchain/src/lib.rs
@@ -71,6 +71,9 @@ impl Tool {
     }
 }
 
+// Prevent rustup from automatically installing toolchains, see https://github.com/rust-lang/rust-analyzer/issues/20719.
+pub const NO_RUSTUP_AUTO_INSTALL_ENV: (&str, &str) = ("RUSTUP_AUTO_INSTALL", "0");
+
 #[allow(clippy::disallowed_types)] /* generic parameter allows for FxHashMap */
 pub fn command(
     cmd: impl AsRef,
@@ -81,6 +84,7 @@ pub fn command(
     #[allow(clippy::disallowed_methods)]
     let mut cmd = Command::new(cmd);
     cmd.current_dir(working_directory);
+    cmd.env(NO_RUSTUP_AUTO_INSTALL_ENV.0, NO_RUSTUP_AUTO_INSTALL_ENV.1);
     for env in extra_env {
         match env {
             (key, Some(val)) => cmd.env(key, val),
diff --git a/src/tools/rust-analyzer/crates/tt/src/lib.rs b/src/tools/rust-analyzer/crates/tt/src/lib.rs
index 44123385c8cc..243a27b83b0d 100644
--- a/src/tools/rust-analyzer/crates/tt/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/tt/src/lib.rs
@@ -357,10 +357,10 @@ impl<'a, S: Copy> TokenTreesView<'a, S> {
     }
 
     pub fn try_into_subtree(self) -> Option> {
-        if let Some(TokenTree::Subtree(subtree)) = self.0.first() {
-            if subtree.usize_len() == (self.0.len() - 1) {
-                return Some(SubtreeView::new(self.0));
-            }
+        if let Some(TokenTree::Subtree(subtree)) = self.0.first()
+            && subtree.usize_len() == (self.0.len() - 1)
+        {
+            return Some(SubtreeView::new(self.0));
         }
         None
     }
@@ -1028,10 +1028,10 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String {
         tkns = rest;
         last = [last, tokentree_to_text(tkn, &mut tkns)].join(if last_to_joint { "" } else { " " });
         last_to_joint = false;
-        if let TokenTree::Leaf(Leaf::Punct(punct)) = tkn {
-            if punct.spacing == Spacing::Joint {
-                last_to_joint = true;
-            }
+        if let TokenTree::Leaf(Leaf::Punct(punct)) = tkn
+            && punct.spacing == Spacing::Joint
+        {
+            last_to_joint = true;
         }
     }
     last
diff --git a/src/tools/rust-analyzer/crates/vfs-notify/src/lib.rs b/src/tools/rust-analyzer/crates/vfs-notify/src/lib.rs
index a03337dbc51e..c6393cc6922a 100644
--- a/src/tools/rust-analyzer/crates/vfs-notify/src/lib.rs
+++ b/src/tools/rust-analyzer/crates/vfs-notify/src/lib.rs
@@ -194,52 +194,49 @@ impl NotifyActor {
                     }
                 },
                 Event::NotifyEvent(event) => {
-                    if let Some(event) = log_notify_error(event) {
-                        if let EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) =
+                    if let Some(event) = log_notify_error(event)
+                        && let EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) =
                             event.kind
-                        {
-                            let files = event
-                                .paths
-                                .into_iter()
-                                .filter_map(|path| {
-                                    Some(
-                                        AbsPathBuf::try_from(
-                                            Utf8PathBuf::from_path_buf(path).ok()?,
-                                        )
+                    {
+                        let files = event
+                            .paths
+                            .into_iter()
+                            .filter_map(|path| {
+                                Some(
+                                    AbsPathBuf::try_from(Utf8PathBuf::from_path_buf(path).ok()?)
                                         .expect("path is absolute"),
-                                    )
-                                })
-                                .filter_map(|path| -> Option<(AbsPathBuf, Option>)> {
-                                    let meta = fs::metadata(&path).ok()?;
-                                    if meta.file_type().is_dir()
-                                        && self
-                                            .watched_dir_entries
-                                            .iter()
-                                            .any(|dir| dir.contains_dir(&path))
-                                    {
-                                        self.watch(path.as_ref());
-                                        return None;
-                                    }
+                                )
+                            })
+                            .filter_map(|path| -> Option<(AbsPathBuf, Option>)> {
+                                let meta = fs::metadata(&path).ok()?;
+                                if meta.file_type().is_dir()
+                                    && self
+                                        .watched_dir_entries
+                                        .iter()
+                                        .any(|dir| dir.contains_dir(&path))
+                                {
+                                    self.watch(path.as_ref());
+                                    return None;
+                                }
 
-                                    if !meta.file_type().is_file() {
-                                        return None;
-                                    }
+                                if !meta.file_type().is_file() {
+                                    return None;
+                                }
 
-                                    if !(self.watched_file_entries.contains(&path)
-                                        || self
-                                            .watched_dir_entries
-                                            .iter()
-                                            .any(|dir| dir.contains_file(&path)))
-                                    {
-                                        return None;
-                                    }
+                                if !(self.watched_file_entries.contains(&path)
+                                    || self
+                                        .watched_dir_entries
+                                        .iter()
+                                        .any(|dir| dir.contains_file(&path)))
+                                {
+                                    return None;
+                                }
 
-                                    let contents = read(&path);
-                                    Some((path, contents))
-                                })
-                                .collect();
-                            self.send(loader::Message::Changed { files });
-                        }
+                                let contents = read(&path);
+                                Some((path, contents))
+                            })
+                            .collect();
+                        self.send(loader::Message::Changed { files });
                     }
                 }
             }
diff --git a/src/tools/rust-analyzer/crates/vfs/src/path_interner.rs b/src/tools/rust-analyzer/crates/vfs/src/path_interner.rs
index 64f51976053d..225bfc7218b4 100644
--- a/src/tools/rust-analyzer/crates/vfs/src/path_interner.rs
+++ b/src/tools/rust-analyzer/crates/vfs/src/path_interner.rs
@@ -28,7 +28,7 @@ impl PathInterner {
     /// - Else, returns a newly allocated id.
     pub(crate) fn intern(&mut self, path: VfsPath) -> FileId {
         let (id, _added) = self.map.insert_full(path);
-        assert!(id < u32::MAX as usize);
+        assert!(id < FileId::MAX as usize);
         FileId(id as u32)
     }
 
diff --git a/src/tools/rust-analyzer/docs/book/README.md b/src/tools/rust-analyzer/docs/book/README.md
index 0a3161f3af38..11f7e8f98ca5 100644
--- a/src/tools/rust-analyzer/docs/book/README.md
+++ b/src/tools/rust-analyzer/docs/book/README.md
@@ -8,6 +8,7 @@ To run the documentation site locally:
 
 ```shell
 cargo install mdbook
+cargo install mdbook-toc
 cargo xtask codegen
 cd docs/book
 mdbook serve
diff --git a/src/tools/rust-analyzer/docs/book/src/README.md b/src/tools/rust-analyzer/docs/book/src/README.md
index 71f34e034664..060bcf0cea39 100644
--- a/src/tools/rust-analyzer/docs/book/src/README.md
+++ b/src/tools/rust-analyzer/docs/book/src/README.md
@@ -1,13 +1,20 @@
 # rust-analyzer
 
-At its core, rust-analyzer is a **library** for semantic analysis of
-Rust code as it changes over time. This manual focuses on a specific
-usage of the library -- running it as part of a server that implements
+rust-analyzer is a language server that provides IDE functionality for
+writing Rust programs. You can use it with any editor that supports
 the [Language Server
-Protocol](https://microsoft.github.io/language-server-protocol/) (LSP).
-The LSP allows various code editors, like VS Code, Emacs or Vim, to
-implement semantic features like completion or goto definition by
-talking to an external language server process.
+Protocol](https://microsoft.github.io/language-server-protocol/) (VS
+Code, Vim, Emacs, Zed, etc).
+
+rust-analyzer features include go-to-definition, find-all-references,
+refactorings and code completion. rust-analyzer also supports
+integrated formatting (with rustfmt) and integrated diagnostics (with
+rustc and clippy).
+
+Internally, rust-analyzer is structured as a set of libraries for
+analyzing Rust code. See
+[Architecture](https://rust-analyzer.github.io/book/contributing/architecture.html)
+for more details.
 
 To improve this document, send a pull request:
 [https://github.com/rust-lang/rust-analyzer](https://github.com/rust-lang/rust-analyzer/blob/master/docs/book/README.md)
diff --git a/src/tools/rust-analyzer/docs/book/src/SUMMARY.md b/src/tools/rust-analyzer/docs/book/src/SUMMARY.md
index dffdae94a6e8..3fb46d59a4a5 100644
--- a/src/tools/rust-analyzer/docs/book/src/SUMMARY.md
+++ b/src/tools/rust-analyzer/docs/book/src/SUMMARY.md
@@ -23,3 +23,4 @@
   - [Setup](contributing/setup.md)
   - [Style](contributing/style.md)
   - [Syntax](contributing/syntax.md)
+  - [Writing Tests](contributing/testing.md)
diff --git a/src/tools/rust-analyzer/docs/book/src/configuration.md b/src/tools/rust-analyzer/docs/book/src/configuration.md
index fd94a4221a92..708eecd43561 100644
--- a/src/tools/rust-analyzer/docs/book/src/configuration.md
+++ b/src/tools/rust-analyzer/docs/book/src/configuration.md
@@ -13,7 +13,7 @@ Vim](./other_editors.md#coc-rust-analyzer) provide `rust-analyzer` specific conf
 UIs. Others may require you to know a bit more about the interaction
 with `rust-analyzer`.
 
-For the later category, it might help to know that the initial
+For the latter category, it might help to know that the initial
 configuration is specified as a value of the `initializationOptions`
 field of the [`InitializeParams` message, in the LSP
 protocol](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#initialize).
@@ -46,6 +46,11 @@ To verify which configuration is actually used by `rust-analyzer`, set
 config-related messages. Logs should show both the JSON that
 `rust-analyzer` sees as well as the updated config.
 
+(Work in progress:) It is also possible to place configuration in a
+`rust-analyzer.toml` file. It should be located in the project root or in your
+user configuration directory (e.g. `~/.config/rust-analyzer/`). This is a work in
+progress, many configuration options aren't supported yet.
+
 This is the list of config options `rust-analyzer` supports:
 
 {{#include configuration_generated.md}}
diff --git a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
index ebac26e1d60a..d768993f501f 100644
--- a/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
+++ b/src/tools/rust-analyzer/docs/book/src/configuration_generated.md
@@ -2,8 +2,7 @@
 
 Default: `false`
 
-Whether to insert #[must_use] when generating `as_` methods
-for enum variants.
+Insert #[must_use] when generating `as_` methods for enum variants.
 
 
 ## rust-analyzer.assist.expressionFillDefault {#assist.expressionFillDefault}
@@ -17,14 +16,15 @@ Placeholder expression to use for missing expressions in assists.
 
 Default: `false`
 
-When inserting a type (e.g. in "fill match arms" assist), prefer to use `Self` over the type name where possible.
+Prefer to use `Self` over the type name when inserting a type (e.g. in "fill match arms" assist).
 
 
 ## rust-analyzer.assist.termSearch.borrowcheck {#assist.termSearch.borrowcheck}
 
 Default: `true`
 
-Enable borrow checking for term search code assists. If set to false, also there will be more suggestions, but some of them may not borrow-check.
+Enable borrow checking for term search code assists. If set to false, also there will be
+more suggestions, but some of them may not borrow-check.
 
 
 ## rust-analyzer.assist.termSearch.fuel {#assist.termSearch.fuel}
@@ -45,7 +45,8 @@ Warm up caches on project load.
 
 Default: `"physical"`
 
-How many worker threads to handle priming caches. The default `0` means to pick automatically.
+How many worker threads to handle priming caches. The default `0` means to pick
+automatically.
 
 
 ## rust-analyzer.cargo.allTargets {#cargo.allTargets}
@@ -103,7 +104,9 @@ targets and features, with the following base command line:
 ```bash
 cargo check --quiet --workspace --message-format=json --all-targets --keep-going
 ```
-.
+
+Note: The option must be specified as an array of command line arguments, with
+the first argument being the name of the command to run.
 
 
 ## rust-analyzer.cargo.buildScripts.rebuildOnSave {#cargo.buildScripts.rebuildOnSave}
@@ -330,7 +333,9 @@ An example command would be:
 ```bash
 cargo check --workspace --message-format=json --all-targets
 ```
-.
+
+Note: The option must be specified as an array of command line arguments, with
+the first argument being the name of the command to run.
 
 
 ## rust-analyzer.check.targets {#check.targets}
@@ -358,7 +363,7 @@ check will be performed.
 
 Default: `true`
 
-Whether to automatically add a semicolon when completing unit-returning functions.
+Automatically add a semicolon when completing unit-returning functions.
 
 In `match` arms it completes a comma instead.
 
@@ -367,22 +372,26 @@ In `match` arms it completes a comma instead.
 
 Default: `true`
 
-Toggles the additional completions that automatically show method calls and field accesses with `await` prefixed to them when completing on a future.
+Show method calls and field accesses completions with `await` prefixed to them when
+completing on a future.
 
 
 ## rust-analyzer.completion.autoIter.enable {#completion.autoIter.enable}
 
 Default: `true`
 
-Toggles the additional completions that automatically show method calls with `iter()` or `into_iter()` prefixed to them when completing on a type that has them.
+Show method call completions with `iter()` or `into_iter()` prefixed to them when
+completing on a type that has them.
 
 
 ## rust-analyzer.completion.autoimport.enable {#completion.autoimport.enable}
 
 Default: `true`
 
-Toggles the additional completions that automatically add imports when completed.
-Note that your client must specify the `additionalTextEdits` LSP client capability to truly have this feature enabled.
+Show completions that automatically add imports when completed.
+
+Note that your client must specify the `additionalTextEdits` LSP client capability to
+truly have this feature enabled.
 
 
 ## rust-analyzer.completion.autoimport.exclude {#completion.autoimport.exclude}
@@ -406,10 +415,11 @@ A list of full paths to items to exclude from auto-importing completions.
 Traits in this list won't have their methods suggested in completions unless the trait
 is in scope.
 
-You can either specify a string path which defaults to type "always" or use the more verbose
-form `{ "path": "path::to::item", type: "always" }`.
+You can either specify a string path which defaults to type "always" or use the more
+verbose form `{ "path": "path::to::item", type: "always" }`.
 
-For traits the type "methods" can be used to only exclude the methods but not the trait itself.
+For traits the type "methods" can be used to only exclude the methods but not the trait
+itself.
 
 This setting also inherits `#rust-analyzer.completion.excludeTraits#`.
 
@@ -418,15 +428,15 @@ This setting also inherits `#rust-analyzer.completion.excludeTraits#`.
 
 Default: `true`
 
-Toggles the additional completions that automatically show method calls and field accesses
-with `self` prefixed to them when inside a method.
+Show method calls and field access completions with `self` prefixed to them when
+inside a method.
 
 
 ## rust-analyzer.completion.callable.snippets {#completion.callable.snippets}
 
 Default: `"fill_arguments"`
 
-Whether to add parenthesis and argument snippets when completing function.
+Add parenthesis and argument snippets when completing function.
 
 
 ## rust-analyzer.completion.excludeTraits {#completion.excludeTraits}
@@ -435,7 +445,9 @@ Default: `[]`
 
 A list of full paths to traits whose methods to exclude from completion.
 
-Methods from these traits won't be completed, even if the trait is in scope. However, they will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or `T where T: Trait`.
+Methods from these traits won't be completed, even if the trait is in scope. However,
+they will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or
+`T where T: Trait`.
 
 Note that the trait themselves can still be completed.
 
@@ -444,14 +456,15 @@ Note that the trait themselves can still be completed.
 
 Default: `false`
 
-Whether to show full function/method signatures in completion docs.
+Show full function / method signatures in completion docs.
 
 
 ## rust-analyzer.completion.hideDeprecated {#completion.hideDeprecated}
 
 Default: `false`
 
-Whether to omit deprecated items from autocompletion. By default they are marked as deprecated but not hidden.
+Omit deprecated items from completions. By default they are marked as deprecated but not
+hidden.
 
 
 ## rust-analyzer.completion.limit {#completion.limit}
@@ -465,14 +478,15 @@ Maximum number of completions to return. If `None`, the limit is infinite.
 
 Default: `true`
 
-Whether to show postfix snippets like `dbg`, `if`, `not`, etc.
+Show postfix snippets like `dbg`, `if`, `not`, etc.
 
 
 ## rust-analyzer.completion.privateEditable.enable {#completion.privateEditable.enable}
 
 Default: `false`
 
-Enables completions of private items and fields that are defined in the current workspace even if they are not visible at the current position.
+Show completions of private items and fields that are defined in the current workspace
+even if they are not visible at the current position.
 
 
 ## rust-analyzer.completion.snippets.custom {#completion.snippets.custom}
@@ -529,7 +543,7 @@ Custom completion snippets.
 
 Default: `false`
 
-Whether to enable term search based snippets like `Some(foo.bar().baz())`.
+Enable term search based snippets like `Some(foo.bar().baz())`.
 
 
 ## rust-analyzer.completion.termSearch.fuel {#completion.termSearch.fuel}
@@ -550,30 +564,30 @@ List of rust-analyzer diagnostics to disable.
 
 Default: `true`
 
-Whether to show native rust-analyzer diagnostics.
+Show native rust-analyzer diagnostics.
 
 
 ## rust-analyzer.diagnostics.experimental.enable {#diagnostics.experimental.enable}
 
 Default: `false`
 
-Whether to show experimental rust-analyzer diagnostics that might
-have more false positives than usual.
+Show experimental rust-analyzer diagnostics that might have more false positives than
+usual.
 
 
 ## rust-analyzer.diagnostics.remapPrefix {#diagnostics.remapPrefix}
 
 Default: `{}`
 
-Map of prefixes to be substituted when parsing diagnostic file paths.
-This should be the reverse mapping of what is passed to `rustc` as `--remap-path-prefix`.
+Map of prefixes to be substituted when parsing diagnostic file paths. This should be the
+reverse mapping of what is passed to `rustc` as `--remap-path-prefix`.
 
 
 ## rust-analyzer.diagnostics.styleLints.enable {#diagnostics.styleLints.enable}
 
 Default: `false`
 
-Whether to run additional style lints.
+Run additional style lints.
 
 
 ## rust-analyzer.diagnostics.warningsAsHint {#diagnostics.warningsAsHint}
@@ -582,8 +596,8 @@ Default: `[]`
 
 List of warnings that should be displayed with hint severity.
 
-The warnings will be indicated by faded text or three dots in code
-and will not show up in the `Problems Panel`.
+The warnings will be indicated by faded text or three dots in code and will not show up
+in the `Problems Panel`.
 
 
 ## rust-analyzer.diagnostics.warningsAsInfo {#diagnostics.warningsAsInfo}
@@ -592,17 +606,26 @@ Default: `[]`
 
 List of warnings that should be displayed with info severity.
 
-The warnings will be indicated by a blue squiggly underline in code
-and a blue icon in the `Problems Panel`.
+The warnings will be indicated by a blue squiggly underline in code and a blue icon in
+the `Problems Panel`.
+
+
+## rust-analyzer.document.symbol.search.excludeLocals {#document.symbol.search.excludeLocals}
+
+Default: `true`
+
+Exclude all locals from document symbol search.
 
 
 ## rust-analyzer.files.exclude {#files.exclude}
 
 Default: `[]`
 
-These paths (file/directories) will be ignored by rust-analyzer. They are
-relative to the workspace root, and globs are not supported. You may
-also need to add the folders to Code's `files.watcherExclude`.
+List of files to ignore
+
+These paths (file/directories) will be ignored by rust-analyzer. They are relative to
+the workspace root, and globs are not supported. You may also need to add the folders to
+Code's `files.watcherExclude`.
 
 
 ## rust-analyzer.files.watcher {#files.watcher}
@@ -616,64 +639,67 @@ Controls file watching implementation.
 
 Default: `true`
 
-Enables highlighting of related return values while the cursor is on any `match`, `if`, or match arm arrow (`=>`).
+Highlight related return values while the cursor is on any `match`, `if`, or match arm
+arrow (`=>`).
 
 
 ## rust-analyzer.highlightRelated.breakPoints.enable {#highlightRelated.breakPoints.enable}
 
 Default: `true`
 
-Enables highlighting of related references while the cursor is on `break`, `loop`, `while`, or `for` keywords.
+Highlight related references while the cursor is on `break`, `loop`, `while`, or `for`
+keywords.
 
 
 ## rust-analyzer.highlightRelated.closureCaptures.enable {#highlightRelated.closureCaptures.enable}
 
 Default: `true`
 
-Enables highlighting of all captures of a closure while the cursor is on the `|` or move keyword of a closure.
+Highlight all captures of a closure while the cursor is on the `|` or move keyword of a closure.
 
 
 ## rust-analyzer.highlightRelated.exitPoints.enable {#highlightRelated.exitPoints.enable}
 
 Default: `true`
 
-Enables highlighting of all exit points while the cursor is on any `return`, `?`, `fn`, or return type arrow (`->`).
+Highlight all exit points while the cursor is on any `return`, `?`, `fn`, or return type
+arrow (`->`).
 
 
 ## rust-analyzer.highlightRelated.references.enable {#highlightRelated.references.enable}
 
 Default: `true`
 
-Enables highlighting of related references while the cursor is on any identifier.
+Highlight related references while the cursor is on any identifier.
 
 
 ## rust-analyzer.highlightRelated.yieldPoints.enable {#highlightRelated.yieldPoints.enable}
 
 Default: `true`
 
-Enables highlighting of all break points for a loop or block context while the cursor is on any `async` or `await` keywords.
+Highlight all break points for a loop or block context while the cursor is on any
+`async` or `await` keywords.
 
 
 ## rust-analyzer.hover.actions.debug.enable {#hover.actions.debug.enable}
 
 Default: `true`
 
-Whether to show `Debug` action. Only applies when
-`#rust-analyzer.hover.actions.enable#` is set.
+Show `Debug` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.
 
 
 ## rust-analyzer.hover.actions.enable {#hover.actions.enable}
 
 Default: `true`
 
-Whether to show HoverActions in Rust files.
+Show HoverActions in Rust files.
 
 
 ## rust-analyzer.hover.actions.gotoTypeDef.enable {#hover.actions.gotoTypeDef.enable}
 
 Default: `true`
 
-Whether to show `Go to Type Definition` action. Only applies when
+Show `Go to Type Definition` action. Only applies when
 `#rust-analyzer.hover.actions.enable#` is set.
 
 
@@ -681,46 +707,45 @@ Whether to show `Go to Type Definition` action. Only applies when
 
 Default: `true`
 
-Whether to show `Implementations` action. Only applies when
-`#rust-analyzer.hover.actions.enable#` is set.
+Show `Implementations` action. Only applies when `#rust-analyzer.hover.actions.enable#`
+is set.
 
 
 ## rust-analyzer.hover.actions.references.enable {#hover.actions.references.enable}
 
 Default: `false`
 
-Whether to show `References` action. Only applies when
-`#rust-analyzer.hover.actions.enable#` is set.
+Show `References` action. Only applies when `#rust-analyzer.hover.actions.enable#` is
+set.
 
 
 ## rust-analyzer.hover.actions.run.enable {#hover.actions.run.enable}
 
 Default: `true`
 
-Whether to show `Run` action. Only applies when
-`#rust-analyzer.hover.actions.enable#` is set.
+Show `Run` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.
 
 
 ## rust-analyzer.hover.actions.updateTest.enable {#hover.actions.updateTest.enable}
 
 Default: `true`
 
-Whether to show `Update Test` action. Only applies when
-`#rust-analyzer.hover.actions.enable#` and `#rust-analyzer.hover.actions.run.enable#` are set.
+Show `Update Test` action. Only applies when `#rust-analyzer.hover.actions.enable#` and
+`#rust-analyzer.hover.actions.run.enable#` are set.
 
 
 ## rust-analyzer.hover.documentation.enable {#hover.documentation.enable}
 
 Default: `true`
 
-Whether to show documentation on hover.
+Show documentation on hover.
 
 
 ## rust-analyzer.hover.documentation.keywords.enable {#hover.documentation.keywords.enable}
 
 Default: `true`
 
-Whether to show keyword hover popups. Only applies when
+Show keyword hover popups. Only applies when
 `#rust-analyzer.hover.documentation.enable#` is set.
 
 
@@ -728,7 +753,7 @@ Whether to show keyword hover popups. Only applies when
 
 Default: `true`
 
-Whether to show drop glue information on hover.
+Show drop glue information on hover.
 
 
 ## rust-analyzer.hover.links.enable {#hover.links.enable}
@@ -742,9 +767,11 @@ Use markdown syntax for links on hover.
 
 Default: `20`
 
-Whether to show what types are used as generic arguments in calls etc. on hover, and what is their max length to show such types, beyond it they will be shown with ellipsis.
+Show what types are used as generic arguments in calls etc. on hover, and limit the max
+length to show such types, beyond which they will be shown with ellipsis.
 
-This can take three values: `null` means "unlimited", the string `"hide"` means to not show generic substitutions at all, and a number means to limit them to X characters.
+This can take three values: `null` means "unlimited", the string `"hide"` means to not
+show generic substitutions at all, and a number means to limit them to X characters.
 
 The default is 20 characters.
 
@@ -760,7 +787,7 @@ How to render the align information in a memory layout hover.
 
 Default: `true`
 
-Whether to show memory layout data on hover.
+Show memory layout data on hover.
 
 
 ## rust-analyzer.hover.memoryLayout.niches {#hover.memoryLayout.niches}
@@ -802,7 +829,8 @@ How many variants of an enum to display when hovering on. Show none if empty.
 
 Default: `5`
 
-How many fields of a struct, variant or union to display when hovering on. Show none if empty.
+How many fields of a struct, variant or union to display when hovering on. Show none if
+empty.
 
 
 ## rust-analyzer.hover.show.traitAssocItems {#hover.show.traitAssocItems}
@@ -816,7 +844,8 @@ How many associated items of a trait to display when hovering a trait.
 
 Default: `false`
 
-Whether to enforce the import granularity setting for all files. If set to false rust-analyzer will try to keep import styles consistent per file.
+Enforce the import granularity setting for all files. If set to false rust-analyzer will
+try to keep import styles consistent per file.
 
 
 ## rust-analyzer.imports.granularity.group {#imports.granularity.group}
@@ -830,14 +859,17 @@ How imports should be grouped into use statements.
 
 Default: `true`
 
-Group inserted imports by the [following order](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are separated by newlines.
+Group inserted imports by the [following
+order](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are
+separated by newlines.
 
 
 ## rust-analyzer.imports.merge.glob {#imports.merge.glob}
 
 Default: `true`
 
-Whether to allow import insertion to merge new imports into single path glob imports like `use std::fmt::*;`.
+Allow import insertion to merge new imports into single path glob imports like `use
+std::fmt::*;`.
 
 
 ## rust-analyzer.imports.preferNoStd {#imports.preferNoStd}
@@ -851,7 +883,7 @@ Prefer to unconditionally use imports of the core and alloc crate, over the std
 
 Default: `false`
 
-Whether to prefer import paths containing a `prelude` module.
+Prefer import paths containing a `prelude` module.
 
 
 ## rust-analyzer.imports.prefix {#imports.prefix}
@@ -865,28 +897,30 @@ The path structure for newly inserted paths to use.
 
 Default: `false`
 
-Whether to prefix external (including std, core) crate imports with `::`. e.g. "use ::std::io::Read;".
+Prefix external (including std, core) crate imports with `::`.
+
+E.g. `use ::std::io::Read;`.
 
 
 ## rust-analyzer.inlayHints.bindingModeHints.enable {#inlayHints.bindingModeHints.enable}
 
 Default: `false`
 
-Whether to show inlay type hints for binding modes.
+Show inlay type hints for binding modes.
 
 
 ## rust-analyzer.inlayHints.chainingHints.enable {#inlayHints.chainingHints.enable}
 
 Default: `true`
 
-Whether to show inlay type hints for method chains.
+Show inlay type hints for method chains.
 
 
 ## rust-analyzer.inlayHints.closingBraceHints.enable {#inlayHints.closingBraceHints.enable}
 
 Default: `true`
 
-Whether to show inlay hints after a closing `}` to indicate what item it belongs to.
+Show inlay hints after a closing `}` to indicate what item it belongs to.
 
 
 ## rust-analyzer.inlayHints.closingBraceHints.minLines {#inlayHints.closingBraceHints.minLines}
@@ -901,14 +935,14 @@ to always show them).
 
 Default: `false`
 
-Whether to show inlay hints for closure captures.
+Show inlay hints for closure captures.
 
 
 ## rust-analyzer.inlayHints.closureReturnTypeHints.enable {#inlayHints.closureReturnTypeHints.enable}
 
 Default: `"never"`
 
-Whether to show inlay type hints for return types of closures.
+Show inlay type hints for return types of closures.
 
 
 ## rust-analyzer.inlayHints.closureStyle {#inlayHints.closureStyle}
@@ -922,77 +956,88 @@ Closure notation in type and chaining inlay hints.
 
 Default: `"never"`
 
-Whether to show enum variant discriminant hints.
+Show enum variant discriminant hints.
+
+
+## rust-analyzer.inlayHints.expressionAdjustmentHints.disableReborrows {#inlayHints.expressionAdjustmentHints.disableReborrows}
+
+Default: `true`
+
+Disable reborrows in expression adjustments inlay hints.
+
+Reborrows are a pair of a builtin deref then borrow, i.e. `&*`. They are inserted by the compiler but are mostly useless to the programmer.
+
+Note: if the deref is not builtin (an overloaded deref), or the borrow is `&raw const`/`&raw mut`, they are not removed.
 
 
 ## rust-analyzer.inlayHints.expressionAdjustmentHints.enable {#inlayHints.expressionAdjustmentHints.enable}
 
 Default: `"never"`
 
-Whether to show inlay hints for type adjustments.
+Show inlay hints for type adjustments.
 
 
 ## rust-analyzer.inlayHints.expressionAdjustmentHints.hideOutsideUnsafe {#inlayHints.expressionAdjustmentHints.hideOutsideUnsafe}
 
 Default: `false`
 
-Whether to hide inlay hints for type adjustments outside of `unsafe` blocks.
+Hide inlay hints for type adjustments outside of `unsafe` blocks.
 
 
 ## rust-analyzer.inlayHints.expressionAdjustmentHints.mode {#inlayHints.expressionAdjustmentHints.mode}
 
 Default: `"prefix"`
 
-Whether to show inlay hints as postfix ops (`.*` instead of `*`, etc).
+Show inlay hints as postfix ops (`.*` instead of `*`, etc).
 
 
 ## rust-analyzer.inlayHints.genericParameterHints.const.enable {#inlayHints.genericParameterHints.const.enable}
 
 Default: `true`
 
-Whether to show const generic parameter name inlay hints.
+Show const generic parameter name inlay hints.
 
 
 ## rust-analyzer.inlayHints.genericParameterHints.lifetime.enable {#inlayHints.genericParameterHints.lifetime.enable}
 
 Default: `false`
 
-Whether to show generic lifetime parameter name inlay hints.
+Show generic lifetime parameter name inlay hints.
 
 
 ## rust-analyzer.inlayHints.genericParameterHints.type.enable {#inlayHints.genericParameterHints.type.enable}
 
 Default: `false`
 
-Whether to show generic type parameter name inlay hints.
+Show generic type parameter name inlay hints.
 
 
 ## rust-analyzer.inlayHints.implicitDrops.enable {#inlayHints.implicitDrops.enable}
 
 Default: `false`
 
-Whether to show implicit drop hints.
+Show implicit drop hints.
 
 
 ## rust-analyzer.inlayHints.implicitSizedBoundHints.enable {#inlayHints.implicitSizedBoundHints.enable}
 
 Default: `false`
 
-Whether to show inlay hints for the implied type parameter `Sized` bound.
+Show inlay hints for the implied type parameter `Sized` bound.
 
 
 ## rust-analyzer.inlayHints.lifetimeElisionHints.enable {#inlayHints.lifetimeElisionHints.enable}
 
 Default: `"never"`
 
-Whether to show inlay type hints for elided lifetimes in function signatures.
+Show inlay type hints for elided lifetimes in function signatures.
 
 
 ## rust-analyzer.inlayHints.lifetimeElisionHints.useParameterNames {#inlayHints.lifetimeElisionHints.useParameterNames}
 
 Default: `false`
 
-Whether to prefer using parameter names as the name for elided lifetime hints if possible.
+Prefer using parameter names as the name for elided lifetime hints if possible.
 
 
 ## rust-analyzer.inlayHints.maxLength {#inlayHints.maxLength}
@@ -1001,28 +1046,31 @@ Default: `25`
 
 Maximum length for inlay hints. Set to null to have an unlimited length.
 
+**Note:** This is mostly a hint, and we don't guarantee to strictly follow the limit.
+
 
 ## rust-analyzer.inlayHints.parameterHints.enable {#inlayHints.parameterHints.enable}
 
 Default: `true`
 
-Whether to show function parameter name inlay hints at the call
-site.
+Show function parameter name inlay hints at the call site.
 
 
 ## rust-analyzer.inlayHints.rangeExclusiveHints.enable {#inlayHints.rangeExclusiveHints.enable}
 
 Default: `false`
 
-Whether to show exclusive range inlay hints.
+Show exclusive range inlay hints.
 
 
 ## rust-analyzer.inlayHints.reborrowHints.enable {#inlayHints.reborrowHints.enable}
 
 Default: `"never"`
 
-Whether to show inlay hints for compiler inserted reborrows.
-This setting is deprecated in favor of #rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.
+Show inlay hints for compiler inserted reborrows.
+
+This setting is deprecated in favor of
+#rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.
 
 
 ## rust-analyzer.inlayHints.renderColons {#inlayHints.renderColons}
@@ -1036,36 +1084,38 @@ Whether to render leading colons for type hints, and trailing colons for paramet
 
 Default: `true`
 
-Whether to show inlay type hints for variables.
+Show inlay type hints for variables.
 
 
 ## rust-analyzer.inlayHints.typeHints.hideClosureInitialization {#inlayHints.typeHints.hideClosureInitialization}
 
 Default: `false`
 
-Whether to hide inlay type hints for `let` statements that initialize to a closure.
-Only applies to closures with blocks, same as `#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.
+Hide inlay type hints for `let` statements that initialize to a closure.
+
+Only applies to closures with blocks, same as
+`#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.
 
 
 ## rust-analyzer.inlayHints.typeHints.hideClosureParameter {#inlayHints.typeHints.hideClosureParameter}
 
 Default: `false`
 
-Whether to hide inlay parameter type hints for closures.
+Hide inlay parameter type hints for closures.
 
 
 ## rust-analyzer.inlayHints.typeHints.hideNamedConstructor {#inlayHints.typeHints.hideNamedConstructor}
 
 Default: `false`
 
-Whether to hide inlay type hints for constructors.
+Hide inlay type hints for constructors.
 
 
 ## rust-analyzer.interpret.tests {#interpret.tests}
 
 Default: `false`
 
-Enables the experimental support for interpreting tests.
+Enable the experimental support for interpreting tests.
 
 
 ## rust-analyzer.joinLines.joinAssignments {#joinLines.joinAssignments}
@@ -1100,23 +1150,21 @@ Join lines unwraps trivial blocks.
 
 Default: `true`
 
-Whether to show `Debug` lens. Only applies when
-`#rust-analyzer.lens.enable#` is set.
+Show `Debug` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.enable {#lens.enable}
 
 Default: `true`
 
-Whether to show CodeLens in Rust files.
+Show CodeLens in Rust files.
 
 
 ## rust-analyzer.lens.implementations.enable {#lens.implementations.enable}
 
 Default: `true`
 
-Whether to show `Implementations` lens. Only applies when
-`#rust-analyzer.lens.enable#` is set.
+Show `Implementations` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.location {#lens.location}
@@ -1130,60 +1178,56 @@ Where to render annotations.
 
 Default: `false`
 
-Whether to show `References` lens for Struct, Enum, and Union.
-Only applies when `#rust-analyzer.lens.enable#` is set.
+Show `References` lens for Struct, Enum, and Union. Only applies when
+`#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.references.enumVariant.enable {#lens.references.enumVariant.enable}
 
 Default: `false`
 
-Whether to show `References` lens for Enum Variants.
-Only applies when `#rust-analyzer.lens.enable#` is set.
+Show `References` lens for Enum Variants. Only applies when
+`#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.references.method.enable {#lens.references.method.enable}
 
 Default: `false`
 
-Whether to show `Method References` lens. Only applies when
-`#rust-analyzer.lens.enable#` is set.
+Show `Method References` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.references.trait.enable {#lens.references.trait.enable}
 
 Default: `false`
 
-Whether to show `References` lens for Trait.
-Only applies when `#rust-analyzer.lens.enable#` is set.
+Show `References` lens for Trait. Only applies when `#rust-analyzer.lens.enable#` is
+set.
 
 
 ## rust-analyzer.lens.run.enable {#lens.run.enable}
 
 Default: `true`
 
-Whether to show `Run` lens. Only applies when
-`#rust-analyzer.lens.enable#` is set.
+Show `Run` lens. Only applies when `#rust-analyzer.lens.enable#` is set.
 
 
 ## rust-analyzer.lens.updateTest.enable {#lens.updateTest.enable}
 
 Default: `true`
 
-Whether to show `Update Test` lens. Only applies when
-`#rust-analyzer.lens.enable#` and `#rust-analyzer.lens.run.enable#` are set.
+Show `Update Test` lens. Only applies when `#rust-analyzer.lens.enable#` and
+`#rust-analyzer.lens.run.enable#` are set.
 
 
 ## rust-analyzer.linkedProjects {#linkedProjects}
 
 Default: `[]`
 
-Disable project auto-discovery in favor of explicitly specified set
-of projects.
+Disable project auto-discovery in favor of explicitly specified set of projects.
 
-Elements must be paths pointing to `Cargo.toml`,
-`rust-project.json`, `.rs` files (which will be treated as standalone files) or JSON
-objects in `rust-project.json` format.
+Elements must be paths pointing to `Cargo.toml`, `rust-project.json`, `.rs` files (which
+will be treated as standalone files) or JSON objects in `rust-project.json` format.
 
 
 ## rust-analyzer.lru.capacity {#lru.capacity}
@@ -1197,21 +1241,22 @@ Number of syntax trees rust-analyzer keeps in memory. Defaults to 128.
 
 Default: `{}`
 
-Sets the LRU capacity of the specified queries.
+The LRU capacity of the specified queries.
 
 
 ## rust-analyzer.notifications.cargoTomlNotFound {#notifications.cargoTomlNotFound}
 
 Default: `true`
 
-Whether to show `can't find Cargo.toml` error message.
+Show `can't find Cargo.toml` error message.
 
 
 ## rust-analyzer.numThreads {#numThreads}
 
 Default: `null`
 
-How many worker threads in the main loop. The default `null` means to pick automatically.
+The number of worker threads in the main loop. The default `null` means to pick
+automatically.
 
 
 ## rust-analyzer.procMacro.attributes.enable {#procMacro.attributes.enable}
@@ -1278,7 +1323,7 @@ tests or binaries. For example, it may be `--release`.
 Default:
 ```json
 [
-  "--show-output"
+  "--nocapture"
 ]
 ```
 
@@ -1322,6 +1367,9 @@ not that of `cargo fmt`. The file contents will be passed on the
 standard input and the formatted result will be read from the
 standard output.
 
+Note: The option must be specified as an array of command line arguments, with
+the first argument being the name of the command to run.
+
 
 ## rust-analyzer.rustfmt.rangeFormatting.enable {#rustfmt.rangeFormatting.enable}
 
@@ -1332,6 +1380,17 @@ Enables the use of rustfmt's unstable range formatting command for the
 available on a nightly build.
 
 
+## rust-analyzer.semanticHighlighting.comments.enable {#semanticHighlighting.comments.enable}
+
+Default: `true`
+
+Use semantic tokens for comments.
+
+In some editors (e.g. vscode) semantic tokens override other highlighting grammars.
+By disabling semantic tokens for comments, other grammars can be used to highlight
+their contents.
+
+
 ## rust-analyzer.semanticHighlighting.doc.comment.inject.enable {#semanticHighlighting.doc.comment.inject.enable}
 
 Default: `true`
@@ -1346,7 +1405,10 @@ doc links.
 
 Default: `true`
 
-Whether the server is allowed to emit non-standard tokens and modifiers.
+Emit non-standard tokens and modifiers
+
+When enabled, rust-analyzer will emit tokens and modifiers that are not part of the
+standard set of semantic tokens.
 
 
 ## rust-analyzer.semanticHighlighting.operator.enable {#semanticHighlighting.operator.enable}
@@ -1427,11 +1489,15 @@ Show documentation.
 Default: `"=."`
 
 Specify the characters allowed to invoke special on typing triggers.
-- typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing expression
+
+- typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing
+  expression
 - typing `=` between two expressions adds `;` when in statement position
-- typing `=` to turn an assignment into an equality comparison removes `;` when in expression position
+- typing `=` to turn an assignment into an equality comparison removes `;` when in
+  expression position
 - typing `.` in a chain method call auto-indents
-- typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the expression
+- typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the
+  expression
 - typing `{` in a use item adds a closing `}` in the right place
 - typing `>` to complete a return type `->` will insert a whitespace after it
 - typing `<` in a path or type position inserts a closing `>` after the path or type.
@@ -1475,8 +1541,8 @@ Below is an example of a valid configuration:
 
 **Warning**: This format is provisional and subject to change.
 
-[`DiscoverWorkspaceConfig::command`] *must* return a JSON object
-corresponding to `DiscoverProjectData::Finished`:
+[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to
+`DiscoverProjectData::Finished`:
 
 ```norun
 #[derive(Debug, Clone, Deserialize, Serialize)]
@@ -1506,12 +1572,11 @@ As JSON, `DiscoverProjectData::Finished` is:
 }
 ```
 
-It is encouraged, but not required, to use the other variants on
-`DiscoverProjectData` to provide a more polished end-user experience.
+It is encouraged, but not required, to use the other variants on `DiscoverProjectData`
+to provide a more polished end-user experience.
 
-`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`,
-which will be substituted with the JSON-serialized form of the following
-enum:
+`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be
+substituted with the JSON-serialized form of the following enum:
 
 ```norun
 #[derive(PartialEq, Clone, Debug, Serialize)]
@@ -1538,11 +1603,10 @@ Similarly, the JSON representation of `DiscoverArgument::Buildfile` is:
 }
 ```
 
-`DiscoverArgument::Path` is used to find and generate a `rust-project.json`,
-and therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to
-to update an existing workspace. As a reference for implementors,
-buck2's `rust-project` will likely be useful:
-https://github.com/facebook/buck2/tree/main/integrations/rust-project.
+`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and
+therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an
+existing workspace. As a reference for implementors, buck2's `rust-project` will likely
+be useful: https://github.com/facebook/buck2/tree/main/integrations/rust-project.
 
 
 ## rust-analyzer.workspace.symbol.search.excludeImports {#workspace.symbol.search.excludeImports}
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/README.md b/src/tools/rust-analyzer/docs/book/src/contributing/README.md
index 57c7a9c5996d..ad2816b18ac1 100644
--- a/src/tools/rust-analyzer/docs/book/src/contributing/README.md
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/README.md
@@ -276,7 +276,7 @@ There are two sets of people with extra permissions:
   Feel free to request a review or assign any PR to a reviewer with the relevant expertise to bring the work to their attention.
   Don't feel pressured to review assigned PRs though.
   If you don't feel like reviewing for whatever reason, someone else will pick the review up (but please speak up if you don't feel like it)!
-* The [rust-lang](https://github.com/rust-lang) team [t-rust-analyzer-contributors]([https://github.com/orgs/rust-analyzer/teams/triage](https://github.com/rust-lang/team/blob/master/teams/rust-analyzer-contributors.toml)).
+* The [rust-lang](https://github.com/rust-lang) team [t-rust-analyzer-contributors](https://github.com/rust-lang/team/blob/master/teams/rust-analyzer-contributors.toml).
   This team has general triaging permissions allowing to label, close and re-open issues.
 
 ## Synchronizing subtree changes
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/guide.md b/src/tools/rust-analyzer/docs/book/src/contributing/guide.md
index 2a2a39af2641..d06840368ccb 100644
--- a/src/tools/rust-analyzer/docs/book/src/contributing/guide.md
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/guide.md
@@ -235,7 +235,7 @@ of type `V`. Queries come in two basic varieties:
   intelligently) when we can re-use these memoized values and when we have to
   recompute them.
 
-For further discussion, its important to understand one bit of "fairly
+For further discussion, it's important to understand one bit of "fairly
 intelligently". Suppose we have two functions, `f1` and `f2`, and one input,
 `z`. We call `f1(X)` which in turn calls `f2(Y)` which inspects `i(Z)`. `i(Z)`
 returns some value `V1`, `f2` uses that and returns `R1`, `f1` uses that and
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/lsp-extensions.md b/src/tools/rust-analyzer/docs/book/src/contributing/lsp-extensions.md
index 8c06f33a9f7b..91154b6f1c87 100644
--- a/src/tools/rust-analyzer/docs/book/src/contributing/lsp-extensions.md
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/lsp-extensions.md
@@ -455,7 +455,7 @@ interface TestItem {
     // A human readable name for this test
     label: string;
     // The kind of this test item. Based on the kind,
-	// an icon is chosen by the editor.
+    // an icon is chosen by the editor.
     kind: "package" | "module" | "test";
     // True if this test may have children not available eagerly
     canResolveChildren: boolean;
@@ -467,7 +467,7 @@ interface TestItem {
     // like debugging, this field is useful.
     // Note that this field includes some information about label and location as well, but
     // those exist just for keeping things in sync with other methods of running runnables
-    // (for example using one consistent name in the vscode's launch.json) so for any propose
+    // (for example using one consistent name in the vscode's launch.json) so for any purpose
     // other than running tests this field should not be used.
     runnable?: Runnable | undefined;
 };
@@ -475,11 +475,11 @@ interface TestItem {
 interface DiscoverTestResults {
     // The discovered tests.
     tests: TestItem[];
-    // For each test which its id is in this list, the response
+    // For each test whose id is in this list, the response
     // contains all tests that are children of this test, and
     // client should remove old tests not included in the response.
     scope: string[] | undefined;
-    // For each file which its uri is in this list, the response
+    // For each file whose uri is in this list, the response
     // contains all tests that are located in this file, and
     // client should remove old tests not included in the response.
     scopeFile: lc.TextDocumentIdentifier[] | undefined;
@@ -491,7 +491,7 @@ interface DiscoverTestResults {
 **Notification:** `DiscoverTestResults`
 
 This notification is sent from the server to the client when the
-server detect changes in the existing tests. The `DiscoverTestResults` is
+server detects changes in the existing tests. The `DiscoverTestResults` is
 the same as the one in `experimental/discoverTest` response.
 
 **Method:** `experimental/runTest`
@@ -527,7 +527,7 @@ after this.
 
 This notification is sent from the client to the server when the user is no longer
 interested in the test results. The server should clean up its resources and send
-a `experimental/endRunTest` when is done.
+a `experimental/endRunTest` when it is done.
 
 **Method:** `experimental/changeTestState`
 
@@ -648,8 +648,8 @@ interface ServerStatusParams {
 ```
 
 This notification is sent from server to client.
-The client can use it to display *persistent* status to the user (in modline).
-It is similar to the `showMessage`, but is intended for stares rather than point-in-time events.
+The client can use it to display *persistent* status to the user (in the mode line).
+It is similar to the `showMessage`, but is intended for status rather than point-in-time events.
 
 Note that this functionality is intended primarily to inform the end user about the state of the server.
 In particular, it's valid for the client to completely ignore this extension.
@@ -1070,13 +1070,13 @@ export interface RecursiveMemoryLayoutNode = {
     size: number;
     /// Alignment of the type in bytes
     alignment: number;
-    /// Offset of the type relative to its parent (or 0 if its the root)
+    /// Offset of the type relative to its parent (or 0 if it's the root)
     offset: number;
-    /// Index of the node's parent (or -1 if its the root)
+    /// Index of the node's parent (or -1 if it's the root)
     parent_idx: number;
     /// Index of the node's children (or -1 if it does not have children)
     children_start: number;
-    /// Number of child nodes (unspecified it does not have children)
+    /// Number of child nodes (unspecified if it does not have children)
     children_len: number;
 };
 
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/style.md b/src/tools/rust-analyzer/docs/book/src/contributing/style.md
index 5654e37753a5..fe09fb6c2fd5 100644
--- a/src/tools/rust-analyzer/docs/book/src/contributing/style.md
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/style.md
@@ -49,8 +49,8 @@ In this case, we'll probably ask you to split API changes into a separate PR.
 Changes of the third group should be pretty rare, so we don't specify any specific process for them.
 That said, adding an innocent-looking `pub use` is a very simple way to break encapsulation, keep an eye on it!
 
-Note: if you enjoyed this abstract hand-waving about boundaries, you might appreciate
-https://www.tedinski.com/2018/02/06/system-boundaries.html
+Note: if you enjoyed this abstract hand-waving about boundaries, you might appreciate [this post](https://www.tedinski.com/2018/02/06/system-boundaries.html).
+
 
 ## Crates.io Dependencies
 
@@ -101,7 +101,7 @@ Including a description and GIF suitable for the changelog means less work for t
 
 ## Clippy
 
-We use Clippy to improve the code, but if some lints annoy you, allow them in the [Cargo.toml](../../Cargo.toml) [workspace.lints.clippy] section.
+We use Clippy to improve the code, but if some lints annoy you, allow them in the [Cargo.toml](https://github.com/rust-lang/rust-analyzer/blob/master/Cargo.toml) [workspace.lints.clippy] section.
 
 # Code
 
@@ -231,7 +231,7 @@ fn is_string_literal(s: &str) -> bool {
 }
 ```
 
-In the "Not as good" version, the precondition that `1` is a valid char boundary is checked in `is_string_literal` and used in `foo`.
+In the "Bad" version, the precondition that `1` and `s.len() - 1` are valid string literal boundaries is checked in `is_string_literal` but used in `main`.
 In the "Good" version, the precondition check and usage are checked in the same block, and then encoded in the types.
 
 **Rationale:** non-local code properties degrade under change.
@@ -271,6 +271,8 @@ fn f() {
 }
 ```
 
+See also [this post](https://matklad.github.io/2023/11/15/push-ifs-up-and-fors-down.html)
+
 ## Assertions
 
 Assert liberally.
@@ -608,7 +610,7 @@ Avoid making a lot of code type parametric, *especially* on the boundaries betwe
 
 ```rust
 // GOOD
-fn frobnicate(f: impl FnMut()) {
+fn frobnicate(mut f: impl FnMut()) {
     frobnicate_impl(&mut f)
 }
 fn frobnicate_impl(f: &mut dyn FnMut()) {
@@ -616,7 +618,7 @@ fn frobnicate_impl(f: &mut dyn FnMut()) {
 }
 
 // BAD
-fn frobnicate(f: impl FnMut()) {
+fn frobnicate(mut f: impl FnMut()) {
     // lots of code
 }
 ```
@@ -975,7 +977,7 @@ Don't use the `ref` keyword.
 **Rationale:** consistency & simplicity.
 `ref` was required before [match ergonomics](https://github.com/rust-lang/rfcs/blob/master/text/2005-match-ergonomics.md).
 Today, it is redundant.
-Between `ref` and mach ergonomics, the latter is more ergonomic in most cases, and is simpler (does not require a keyword).
+Between `ref` and match ergonomics, the latter is more ergonomic in most cases, and is simpler (does not require a keyword).
 
 ## Empty Match Arms
 
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/syntax.md b/src/tools/rust-analyzer/docs/book/src/contributing/syntax.md
index 3dcd430cea5d..93eed7228d05 100644
--- a/src/tools/rust-analyzer/docs/book/src/contributing/syntax.md
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/syntax.md
@@ -192,7 +192,7 @@ Modeling this with immutable trees is possible, but annoying.
 
 A function green tree is not super-convenient to use.
 The biggest problem is accessing parents (there are no parent pointers!).
-But there are also "identify" issues.
+But there are also "identity" issues.
 Let's say you want to write a code which builds a list of expressions in a file: `fn collect_expressions(file: GreenNode) -> HashSet`.
 For the input like
 
diff --git a/src/tools/rust-analyzer/docs/book/src/contributing/testing.md b/src/tools/rust-analyzer/docs/book/src/contributing/testing.md
new file mode 100644
index 000000000000..ccee9b847b6e
--- /dev/null
+++ b/src/tools/rust-analyzer/docs/book/src/contributing/testing.md
@@ -0,0 +1,106 @@
+rust-analyzer's testing is based on *snapshot tests*: a test is a piece of input text, usually a Rust code, and some output text. There is then some testing helper that runs the feature on the input text and compares the result to the output text.
+
+rust-analyzer uses a combination of the crate [`expect-test`](https://docs.rs/expect-test) and a custom testing framework.
+
+This all may sound too abstract, so let's demonstrate with an example.
+
+Type inference tests are located at `crates/hir-ty/src/tests`. There are various test helpers you can use. One of the simplest is `check_no_mismatches()`: it is given a piece of Rust code (we'll talk more about Rust code in tests later) and asserts that there are no type mismatches in it, that is, one type was expected but another was found (for example, `let x: () = 1` is a type mismatch). Note that we determine type mismatches via rust-analyzer's own analysis, not via the compiler (this is what we are testing, after all), which means there are often missed mismatches and sometimes bogus ones as well.
+
+For example, the following test will fail:
+```rust
+#[test]
+fn this_will_fail() {
+    check_no_mismatches(
+        r#"
+fn main() {
+    let x: () = 1;
+}
+    "#,
+    );
+}
+```
+
+Sometimes we want to check more that there are no type mismatches. For that we use other helpers. For example, often we want to assert that the type of some expression is some specific type. For that we use the `check_types()` function. It takes a Rust code string with custom annotation, that are common in our test suite. The general scheme of annotation is:
+
+ - `$0` marks a position. What to do with it is determined by the testing helper. Commonly it denotes the cursor position in IDE tests (for example, hover).
+ - `$0...$0` marks a range, commonly a selection in IDE tests.
+ - `^...^`, commonly seen in a comment (`// ^^^^`), labels the line above. For example, the following will attach the label `hey` to the range of the variable name `cool`:
+
+    ```rust
+    let cool;
+     // ^^^^ hey
+    ```
+
+`check_types()` uses labels to assert type: when you attach a label to a range, `check_types()` assert that the type of this range will be what written in the label.
+
+It's all too abstract without an example:
+```rust
+#[test]
+fn my_test() {
+    check_types(
+        r#"
+fn main() {
+    let x = 1;
+     // ^ i32
+}
+    "#,
+    );
+}
+```
+Here, we assert that the type of the variable `x` is `i32`. Which is true, of course, so the test will pass.
+
+Oftentimes it is convenient to assert the types of all of the expressions at once, and that brings us to the last kind of test. It uses `expect-test` to match an output text:
+```rust
+#[test]
+fn my_test() {
+    check_infer(
+        r#"
+fn main() {
+    let x = 1;
+}
+    "#,
+        expect![[r#"
+            10..28 '{     ...= 1; }': ()
+            20..21 'x': i32
+            24..25 '1': i32
+        "#]],
+    );
+}
+```
+The text inside the `expect![[]]` is determined by the helper, `check_infer()` in this case. For `check_infer()`, each line is a range in the source code (the range is counted in bytes and the source is trimmed, indentation is stripped), next to it there is the text in that range, or some part of it with `...` if it's too long, and finally comes the type of that range.
+
+The important feature of `expect-test` is that it allows easy update of the expectation. Say you changed something in the code, maybe fixed a bug, and the output in `expect![[]]` needs to change. Or maybe you are writing it from scratch. Writing it by hand is very tedious and prone to mistakes. But `expect-trait` has a magic. You can set the environment variable `UPDATE_EXPECT=1`, then run the test, and it will update automatically! Some editors (e.g. VSCode) make it even more convenient: on them, on the top of every test that uses `expect-test`, next to the usual `Run | Debug` buttons, rust-analyzer also shows an `Update Expect` button. Clicking it will run that test in updating mode.
+
+## Rust code in the tests
+
+The first thing that you probably already noticed is that the Rust code in the tests is syntax highlighted! In fact, it even uses semantic highlighting. rust-analyzer highlights strings "as if" they contain Rust code if they are passed to a parameter marked `#[rust_analyzer::rust_fixture]`, and rust-analyzer test helpers do that (in fact, this was designed for them).
+
+The syntax highlighting is very important, not just because it's nice to the eye: it's very easy to make mistakes in test code, and debugging that can be very hard. Often the test will just fail, printing an `{unknown}` type, and you'll have no clue what's going wrong. The syntax is the clue; if something isn't highlighted correctly, that probably means there is an error (there is one exception to this, which we'll discuss later). You can even set the semantic highlighting tag `unresolved_reference` to e.g. red, so you will see such things clearly.
+
+Still, often you won't know what's going wrong. Why you can't fix the test, or worse, you expect it to fail but it doesn't. You can try the code on a real IDE to be sure it works. Later we'll give some tips to fix the test.
+
+### The fixture
+
+The Rust code in a test is not, a fact, a single Rust file. It has a mini-language that allows you to express multiple files, multiple crates, different configs, and more. All options are documented in `crates/test-utils/src/fixture.rs`, but here are some of the common ones:
+
+ - `//- minicore: flag1, flag2, ...`. This is by far the most common flag. Tests in rust-analyzer don't have access by default to any other type - not `Option`, not `Iterator`, not even `Sized`. This flag allows you to include parts of the `crates/test-utils/src/minicore.rs` file, which mimics `core`. All possible flags are listed at the top of `minicore` along with the flags they imply, then later you can see by `// region:flag` and `// endregion:flag` what code each flag enables.
+ - `// /path/to/file.rs crate:crate deps:dep_a,dep_b`. The first component is the filename of the code that follows (until the next file). It is required, but only if you supply this line. Other components in this line are optional. They include `crate:crate_name`, to start a new crate, or `deps:dep_a,dep_b`, to declare dependencies between crates. You can also declare modules as usual in Rust - just name your paths `/foo.rs` or `/foo/mod.rs`, declare `mod foo` and that's it!
+
+So the following snippet:
+```rust
+//- minicore: sized, fn
+// /lib.rs crate:foo
+pub mod bar;
+// /bar.rs
+pub struct Bar;
+// /main.rs crate:main deps:foo
+use foo::Bar;
+```
+Declares two crates `foo` and `main` where `main` depends on `foo`, with dependency in `Sized` and the `FnX` traits from `core`, and a module of `foo` called `bar`.
+
+And as promised, here are some tips to make your test work:
+
+ - If you use some type/trait, you must *always* include it in `minicore`. Note - not all types from core/std are available there, you can add new (under flags) if you need. And import them if they are not in the prelude.
+ - If you use unsized types (`dyn Trait`/slices), you may want to include some or all of the following `minicore` flags: `sized`, `unsize`, `coerce_unsized`, `dispatch_from_dyn`.
+ - If you use closures, consider including the `fn` minicore flag. Async closures need the `async_fn` flag.
+ - `sized` is commonly needed, consider adding it if you're stuck.
diff --git a/src/tools/rust-analyzer/docs/book/src/faq.md b/src/tools/rust-analyzer/docs/book/src/faq.md
index c87203309011..8c143ab94935 100644
--- a/src/tools/rust-analyzer/docs/book/src/faq.md
+++ b/src/tools/rust-analyzer/docs/book/src/faq.md
@@ -5,3 +5,12 @@
 rust-analyzer fails to resolve `None`, and thinks you are binding to a variable
 named `None`. That's usually a sign of a corrupted sysroot. Try removing and re-installing
 it: `rustup component remove rust-src` then `rustup component install rust-src`.
+
+### Rust Analyzer and Cargo compete over the build lock
+
+Rust Analyzer invokes Cargo in the background, and it can thus block manually executed
+`cargo` commands from making progress (or vice-versa). In some cases, this can also cause
+unnecessary recompilations caused by cache thrashing. To avoid this, you can configure
+Rust Analyzer to use a [different target directory](./configuration.md#cargo.targetDir).
+This will allow both the IDE and Cargo to make progress independently, at the cost of
+increased disk space usage caused by the duplicated artifact directories.
diff --git a/src/tools/rust-analyzer/editors/code/icon.png b/src/tools/rust-analyzer/editors/code/icon.png
index 072090c6f4a1..29a967903901 100644
Binary files a/src/tools/rust-analyzer/editors/code/icon.png and b/src/tools/rust-analyzer/editors/code/icon.png differ
diff --git a/src/tools/rust-analyzer/editors/code/package-lock.json b/src/tools/rust-analyzer/editors/code/package-lock.json
index 534c24be52e8..6dd448522379 100644
--- a/src/tools/rust-analyzer/editors/code/package-lock.json
+++ b/src/tools/rust-analyzer/editors/code/package-lock.json
@@ -21,12 +21,13 @@
                 "@stylistic/eslint-plugin": "^4.1.0",
                 "@stylistic/eslint-plugin-js": "^4.1.0",
                 "@tsconfig/strictest": "^2.0.5",
+                "@types/lodash": "^4.17.20",
                 "@types/node": "~22.13.4",
                 "@types/vscode": "~1.93.0",
                 "@typescript-eslint/eslint-plugin": "^8.25.0",
                 "@typescript-eslint/parser": "^8.25.0",
                 "@vscode/test-electron": "^2.4.1",
-                "@vscode/vsce": "^3.2.2",
+                "@vscode/vsce": "^3.6.0",
                 "esbuild": "^0.25.0",
                 "eslint": "^9.21.0",
                 "eslint-config-prettier": "^10.0.2",
@@ -41,6 +42,23 @@
                 "vscode": "^1.93.0"
             }
         },
+        "node_modules/@azu/format-text": {
+            "version": "1.0.2",
+            "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.2.tgz",
+            "integrity": "sha512-Swi4N7Edy1Eqq82GxgEECXSSLyn6GOb5htRFPzBDdUkECGXtlf12ynO5oJSpWKPwCaUssOu7NfhDcCWpIC6Ywg==",
+            "dev": true,
+            "license": "BSD-3-Clause"
+        },
+        "node_modules/@azu/style-format": {
+            "version": "1.0.1",
+            "resolved": "https://registry.npmjs.org/@azu/style-format/-/style-format-1.0.1.tgz",
+            "integrity": "sha512-AHcTojlNBdD/3/KxIKlg8sxIWHfOtQszLvOpagLTO+bjC3u7SAszu1lf//u7JJC50aUSH+BVWDD/KvaA6Gfn5g==",
+            "dev": true,
+            "license": "WTFPL",
+            "dependencies": {
+                "@azu/format-text": "^1.0.1"
+            }
+        },
         "node_modules/@azure/abort-controller": {
             "version": "2.1.2",
             "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz",
@@ -212,6 +230,31 @@
                 "node": ">=16"
             }
         },
+        "node_modules/@babel/code-frame": {
+            "version": "7.27.1",
+            "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz",
+            "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@babel/helper-validator-identifier": "^7.27.1",
+                "js-tokens": "^4.0.0",
+                "picocolors": "^1.1.1"
+            },
+            "engines": {
+                "node": ">=6.9.0"
+            }
+        },
+        "node_modules/@babel/helper-validator-identifier": {
+            "version": "7.27.1",
+            "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz",
+            "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=6.9.0"
+            }
+        },
         "node_modules/@esbuild/aix-ppc64": {
             "version": "0.25.0",
             "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz",
@@ -947,6 +990,217 @@
                 "node": ">= 8"
             }
         },
+        "node_modules/@secretlint/config-creator": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/config-creator/-/config-creator-10.2.2.tgz",
+            "integrity": "sha512-BynOBe7Hn3LJjb3CqCHZjeNB09s/vgf0baBaHVw67w7gHF0d25c3ZsZ5+vv8TgwSchRdUCRrbbcq5i2B1fJ2QQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/types": "^10.2.2"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/config-loader": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/config-loader/-/config-loader-10.2.2.tgz",
+            "integrity": "sha512-ndjjQNgLg4DIcMJp4iaRD6xb9ijWQZVbd9694Ol2IszBIbGPPkwZHzJYKICbTBmh6AH/pLr0CiCaWdGJU7RbpQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/profiler": "^10.2.2",
+                "@secretlint/resolver": "^10.2.2",
+                "@secretlint/types": "^10.2.2",
+                "ajv": "^8.17.1",
+                "debug": "^4.4.1",
+                "rc-config-loader": "^4.1.3"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/config-loader/node_modules/ajv": {
+            "version": "8.17.1",
+            "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
+            "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "fast-deep-equal": "^3.1.3",
+                "fast-uri": "^3.0.1",
+                "json-schema-traverse": "^1.0.0",
+                "require-from-string": "^2.0.2"
+            },
+            "funding": {
+                "type": "github",
+                "url": "https://github.com/sponsors/epoberezkin"
+            }
+        },
+        "node_modules/@secretlint/config-loader/node_modules/json-schema-traverse": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+            "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@secretlint/core": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/core/-/core-10.2.2.tgz",
+            "integrity": "sha512-6rdwBwLP9+TO3rRjMVW1tX+lQeo5gBbxl1I5F8nh8bgGtKwdlCMhMKsBWzWg1ostxx/tIG7OjZI0/BxsP8bUgw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/profiler": "^10.2.2",
+                "@secretlint/types": "^10.2.2",
+                "debug": "^4.4.1",
+                "structured-source": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/formatter": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/formatter/-/formatter-10.2.2.tgz",
+            "integrity": "sha512-10f/eKV+8YdGKNQmoDUD1QnYL7TzhI2kzyx95vsJKbEa8akzLAR5ZrWIZ3LbcMmBLzxlSQMMccRmi05yDQ5YDA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/resolver": "^10.2.2",
+                "@secretlint/types": "^10.2.2",
+                "@textlint/linter-formatter": "^15.2.0",
+                "@textlint/module-interop": "^15.2.0",
+                "@textlint/types": "^15.2.0",
+                "chalk": "^5.4.1",
+                "debug": "^4.4.1",
+                "pluralize": "^8.0.0",
+                "strip-ansi": "^7.1.0",
+                "table": "^6.9.0",
+                "terminal-link": "^4.0.0"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/formatter/node_modules/chalk": {
+            "version": "5.5.0",
+            "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.5.0.tgz",
+            "integrity": "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": "^12.17.0 || ^14.13 || >=16.0.0"
+            },
+            "funding": {
+                "url": "https://github.com/chalk/chalk?sponsor=1"
+            }
+        },
+        "node_modules/@secretlint/node": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/node/-/node-10.2.2.tgz",
+            "integrity": "sha512-eZGJQgcg/3WRBwX1bRnss7RmHHK/YlP/l7zOQsrjexYt6l+JJa5YhUmHbuGXS94yW0++3YkEJp0kQGYhiw1DMQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/config-loader": "^10.2.2",
+                "@secretlint/core": "^10.2.2",
+                "@secretlint/formatter": "^10.2.2",
+                "@secretlint/profiler": "^10.2.2",
+                "@secretlint/source-creator": "^10.2.2",
+                "@secretlint/types": "^10.2.2",
+                "debug": "^4.4.1",
+                "p-map": "^7.0.3"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/profiler": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/profiler/-/profiler-10.2.2.tgz",
+            "integrity": "sha512-qm9rWfkh/o8OvzMIfY8a5bCmgIniSpltbVlUVl983zDG1bUuQNd1/5lUEeWx5o/WJ99bXxS7yNI4/KIXfHexig==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@secretlint/resolver": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/resolver/-/resolver-10.2.2.tgz",
+            "integrity": "sha512-3md0cp12e+Ae5V+crPQYGd6aaO7ahw95s28OlULGyclyyUtf861UoRGS2prnUrKh7MZb23kdDOyGCYb9br5e4w==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@secretlint/secretlint-formatter-sarif": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/secretlint-formatter-sarif/-/secretlint-formatter-sarif-10.2.2.tgz",
+            "integrity": "sha512-ojiF9TGRKJJw308DnYBucHxkpNovDNu1XvPh7IfUp0A12gzTtxuWDqdpuVezL7/IP8Ua7mp5/VkDMN9OLp1doQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "node-sarif-builder": "^3.2.0"
+            }
+        },
+        "node_modules/@secretlint/secretlint-rule-no-dotenv": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-no-dotenv/-/secretlint-rule-no-dotenv-10.2.2.tgz",
+            "integrity": "sha512-KJRbIShA9DVc5Va3yArtJ6QDzGjg3PRa1uYp9As4RsyKtKSSZjI64jVca57FZ8gbuk4em0/0Jq+uy6485wxIdg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/types": "^10.2.2"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/secretlint-rule-preset-recommend": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/secretlint-rule-preset-recommend/-/secretlint-rule-preset-recommend-10.2.2.tgz",
+            "integrity": "sha512-K3jPqjva8bQndDKJqctnGfwuAxU2n9XNCPtbXVI5JvC7FnQiNg/yWlQPbMUlBXtBoBGFYp08A94m6fvtc9v+zA==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/source-creator": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/source-creator/-/source-creator-10.2.2.tgz",
+            "integrity": "sha512-h6I87xJfwfUTgQ7irWq7UTdq/Bm1RuQ/fYhA3dtTIAop5BwSFmZyrchph4WcoEvbN460BWKmk4RYSvPElIIvxw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/types": "^10.2.2",
+                "istextorbinary": "^9.5.0"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@secretlint/types": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/@secretlint/types/-/types-10.2.2.tgz",
+            "integrity": "sha512-Nqc90v4lWCXyakD6xNyNACBJNJ0tNCwj2WNk/7ivyacYHxiITVgmLUFXTBOeCdy79iz6HtN9Y31uw/jbLrdOAg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
+        "node_modules/@sindresorhus/merge-streams": {
+            "version": "2.3.0",
+            "resolved": "https://registry.npmjs.org/@sindresorhus/merge-streams/-/merge-streams-2.3.0.tgz",
+            "integrity": "sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/@stylistic/eslint-plugin": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-4.1.0.tgz",
@@ -984,6 +1238,136 @@
                 "eslint": ">=9.0.0"
             }
         },
+        "node_modules/@textlint/ast-node-types": {
+            "version": "15.2.1",
+            "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-15.2.1.tgz",
+            "integrity": "sha512-20fEcLPsXg81yWpApv4FQxrZmlFF/Ta7/kz1HGIL+pJo5cSTmkc+eCki3GpOPZIoZk0tbJU8hrlwUb91F+3SNQ==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@textlint/linter-formatter": {
+            "version": "15.2.1",
+            "resolved": "https://registry.npmjs.org/@textlint/linter-formatter/-/linter-formatter-15.2.1.tgz",
+            "integrity": "sha512-oollG/BHa07+mMt372amxHohteASC+Zxgollc1sZgiyxo4S6EuureV3a4QIQB0NecA+Ak3d0cl0WI/8nou38jw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@azu/format-text": "^1.0.2",
+                "@azu/style-format": "^1.0.1",
+                "@textlint/module-interop": "15.2.1",
+                "@textlint/resolver": "15.2.1",
+                "@textlint/types": "15.2.1",
+                "chalk": "^4.1.2",
+                "debug": "^4.4.1",
+                "js-yaml": "^3.14.1",
+                "lodash": "^4.17.21",
+                "pluralize": "^2.0.0",
+                "string-width": "^4.2.3",
+                "strip-ansi": "^6.0.1",
+                "table": "^6.9.0",
+                "text-table": "^0.2.0"
+            }
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/ansi-regex": {
+            "version": "5.0.1",
+            "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+            "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/argparse": {
+            "version": "1.0.10",
+            "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+            "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "sprintf-js": "~1.0.2"
+            }
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/emoji-regex": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+            "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/js-yaml": {
+            "version": "3.14.1",
+            "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+            "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "argparse": "^1.0.7",
+                "esprima": "^4.0.0"
+            },
+            "bin": {
+                "js-yaml": "bin/js-yaml.js"
+            }
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/pluralize": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-2.0.0.tgz",
+            "integrity": "sha512-TqNZzQCD4S42De9IfnnBvILN7HAW7riLqsCyp8lgjXeysyPlX5HhqKAcJHHHb9XskE4/a+7VGC9zzx8Ls0jOAw==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/string-width": {
+            "version": "4.2.3",
+            "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+            "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "emoji-regex": "^8.0.0",
+                "is-fullwidth-code-point": "^3.0.0",
+                "strip-ansi": "^6.0.1"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/@textlint/linter-formatter/node_modules/strip-ansi": {
+            "version": "6.0.1",
+            "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+            "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "ansi-regex": "^5.0.1"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/@textlint/module-interop": {
+            "version": "15.2.1",
+            "resolved": "https://registry.npmjs.org/@textlint/module-interop/-/module-interop-15.2.1.tgz",
+            "integrity": "sha512-b/C/ZNrm05n1ypymDknIcpkBle30V2ZgE3JVqQlA9PnQV46Ky510qrZk6s9yfKgA3m1YRnAw04m8xdVtqjq1qg==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@textlint/resolver": {
+            "version": "15.2.1",
+            "resolved": "https://registry.npmjs.org/@textlint/resolver/-/resolver-15.2.1.tgz",
+            "integrity": "sha512-FY3aK4tElEcOJVUsaMj4Zro4jCtKEEwUMIkDL0tcn6ljNcgOF7Em+KskRRk/xowFWayqDtdz5T3u7w/6fjjuJQ==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@textlint/types": {
+            "version": "15.2.1",
+            "resolved": "https://registry.npmjs.org/@textlint/types/-/types-15.2.1.tgz",
+            "integrity": "sha512-zyqNhSatK1cwxDUgosEEN43hFh3WCty9Zm2Vm3ogU566IYegifwqN54ey/CiRy/DiO4vMcFHykuQnh2Zwp6LLw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@textlint/ast-node-types": "15.2.1"
+            }
+        },
         "node_modules/@tsconfig/strictest": {
             "version": "2.0.5",
             "resolved": "https://registry.npmjs.org/@tsconfig/strictest/-/strictest-2.0.5.tgz",
@@ -1005,6 +1389,13 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/@types/lodash": {
+            "version": "4.17.20",
+            "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.20.tgz",
+            "integrity": "sha512-H3MHACvFUEiujabxhaI/ImO6gUrd8oOurg7LQtS7mbwIXA/cUqWrvBsaeJ23aZEPk1TAYkurjfMbSELfoCXlGA==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@types/node": {
             "version": "22.13.5",
             "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.5.tgz",
@@ -1015,6 +1406,20 @@
                 "undici-types": "~6.20.0"
             }
         },
+        "node_modules/@types/normalize-package-data": {
+            "version": "2.4.4",
+            "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz",
+            "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/@types/sarif": {
+            "version": "2.1.7",
+            "resolved": "https://registry.npmjs.org/@types/sarif/-/sarif-2.1.7.tgz",
+            "integrity": "sha512-kRz0VEkJqWLf1LLVN4pT1cg1Z9wAuvI6L97V3m2f5B76Tg8d413ddvLBPTEHAZJlnn4XSvu0FkZtViCQGVyrXQ==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/@types/vscode": {
             "version": "1.93.0",
             "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.93.0.tgz",
@@ -1220,16 +1625,20 @@
             }
         },
         "node_modules/@vscode/vsce": {
-            "version": "3.2.2",
-            "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.2.2.tgz",
-            "integrity": "sha512-4TqdUq/yKlQTHcQMk/DamR632bq/+IJDomSbexOMee/UAYWqYm0XHWA6scGslsCpzY+sCWEhhl0nqdOB0XW1kw==",
+            "version": "3.6.0",
+            "resolved": "https://registry.npmjs.org/@vscode/vsce/-/vsce-3.6.0.tgz",
+            "integrity": "sha512-u2ZoMfymRNJb14aHNawnXJtXHLXDVKc1oKZaH4VELKT/9iWKRVgtQOdwxCgtwSxJoqYvuK4hGlBWQJ05wxADhg==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
                 "@azure/identity": "^4.1.0",
+                "@secretlint/node": "^10.1.1",
+                "@secretlint/secretlint-formatter-sarif": "^10.1.1",
+                "@secretlint/secretlint-rule-no-dotenv": "^10.1.1",
+                "@secretlint/secretlint-rule-preset-recommend": "^10.1.1",
                 "@vscode/vsce-sign": "^2.0.0",
                 "azure-devops-node-api": "^12.5.0",
-                "chalk": "^2.4.2",
+                "chalk": "^4.1.2",
                 "cheerio": "^1.0.0-rc.9",
                 "cockatiel": "^3.1.2",
                 "commander": "^12.1.0",
@@ -1243,6 +1652,7 @@
                 "minimatch": "^3.0.3",
                 "parse-semver": "^1.1.1",
                 "read": "^1.0.7",
+                "secretlint": "^10.1.1",
                 "semver": "^7.5.2",
                 "tmp": "^0.2.3",
                 "typed-rest-client": "^1.8.4",
@@ -1486,6 +1896,22 @@
             "integrity": "sha512-PMqBCBvrOVDRqLGooQb+z+t1Q0PiPyurUQeZRR5uHBOVZcW8B04KMmnT12USnhpNX2wCPagWzLVppQMUG3u0Dw==",
             "license": "MIT"
         },
+        "node_modules/ansi-escapes": {
+            "version": "7.0.0",
+            "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.0.0.tgz",
+            "integrity": "sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "environment": "^1.0.0"
+            },
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/ansi-regex": {
             "version": "6.1.0",
             "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz",
@@ -1500,16 +1926,18 @@
             }
         },
         "node_modules/ansi-styles": {
-            "version": "3.2.1",
-            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-            "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-            "dev": true,
+            "version": "4.3.0",
+            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+            "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
             "license": "MIT",
             "dependencies": {
-                "color-convert": "^1.9.0"
+                "color-convert": "^2.0.1"
             },
             "engines": {
-                "node": ">=4"
+                "node": ">=8"
+            },
+            "funding": {
+                "url": "https://github.com/chalk/ansi-styles?sponsor=1"
             }
         },
         "node_modules/argparse": {
@@ -1519,6 +1947,16 @@
             "dev": true,
             "license": "Python-2.0"
         },
+        "node_modules/astral-regex": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+            "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=8"
+            }
+        },
         "node_modules/asynckit": {
             "version": "0.4.0",
             "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -1564,6 +2002,22 @@
             ],
             "license": "MIT"
         },
+        "node_modules/binaryextensions": {
+            "version": "6.11.0",
+            "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-6.11.0.tgz",
+            "integrity": "sha512-sXnYK/Ij80TO3lcqZVV2YgfKN5QjUWIRk/XSm2J/4bd/lPko3lvk0O4ZppH6m+6hB2/GTu+ptNwVFe1xh+QLQw==",
+            "dev": true,
+            "license": "Artistic-2.0",
+            "dependencies": {
+                "editions": "^6.21.0"
+            },
+            "engines": {
+                "node": ">=4"
+            },
+            "funding": {
+                "url": "https://bevry.me/fund"
+            }
+        },
         "node_modules/bl": {
             "version": "5.1.0",
             "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz",
@@ -1598,6 +2052,13 @@
             "dev": true,
             "license": "ISC"
         },
+        "node_modules/boundary": {
+            "version": "2.0.0",
+            "resolved": "https://registry.npmjs.org/boundary/-/boundary-2.0.0.tgz",
+            "integrity": "sha512-rJKn5ooC9u8q13IMCrW0RSp31pxBCHE3y9V/tp3TdWSLf8Em3p6Di4NBpfzbJge9YjjFEsD0RtFEjtvHL5VyEA==",
+            "dev": true,
+            "license": "BSD-2-Clause"
+        },
         "node_modules/brace-expansion": {
             "version": "2.0.2",
             "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz",
@@ -1720,18 +2181,20 @@
             }
         },
         "node_modules/chalk": {
-            "version": "2.4.2",
-            "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
-            "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==",
+            "version": "4.1.2",
+            "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+            "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
-                "ansi-styles": "^3.2.1",
-                "escape-string-regexp": "^1.0.5",
-                "supports-color": "^5.3.0"
+                "ansi-styles": "^4.1.0",
+                "supports-color": "^7.1.0"
             },
             "engines": {
-                "node": ">=4"
+                "node": ">=10"
+            },
+            "funding": {
+                "url": "https://github.com/chalk/chalk?sponsor=1"
             }
         },
         "node_modules/cheerio": {
@@ -1845,39 +2308,6 @@
                 "node": ">=8"
             }
         },
-        "node_modules/cliui/node_modules/ansi-styles": {
-            "version": "4.3.0",
-            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-            "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-            "license": "MIT",
-            "dependencies": {
-                "color-convert": "^2.0.1"
-            },
-            "engines": {
-                "node": ">=8"
-            },
-            "funding": {
-                "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-            }
-        },
-        "node_modules/cliui/node_modules/color-convert": {
-            "version": "2.0.1",
-            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-            "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-            "license": "MIT",
-            "dependencies": {
-                "color-name": "~1.1.4"
-            },
-            "engines": {
-                "node": ">=7.0.0"
-            }
-        },
-        "node_modules/cliui/node_modules/color-name": {
-            "version": "1.1.4",
-            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-            "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-            "license": "MIT"
-        },
         "node_modules/cliui/node_modules/emoji-regex": {
             "version": "8.0.0",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -1938,20 +2368,21 @@
             }
         },
         "node_modules/color-convert": {
-            "version": "1.9.3",
-            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz",
-            "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==",
-            "dev": true,
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+            "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
             "license": "MIT",
             "dependencies": {
-                "color-name": "1.1.3"
+                "color-name": "~1.1.4"
+            },
+            "engines": {
+                "node": ">=7.0.0"
             }
         },
         "node_modules/color-name": {
-            "version": "1.1.3",
-            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-            "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
-            "dev": true,
+            "version": "1.1.4",
+            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+            "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
             "license": "MIT"
         },
         "node_modules/combined-stream": {
@@ -2469,9 +2900,9 @@
             }
         },
         "node_modules/debug": {
-            "version": "4.4.0",
-            "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
-            "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+            "version": "4.4.1",
+            "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
+            "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
@@ -2685,6 +3116,23 @@
                 "safe-buffer": "^5.0.1"
             }
         },
+        "node_modules/editions": {
+            "version": "6.22.0",
+            "resolved": "https://registry.npmjs.org/editions/-/editions-6.22.0.tgz",
+            "integrity": "sha512-UgGlf8IW75je7HZjNDpJdCv4cGJWIi6yumFdZ0R7A8/CIhQiWUjyGLCxdHpd8bmyD1gnkfUNK0oeOXqUS2cpfQ==",
+            "dev": true,
+            "license": "Artistic-2.0",
+            "dependencies": {
+                "version-range": "^4.15.0"
+            },
+            "engines": {
+                "ecmascript": ">= es5",
+                "node": ">=4"
+            },
+            "funding": {
+                "url": "https://bevry.me/fund"
+            }
+        },
         "node_modules/emoji-regex": {
             "version": "9.2.2",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
@@ -2730,6 +3178,19 @@
                 "url": "https://github.com/fb55/entities?sponsor=1"
             }
         },
+        "node_modules/environment": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz",
+            "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/es-define-property": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -2829,16 +3290,6 @@
                 "node": ">=6"
             }
         },
-        "node_modules/escape-string-regexp": {
-            "version": "1.0.5",
-            "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-            "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
-            "dev": true,
-            "license": "MIT",
-            "engines": {
-                "node": ">=0.8.0"
-            }
-        },
         "node_modules/eslint": {
             "version": "9.21.0",
             "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.21.0.tgz",
@@ -2964,22 +3415,6 @@
                 "url": "https://opencollective.com/eslint"
             }
         },
-        "node_modules/eslint/node_modules/ansi-styles": {
-            "version": "4.3.0",
-            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-            "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "color-convert": "^2.0.1"
-            },
-            "engines": {
-                "node": ">=8"
-            },
-            "funding": {
-                "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-            }
-        },
         "node_modules/eslint/node_modules/brace-expansion": {
             "version": "1.1.12",
             "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
@@ -2991,43 +3426,6 @@
                 "concat-map": "0.0.1"
             }
         },
-        "node_modules/eslint/node_modules/chalk": {
-            "version": "4.1.2",
-            "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-            "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "ansi-styles": "^4.1.0",
-                "supports-color": "^7.1.0"
-            },
-            "engines": {
-                "node": ">=10"
-            },
-            "funding": {
-                "url": "https://github.com/chalk/chalk?sponsor=1"
-            }
-        },
-        "node_modules/eslint/node_modules/color-convert": {
-            "version": "2.0.1",
-            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-            "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "color-name": "~1.1.4"
-            },
-            "engines": {
-                "node": ">=7.0.0"
-            }
-        },
-        "node_modules/eslint/node_modules/color-name": {
-            "version": "1.1.4",
-            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-            "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-            "dev": true,
-            "license": "MIT"
-        },
         "node_modules/eslint/node_modules/escape-string-regexp": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@@ -3041,16 +3439,6 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
-        "node_modules/eslint/node_modules/has-flag": {
-            "version": "4.0.0",
-            "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-            "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-            "dev": true,
-            "license": "MIT",
-            "engines": {
-                "node": ">=8"
-            }
-        },
         "node_modules/eslint/node_modules/minimatch": {
             "version": "3.1.2",
             "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -3064,19 +3452,6 @@
                 "node": "*"
             }
         },
-        "node_modules/eslint/node_modules/supports-color": {
-            "version": "7.2.0",
-            "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-            "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "has-flag": "^4.0.0"
-            },
-            "engines": {
-                "node": ">=8"
-            }
-        },
         "node_modules/espree": {
             "version": "10.3.0",
             "resolved": "https://registry.npmjs.org/espree/-/espree-10.3.0.tgz",
@@ -3095,6 +3470,20 @@
                 "url": "https://opencollective.com/eslint"
             }
         },
+        "node_modules/esprima": {
+            "version": "4.0.1",
+            "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+            "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+            "dev": true,
+            "license": "BSD-2-Clause",
+            "bin": {
+                "esparse": "bin/esparse.js",
+                "esvalidate": "bin/esvalidate.js"
+            },
+            "engines": {
+                "node": ">=4"
+            }
+        },
         "node_modules/esquery": {
             "version": "1.6.0",
             "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.6.0.tgz",
@@ -3213,6 +3602,23 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/fast-uri": {
+            "version": "3.0.6",
+            "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.6.tgz",
+            "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==",
+            "dev": true,
+            "funding": [
+                {
+                    "type": "github",
+                    "url": "https://github.com/sponsors/fastify"
+                },
+                {
+                    "type": "opencollective",
+                    "url": "https://opencollective.com/fastify"
+                }
+            ],
+            "license": "BSD-3-Clause"
+        },
         "node_modules/fastq": {
             "version": "1.19.0",
             "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz",
@@ -3360,6 +3766,21 @@
             "license": "MIT",
             "optional": true
         },
+        "node_modules/fs-extra": {
+            "version": "11.3.1",
+            "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.1.tgz",
+            "integrity": "sha512-eXvGGwZ5CL17ZSwHWd3bbgk7UUpF6IFHtP57NYYakPvHOs8GDgDe5KJI36jIJzDkJ6eJjuzRA8eBQb6SkKue0g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "graceful-fs": "^4.2.0",
+                "jsonfile": "^6.0.1",
+                "universalify": "^2.0.0"
+            },
+            "engines": {
+                "node": ">=14.14"
+            }
+        },
         "node_modules/function-bind": {
             "version": "1.1.2",
             "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -3492,6 +3913,37 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/globby": {
+            "version": "14.1.0",
+            "resolved": "https://registry.npmjs.org/globby/-/globby-14.1.0.tgz",
+            "integrity": "sha512-0Ia46fDOaT7k4og1PDW4YbodWWr3scS2vAr2lTbsplOt2WkKp0vQbkI9wKis/T5LV/dqPjO3bpS/z6GTJB82LA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@sindresorhus/merge-streams": "^2.1.0",
+                "fast-glob": "^3.3.3",
+                "ignore": "^7.0.3",
+                "path-type": "^6.0.0",
+                "slash": "^5.1.0",
+                "unicorn-magic": "^0.3.0"
+            },
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/globby/node_modules/ignore": {
+            "version": "7.0.5",
+            "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz",
+            "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">= 4"
+            }
+        },
         "node_modules/gopd": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -3505,6 +3957,13 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/graceful-fs": {
+            "version": "4.2.11",
+            "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+            "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+            "dev": true,
+            "license": "ISC"
+        },
         "node_modules/graphemer": {
             "version": "1.4.0",
             "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz",
@@ -3513,13 +3972,13 @@
             "license": "MIT"
         },
         "node_modules/has-flag": {
-            "version": "3.0.0",
-            "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-            "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+            "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
             "dev": true,
             "license": "MIT",
             "engines": {
-                "node": ">=4"
+                "node": ">=8"
             }
         },
         "node_modules/has-symbols": {
@@ -3702,6 +4161,19 @@
                 "node": ">=0.8.19"
             }
         },
+        "node_modules/index-to-position": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/index-to-position/-/index-to-position-1.1.0.tgz",
+            "integrity": "sha512-XPdx9Dq4t9Qk1mTMbWONJqU7boCoumEH7fRET37HX5+khDUl3J2W6PdALxhILYlIYx2amlwYcRPp28p0tSiojg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/inherits": {
             "version": "2.0.4",
             "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
@@ -3872,6 +4344,24 @@
             "dev": true,
             "license": "ISC"
         },
+        "node_modules/istextorbinary": {
+            "version": "9.5.0",
+            "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-9.5.0.tgz",
+            "integrity": "sha512-5mbUj3SiZXCuRf9fT3ibzbSSEWiy63gFfksmGfdOzujPjW3k+z8WvIBxcJHBoQNlaZaiyB25deviif2+osLmLw==",
+            "dev": true,
+            "license": "Artistic-2.0",
+            "dependencies": {
+                "binaryextensions": "^6.11.0",
+                "editions": "^6.21.0",
+                "textextensions": "^6.11.0"
+            },
+            "engines": {
+                "node": ">=4"
+            },
+            "funding": {
+                "url": "https://bevry.me/fund"
+            }
+        },
         "node_modules/jackspeak": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.0.tgz",
@@ -3897,6 +4387,13 @@
                 "jiti": "lib/jiti-cli.mjs"
             }
         },
+        "node_modules/js-tokens": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
+            "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/js-yaml": {
             "version": "4.1.0",
             "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
@@ -3931,6 +4428,19 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/json5": {
+            "version": "2.2.3",
+            "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+            "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
+            "dev": true,
+            "license": "MIT",
+            "bin": {
+                "json5": "lib/cli.js"
+            },
+            "engines": {
+                "node": ">=6"
+            }
+        },
         "node_modules/jsonc-parser": {
             "version": "3.3.1",
             "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz",
@@ -3938,6 +4448,19 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/jsonfile": {
+            "version": "6.2.0",
+            "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz",
+            "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "universalify": "^2.0.0"
+            },
+            "optionalDependencies": {
+                "graceful-fs": "^4.1.6"
+            }
+        },
         "node_modules/jsonwebtoken": {
             "version": "9.0.2",
             "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz",
@@ -4103,6 +4626,13 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/lodash": {
+            "version": "4.17.21",
+            "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+            "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/lodash.includes": {
             "version": "4.3.0",
             "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -4159,6 +4689,13 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/lodash.truncate": {
+            "version": "4.4.2",
+            "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz",
+            "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==",
+            "dev": true,
+            "license": "MIT"
+        },
         "node_modules/log-symbols": {
             "version": "5.1.0",
             "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz",
@@ -4430,6 +4967,55 @@
             "license": "MIT",
             "optional": true
         },
+        "node_modules/node-sarif-builder": {
+            "version": "3.2.0",
+            "resolved": "https://registry.npmjs.org/node-sarif-builder/-/node-sarif-builder-3.2.0.tgz",
+            "integrity": "sha512-kVIOdynrF2CRodHZeP/97Rh1syTUHBNiw17hUCIVhlhEsWlfJm19MuO56s4MdKbr22xWx6mzMnNAgXzVlIYM9Q==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/sarif": "^2.1.7",
+                "fs-extra": "^11.1.1"
+            },
+            "engines": {
+                "node": ">=18"
+            }
+        },
+        "node_modules/normalize-package-data": {
+            "version": "6.0.2",
+            "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz",
+            "integrity": "sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==",
+            "dev": true,
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "hosted-git-info": "^7.0.0",
+                "semver": "^7.3.5",
+                "validate-npm-package-license": "^3.0.4"
+            },
+            "engines": {
+                "node": "^16.14.0 || >=18.0.0"
+            }
+        },
+        "node_modules/normalize-package-data/node_modules/hosted-git-info": {
+            "version": "7.0.2",
+            "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz",
+            "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==",
+            "dev": true,
+            "license": "ISC",
+            "dependencies": {
+                "lru-cache": "^10.0.1"
+            },
+            "engines": {
+                "node": "^16.14.0 || >=18.0.0"
+            }
+        },
+        "node_modules/normalize-package-data/node_modules/lru-cache": {
+            "version": "10.4.3",
+            "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz",
+            "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==",
+            "dev": true,
+            "license": "ISC"
+        },
         "node_modules/nth-check": {
             "version": "2.1.1",
             "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
@@ -4661,6 +5247,19 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/p-map": {
+            "version": "7.0.3",
+            "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.3.tgz",
+            "integrity": "sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/package-json-from-dist": {
             "version": "1.0.1",
             "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
@@ -4688,6 +5287,24 @@
                 "node": ">=6"
             }
         },
+        "node_modules/parse-json": {
+            "version": "8.3.0",
+            "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-8.3.0.tgz",
+            "integrity": "sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@babel/code-frame": "^7.26.2",
+                "index-to-position": "^1.1.0",
+                "type-fest": "^4.39.1"
+            },
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/parse-semver": {
             "version": "1.1.1",
             "resolved": "https://registry.npmjs.org/parse-semver/-/parse-semver-1.1.1.tgz",
@@ -4795,6 +5412,19 @@
                 "node": "20 || >=22"
             }
         },
+        "node_modules/path-type": {
+            "version": "6.0.0",
+            "resolved": "https://registry.npmjs.org/path-type/-/path-type-6.0.0.tgz",
+            "integrity": "sha512-Vj7sf++t5pBD637NSfkxpHSMfWaeig5+DKWLhcqIYx6mWQz5hdJTGDVMQiJcw1ZYkhs7AazKDGpRVji1LJCZUQ==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/pend": {
             "version": "1.2.0",
             "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@@ -4802,6 +5432,13 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/picocolors": {
+            "version": "1.1.1",
+            "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
+            "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==",
+            "dev": true,
+            "license": "ISC"
+        },
         "node_modules/picomatch": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz",
@@ -4815,6 +5452,16 @@
                 "url": "https://github.com/sponsors/jonschlinkert"
             }
         },
+        "node_modules/pluralize": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-8.0.0.tgz",
+            "integrity": "sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=4"
+            }
+        },
         "node_modules/prebuild-install": {
             "version": "7.1.3",
             "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
@@ -4962,6 +5609,19 @@
                 "rc": "cli.js"
             }
         },
+        "node_modules/rc-config-loader": {
+            "version": "4.1.3",
+            "resolved": "https://registry.npmjs.org/rc-config-loader/-/rc-config-loader-4.1.3.tgz",
+            "integrity": "sha512-kD7FqML7l800i6pS6pvLyIE2ncbk9Du8Q0gp/4hMPhJU6ZxApkoLcGD8ZeqgiAlfwZ6BlETq6qqe+12DUL207w==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "debug": "^4.3.4",
+                "js-yaml": "^4.1.0",
+                "json5": "^2.2.2",
+                "require-from-string": "^2.0.2"
+            }
+        },
         "node_modules/rc/node_modules/strip-json-comments": {
             "version": "2.0.1",
             "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
@@ -4986,6 +5646,39 @@
                 "node": ">=0.8"
             }
         },
+        "node_modules/read-pkg": {
+            "version": "9.0.1",
+            "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-9.0.1.tgz",
+            "integrity": "sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@types/normalize-package-data": "^2.4.3",
+                "normalize-package-data": "^6.0.0",
+                "parse-json": "^8.0.0",
+                "type-fest": "^4.6.0",
+                "unicorn-magic": "^0.1.0"
+            },
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/read-pkg/node_modules/unicorn-magic": {
+            "version": "0.1.0",
+            "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.1.0.tgz",
+            "integrity": "sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/readable-stream": {
             "version": "2.3.8",
             "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
@@ -5018,6 +5711,16 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/require-from-string": {
+            "version": "2.0.2",
+            "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
+            "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=0.10.0"
+            }
+        },
         "node_modules/resolve-from": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@@ -5146,6 +5849,28 @@
             "dev": true,
             "license": "ISC"
         },
+        "node_modules/secretlint": {
+            "version": "10.2.2",
+            "resolved": "https://registry.npmjs.org/secretlint/-/secretlint-10.2.2.tgz",
+            "integrity": "sha512-xVpkeHV/aoWe4vP4TansF622nBEImzCY73y/0042DuJ29iKIaqgoJ8fGxre3rVSHHbxar4FdJobmTnLp9AU0eg==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "@secretlint/config-creator": "^10.2.2",
+                "@secretlint/formatter": "^10.2.2",
+                "@secretlint/node": "^10.2.2",
+                "@secretlint/profiler": "^10.2.2",
+                "debug": "^4.4.1",
+                "globby": "^14.1.0",
+                "read-pkg": "^9.0.1"
+            },
+            "bin": {
+                "secretlint": "bin/secretlint.js"
+            },
+            "engines": {
+                "node": ">=20.0.0"
+            }
+        },
         "node_modules/semver": {
             "version": "7.7.1",
             "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
@@ -5326,6 +6051,80 @@
                 "simple-concat": "^1.0.0"
             }
         },
+        "node_modules/slash": {
+            "version": "5.1.0",
+            "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz",
+            "integrity": "sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=14.16"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/slice-ansi": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
+            "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "ansi-styles": "^4.0.0",
+                "astral-regex": "^2.0.0",
+                "is-fullwidth-code-point": "^3.0.0"
+            },
+            "engines": {
+                "node": ">=10"
+            },
+            "funding": {
+                "url": "https://github.com/chalk/slice-ansi?sponsor=1"
+            }
+        },
+        "node_modules/spdx-correct": {
+            "version": "3.2.0",
+            "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz",
+            "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==",
+            "dev": true,
+            "license": "Apache-2.0",
+            "dependencies": {
+                "spdx-expression-parse": "^3.0.0",
+                "spdx-license-ids": "^3.0.0"
+            }
+        },
+        "node_modules/spdx-exceptions": {
+            "version": "2.5.0",
+            "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz",
+            "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==",
+            "dev": true,
+            "license": "CC-BY-3.0"
+        },
+        "node_modules/spdx-expression-parse": {
+            "version": "3.0.1",
+            "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz",
+            "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "spdx-exceptions": "^2.1.0",
+                "spdx-license-ids": "^3.0.0"
+            }
+        },
+        "node_modules/spdx-license-ids": {
+            "version": "3.0.22",
+            "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz",
+            "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==",
+            "dev": true,
+            "license": "CC0-1.0"
+        },
+        "node_modules/sprintf-js": {
+            "version": "1.0.3",
+            "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+            "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
+            "dev": true,
+            "license": "BSD-3-Clause"
+        },
         "node_modules/stdin-discarder": {
             "version": "0.1.0",
             "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.1.0.tgz",
@@ -5487,23 +6286,136 @@
                 "url": "https://github.com/sponsors/sindresorhus"
             }
         },
+        "node_modules/structured-source": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-4.0.0.tgz",
+            "integrity": "sha512-qGzRFNJDjFieQkl/sVOI2dUjHKRyL9dAJi2gCPGJLbJHBIkyOHxjuocpIEfbLioX+qSJpvbYdT49/YCdMznKxA==",
+            "dev": true,
+            "license": "BSD-2-Clause",
+            "dependencies": {
+                "boundary": "^2.0.0"
+            }
+        },
         "node_modules/supports-color": {
-            "version": "5.5.0",
-            "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-            "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+            "version": "7.2.0",
+            "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+            "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
             "dev": true,
             "license": "MIT",
             "dependencies": {
-                "has-flag": "^3.0.0"
+                "has-flag": "^4.0.0"
             },
             "engines": {
-                "node": ">=4"
+                "node": ">=8"
+            }
+        },
+        "node_modules/supports-hyperlinks": {
+            "version": "3.2.0",
+            "resolved": "https://registry.npmjs.org/supports-hyperlinks/-/supports-hyperlinks-3.2.0.tgz",
+            "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "has-flag": "^4.0.0",
+                "supports-color": "^7.0.0"
+            },
+            "engines": {
+                "node": ">=14.18"
+            },
+            "funding": {
+                "url": "https://github.com/chalk/supports-hyperlinks?sponsor=1"
+            }
+        },
+        "node_modules/table": {
+            "version": "6.9.0",
+            "resolved": "https://registry.npmjs.org/table/-/table-6.9.0.tgz",
+            "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==",
+            "dev": true,
+            "license": "BSD-3-Clause",
+            "dependencies": {
+                "ajv": "^8.0.1",
+                "lodash.truncate": "^4.4.2",
+                "slice-ansi": "^4.0.0",
+                "string-width": "^4.2.3",
+                "strip-ansi": "^6.0.1"
+            },
+            "engines": {
+                "node": ">=10.0.0"
+            }
+        },
+        "node_modules/table/node_modules/ajv": {
+            "version": "8.17.1",
+            "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
+            "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "fast-deep-equal": "^3.1.3",
+                "fast-uri": "^3.0.1",
+                "json-schema-traverse": "^1.0.0",
+                "require-from-string": "^2.0.2"
+            },
+            "funding": {
+                "type": "github",
+                "url": "https://github.com/sponsors/epoberezkin"
+            }
+        },
+        "node_modules/table/node_modules/ansi-regex": {
+            "version": "5.0.1",
+            "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+            "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/table/node_modules/emoji-regex": {
+            "version": "8.0.0",
+            "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+            "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/table/node_modules/json-schema-traverse": {
+            "version": "1.0.0",
+            "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+            "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/table/node_modules/string-width": {
+            "version": "4.2.3",
+            "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+            "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "emoji-regex": "^8.0.0",
+                "is-fullwidth-code-point": "^3.0.0",
+                "strip-ansi": "^6.0.1"
+            },
+            "engines": {
+                "node": ">=8"
+            }
+        },
+        "node_modules/table/node_modules/strip-ansi": {
+            "version": "6.0.1",
+            "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+            "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "ansi-regex": "^5.0.1"
+            },
+            "engines": {
+                "node": ">=8"
             }
         },
         "node_modules/tar-fs": {
-            "version": "2.1.3",
-            "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz",
-            "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==",
+            "version": "2.1.4",
+            "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
+            "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
             "dev": true,
             "license": "MIT",
             "optional": true,
@@ -5587,10 +6499,50 @@
                 "node": ">= 6"
             }
         },
+        "node_modules/terminal-link": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-4.0.0.tgz",
+            "integrity": "sha512-lk+vH+MccxNqgVqSnkMVKx4VLJfnLjDBGzH16JVZjKE2DoxP57s6/vt6JmXV5I3jBcfGrxNrYtC+mPtU7WJztA==",
+            "dev": true,
+            "license": "MIT",
+            "dependencies": {
+                "ansi-escapes": "^7.0.0",
+                "supports-hyperlinks": "^3.2.0"
+            },
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/text-table": {
+            "version": "0.2.0",
+            "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
+            "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
+            "dev": true,
+            "license": "MIT"
+        },
+        "node_modules/textextensions": {
+            "version": "6.11.0",
+            "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-6.11.0.tgz",
+            "integrity": "sha512-tXJwSr9355kFJI3lbCkPpUH5cP8/M0GGy2xLO34aZCjMXBaK3SoPnZwr/oWmo1FdCnELcs4npdCIOFtq9W3ruQ==",
+            "dev": true,
+            "license": "Artistic-2.0",
+            "dependencies": {
+                "editions": "^6.21.0"
+            },
+            "engines": {
+                "node": ">=4"
+            },
+            "funding": {
+                "url": "https://bevry.me/fund"
+            }
+        },
         "node_modules/tmp": {
-            "version": "0.2.3",
-            "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz",
-            "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==",
+            "version": "0.2.4",
+            "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.4.tgz",
+            "integrity": "sha512-UdiSoX6ypifLmrfQ/XfiawN6hkjSBpCjhKxxZcWlUUmoXLaCKQU0bx4HF/tdDK2uzRuchf1txGvrWBzYREssoQ==",
             "dev": true,
             "license": "MIT",
             "engines": {
@@ -5667,6 +6619,19 @@
                 "node": ">= 0.8.0"
             }
         },
+        "node_modules/type-fest": {
+            "version": "4.41.0",
+            "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz",
+            "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==",
+            "dev": true,
+            "license": "(MIT OR CC0-1.0)",
+            "engines": {
+                "node": ">=16"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
         "node_modules/typed-rest-client": {
             "version": "1.8.11",
             "resolved": "https://registry.npmjs.org/typed-rest-client/-/typed-rest-client-1.8.11.tgz",
@@ -5747,6 +6712,29 @@
             "dev": true,
             "license": "MIT"
         },
+        "node_modules/unicorn-magic": {
+            "version": "0.3.0",
+            "resolved": "https://registry.npmjs.org/unicorn-magic/-/unicorn-magic-0.3.0.tgz",
+            "integrity": "sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">=18"
+            },
+            "funding": {
+                "url": "https://github.com/sponsors/sindresorhus"
+            }
+        },
+        "node_modules/universalify": {
+            "version": "2.0.1",
+            "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+            "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+            "dev": true,
+            "license": "MIT",
+            "engines": {
+                "node": ">= 10.0.0"
+            }
+        },
         "node_modules/uri-js": {
             "version": "4.4.1",
             "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
@@ -5781,6 +6769,30 @@
                 "uuid": "dist/bin/uuid"
             }
         },
+        "node_modules/validate-npm-package-license": {
+            "version": "3.0.4",
+            "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz",
+            "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==",
+            "dev": true,
+            "license": "Apache-2.0",
+            "dependencies": {
+                "spdx-correct": "^3.0.0",
+                "spdx-expression-parse": "^3.0.0"
+            }
+        },
+        "node_modules/version-range": {
+            "version": "4.15.0",
+            "resolved": "https://registry.npmjs.org/version-range/-/version-range-4.15.0.tgz",
+            "integrity": "sha512-Ck0EJbAGxHwprkzFO966t4/5QkRuzh+/I1RxhLgUKKwEn+Cd8NwM60mE3AqBZg5gYODoXW0EFsQvbZjRlvdqbg==",
+            "dev": true,
+            "license": "Artistic-2.0",
+            "engines": {
+                "node": ">=4"
+            },
+            "funding": {
+                "url": "https://bevry.me/fund"
+            }
+        },
         "node_modules/vscode-jsonrpc": {
             "version": "8.2.0",
             "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz",
@@ -5928,42 +6940,6 @@
                 "node": ">=8"
             }
         },
-        "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
-            "version": "4.3.0",
-            "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-            "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "color-convert": "^2.0.1"
-            },
-            "engines": {
-                "node": ">=8"
-            },
-            "funding": {
-                "url": "https://github.com/chalk/ansi-styles?sponsor=1"
-            }
-        },
-        "node_modules/wrap-ansi-cjs/node_modules/color-convert": {
-            "version": "2.0.1",
-            "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
-            "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
-            "dev": true,
-            "license": "MIT",
-            "dependencies": {
-                "color-name": "~1.1.4"
-            },
-            "engines": {
-                "node": ">=7.0.0"
-            }
-        },
-        "node_modules/wrap-ansi-cjs/node_modules/color-name": {
-            "version": "1.1.4",
-            "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
-            "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-            "dev": true,
-            "license": "MIT"
-        },
         "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": {
             "version": "8.0.0",
             "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
diff --git a/src/tools/rust-analyzer/editors/code/package.json b/src/tools/rust-analyzer/editors/code/package.json
index 3cb4c21ee1fb..d659421a0299 100644
--- a/src/tools/rust-analyzer/editors/code/package.json
+++ b/src/tools/rust-analyzer/editors/code/package.json
@@ -58,12 +58,13 @@
         "@stylistic/eslint-plugin": "^4.1.0",
         "@stylistic/eslint-plugin-js": "^4.1.0",
         "@tsconfig/strictest": "^2.0.5",
+        "@types/lodash": "^4.17.20",
         "@types/node": "~22.13.4",
         "@types/vscode": "~1.93.0",
         "@typescript-eslint/eslint-plugin": "^8.25.0",
         "@typescript-eslint/parser": "^8.25.0",
         "@vscode/test-electron": "^2.4.1",
-        "@vscode/vsce": "^3.2.2",
+        "@vscode/vsce": "^3.6.0",
         "esbuild": "^0.25.0",
         "eslint": "^9.21.0",
         "eslint-config-prettier": "^10.0.2",
@@ -354,35 +355,122 @@
         ],
         "configuration": [
             {
-                "title": "general",
+                "title": "Rust Analyzer"
+            },
+            {
+                "title": "Assist"
+            },
+            {
+                "title": "Cache Priming"
+            },
+            {
+                "title": "Cargo"
+            },
+            {
+                "title": "Cfg"
+            },
+            {
+                "title": "Check"
+            },
+            {
+                "title": "Completion"
+            },
+            {
+                "title": "Debug"
+            },
+            {
+                "title": "Diagnostics"
+            },
+            {
+                "title": "Files"
+            },
+            {
+                "title": "Highlight Related"
+            },
+            {
+                "title": "Hover"
+            },
+            {
+                "title": "Imports"
+            },
+            {
+                "title": "Inlay Hints"
+            },
+            {
+                "title": "Interpret"
+            },
+            {
+                "title": "Join Lines"
+            },
+            {
+                "title": "Lens"
+            },
+            {
+                "title": "Lru"
+            },
+            {
+                "title": "Notifications"
+            },
+            {
+                "title": "Proc Macro"
+            },
+            {
+                "title": "References"
+            },
+            {
+                "title": "Runnables"
+            },
+            {
+                "title": "Rustc"
+            },
+            {
+                "title": "Rustfmt"
+            },
+            {
+                "title": "Semantic Highlighting"
+            },
+            {
+                "title": "Signature Info"
+            },
+            {
+                "title": "Typing"
+            },
+            {
+                "title": "Vfs"
+            },
+            {
+                "title": "Workspace"
+            },
+            {
+                "title": "rust-analyzer",
                 "properties": {
                     "rust-analyzer.restartServerOnConfigChange": {
-                        "markdownDescription": "Whether to restart the server automatically when certain settings that require a restart are changed.",
+                        "description": "Restart the server automatically when settings that require a restart are changed.",
                         "default": false,
                         "type": "boolean"
                     },
                     "rust-analyzer.showUnlinkedFileNotification": {
-                        "markdownDescription": "Whether to show a notification for unlinked files asking the user to add the corresponding Cargo.toml to the linked projects setting.",
+                        "description": "Show a notification for unlinked files, prompting the user to add the corresponding Cargo.toml to the linked projects setting.",
                         "default": true,
                         "type": "boolean"
                     },
                     "rust-analyzer.showRequestFailedErrorNotification": {
-                        "markdownDescription": "Whether to show error notifications for failing requests.",
+                        "description": "Show error notifications when requests fail.",
                         "default": true,
                         "type": "boolean"
                     },
                     "rust-analyzer.showDependenciesExplorer": {
-                        "markdownDescription": "Whether to show the dependencies view.",
+                        "description": "Show Rust Dependencies in the Explorer view.",
                         "default": true,
                         "type": "boolean"
                     },
                     "rust-analyzer.showSyntaxTree": {
-                        "markdownDescription": "Whether to show the syntax tree view.",
+                        "description": "Show Syntax Tree in the Explorer view.",
                         "default": false,
                         "type": "boolean"
                     },
                     "rust-analyzer.testExplorer": {
-                        "markdownDescription": "Whether to show the test explorer.",
+                        "description": "Show the Test Explorer view.",
                         "default": false,
                         "type": "boolean"
                     },
@@ -394,7 +482,7 @@
                 }
             },
             {
-                "title": "runnables",
+                "title": "Runnables",
                 "properties": {
                     "rust-analyzer.runnables.extraEnv": {
                         "anyOf": [
@@ -452,7 +540,7 @@
                 }
             },
             {
-                "title": "statusBar",
+                "title": "Status Bar",
                 "properties": {
                     "rust-analyzer.statusBar.clickAction": {
                         "type": "string",
@@ -524,7 +612,7 @@
                 }
             },
             {
-                "title": "server",
+                "title": "Server",
                 "properties": {
                     "rust-analyzer.server.path": {
                         "type": [
@@ -553,7 +641,7 @@
                 }
             },
             {
-                "title": "trace",
+                "title": "Trace",
                 "properties": {
                     "rust-analyzer.trace.server": {
                         "type": "string",
@@ -580,7 +668,7 @@
                 }
             },
             {
-                "title": "debug",
+                "title": "Debug",
                 "properties": {
                     "rust-analyzer.debug.engine": {
                         "type": "string",
@@ -625,7 +713,7 @@
                 }
             },
             {
-                "title": "typing",
+                "title": "Typing",
                 "properties": {
                     "rust-analyzer.typing.continueCommentsOnNewline": {
                         "markdownDescription": "Whether to prefix newlines after comments with the corresponding comment prefix.",
@@ -635,7 +723,7 @@
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.previewRustcOutput": {
                         "markdownDescription": "Whether to show the main part of the rendered rustc output of a diagnostic message.",
@@ -653,17 +741,17 @@
                 "title": "$generated-start"
             },
             {
-                "title": "assist",
+                "title": "Assist",
                 "properties": {
                     "rust-analyzer.assist.emitMustUse": {
-                        "markdownDescription": "Whether to insert #[must_use] when generating `as_` methods\nfor enum variants.",
+                        "markdownDescription": "Insert #[must_use] when generating `as_` methods for enum variants.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "assist",
+                "title": "Assist",
                 "properties": {
                     "rust-analyzer.assist.expressionFillDefault": {
                         "markdownDescription": "Placeholder expression to use for missing expressions in assists.",
@@ -681,27 +769,27 @@
                 }
             },
             {
-                "title": "assist",
+                "title": "Assist",
                 "properties": {
                     "rust-analyzer.assist.preferSelf": {
-                        "markdownDescription": "When inserting a type (e.g. in \"fill match arms\" assist), prefer to use `Self` over the type name where possible.",
+                        "markdownDescription": "Prefer to use `Self` over the type name when inserting a type (e.g. in \"fill match arms\" assist).",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "assist",
+                "title": "Assist",
                 "properties": {
                     "rust-analyzer.assist.termSearch.borrowcheck": {
-                        "markdownDescription": "Enable borrow checking for term search code assists. If set to false, also there will be more suggestions, but some of them may not borrow-check.",
+                        "markdownDescription": "Enable borrow checking for term search code assists. If set to false, also there will be\nmore suggestions, but some of them may not borrow-check.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "assist",
+                "title": "Assist",
                 "properties": {
                     "rust-analyzer.assist.termSearch.fuel": {
                         "markdownDescription": "Term search fuel in \"units of work\" for assists (Defaults to 1800).",
@@ -712,7 +800,7 @@
                 }
             },
             {
-                "title": "cachePriming",
+                "title": "Cache Priming",
                 "properties": {
                     "rust-analyzer.cachePriming.enable": {
                         "markdownDescription": "Warm up caches on project load.",
@@ -722,10 +810,10 @@
                 }
             },
             {
-                "title": "cachePriming",
+                "title": "Cache Priming",
                 "properties": {
                     "rust-analyzer.cachePriming.numThreads": {
-                        "markdownDescription": "How many worker threads to handle priming caches. The default `0` means to pick automatically.",
+                        "markdownDescription": "How many worker threads to handle priming caches. The default `0` means to pick\nautomatically.",
                         "default": "physical",
                         "anyOf": [
                             {
@@ -749,7 +837,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.allTargets": {
                         "markdownDescription": "Pass `--all-targets` to cargo invocation.",
@@ -759,7 +847,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.autoreload": {
                         "markdownDescription": "Automatically refresh project info via `cargo metadata` on\n`Cargo.toml` or `.cargo/config.toml` changes.",
@@ -769,7 +857,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.buildScripts.enable": {
                         "markdownDescription": "Run build scripts (`build.rs`) for more precise code analysis.",
@@ -779,7 +867,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.buildScripts.invocationStrategy": {
                         "markdownDescription": "Specifies the invocation strategy to use when running the build scripts command.\nIf `per_workspace` is set, the command will be executed for each Rust workspace with the\nworkspace as the working directory.\nIf `once` is set, the command will be executed once with the opened project as the\nworking directory.\nThis config only has an effect when `#rust-analyzer.cargo.buildScripts.overrideCommand#`\nis set.",
@@ -797,10 +885,10 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.buildScripts.overrideCommand": {
-                        "markdownDescription": "Override the command rust-analyzer uses to run build scripts and\nbuild procedural macros. The command is required to output json\nand should therefore include `--message-format=json` or a similar\noption.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.cargo.buildScripts.invocationStrategy#`.\n\nBy default, a cargo invocation will be constructed for the configured\ntargets and features, with the following base command line:\n\n```bash\ncargo check --quiet --workspace --message-format=json --all-targets --keep-going\n```\n.",
+                        "markdownDescription": "Override the command rust-analyzer uses to run build scripts and\nbuild procedural macros. The command is required to output json\nand should therefore include `--message-format=json` or a similar\noption.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.cargo.buildScripts.invocationStrategy#`.\n\nBy default, a cargo invocation will be constructed for the configured\ntargets and features, with the following base command line:\n\n```bash\ncargo check --quiet --workspace --message-format=json --all-targets --keep-going\n```\n\nNote: The option must be specified as an array of command line arguments, with\nthe first argument being the name of the command to run.",
                         "default": null,
                         "type": [
                             "null",
@@ -813,7 +901,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.buildScripts.rebuildOnSave": {
                         "markdownDescription": "Rerun proc-macros building/build-scripts running when proc-macro\nor build-script sources change and are saved.",
@@ -823,7 +911,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.buildScripts.useRustcWrapper": {
                         "markdownDescription": "Use `RUSTC_WRAPPER=rust-analyzer` when running build scripts to\navoid checking unnecessary things.",
@@ -833,7 +921,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.cfgs": {
                         "markdownDescription": "List of cfg options to enable with the given values.\n\nTo enable a name without a value, use `\"key\"`.\nTo enable a name with a value, use `\"key=value\"`.\nTo disable, prefix the entry with a `!`.",
@@ -849,7 +937,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.extraArgs": {
                         "markdownDescription": "Extra arguments that are passed to every cargo invocation.",
@@ -862,7 +950,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.extraEnv": {
                         "markdownDescription": "Extra environment variables that will be set when running cargo, rustc\nor other commands within the workspace. Useful for setting RUSTFLAGS.",
@@ -872,7 +960,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.features": {
                         "markdownDescription": "List of features to activate.\n\nSet this to `\"all\"` to pass `--all-features` to cargo.",
@@ -898,7 +986,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.noDefaultFeatures": {
                         "markdownDescription": "Whether to pass `--no-default-features` to cargo.",
@@ -908,7 +996,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.noDeps": {
                         "markdownDescription": "Whether to skip fetching dependencies. If set to \"true\", the analysis is performed\nentirely offline, and Cargo metadata for dependencies is not fetched.",
@@ -918,7 +1006,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.sysroot": {
                         "markdownDescription": "Relative path to the sysroot, or \"discover\" to try to automatically find it via\n\"rustc --print sysroot\".\n\nUnsetting this disables sysroot loading.\n\nThis option does not take effect until rust-analyzer is restarted.",
@@ -931,7 +1019,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.sysrootSrc": {
                         "markdownDescription": "Relative path to the sysroot library sources. If left unset, this will default to\n`{cargo.sysroot}/lib/rustlib/src/rust/library`.\n\nThis option does not take effect until rust-analyzer is restarted.",
@@ -944,7 +1032,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.target": {
                         "markdownDescription": "Compilation target override (target tuple).",
@@ -957,7 +1045,7 @@
                 }
             },
             {
-                "title": "cargo",
+                "title": "Cargo",
                 "properties": {
                     "rust-analyzer.cargo.targetDir": {
                         "markdownDescription": "Optional path to a rust-analyzer specific target directory.\nThis prevents rust-analyzer's `cargo check` and initial build-script and proc-macro\nbuilding from locking the `Cargo.lock` at the expense of duplicating build artifacts.\n\nSet to `true` to use a subdirectory of the existing target directory or\nset to a path relative to the workspace to use that path.",
@@ -977,7 +1065,7 @@
                 }
             },
             {
-                "title": "cfg",
+                "title": "Cfg",
                 "properties": {
                     "rust-analyzer.cfg.setTest": {
                         "markdownDescription": "Set `cfg(test)` for local crates. Defaults to true.",
@@ -987,7 +1075,7 @@
                 }
             },
             {
-                "title": "general",
+                "title": "rust-analyzer",
                 "properties": {
                     "rust-analyzer.checkOnSave": {
                         "markdownDescription": "Run the check command for diagnostics on save.",
@@ -997,7 +1085,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.allTargets": {
                         "markdownDescription": "Check all targets and tests (`--all-targets`). Defaults to\n`#rust-analyzer.cargo.allTargets#`.",
@@ -1010,7 +1098,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.command": {
                         "markdownDescription": "Cargo command to use for `cargo check`.",
@@ -1020,7 +1108,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.extraArgs": {
                         "markdownDescription": "Extra arguments for `cargo check`.",
@@ -1033,7 +1121,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.extraEnv": {
                         "markdownDescription": "Extra environment variables that will be set when running `cargo check`.\nExtends `#rust-analyzer.cargo.extraEnv#`.",
@@ -1043,7 +1131,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.features": {
                         "markdownDescription": "List of features to activate. Defaults to\n`#rust-analyzer.cargo.features#`.\n\nSet to `\"all\"` to pass `--all-features` to Cargo.",
@@ -1072,7 +1160,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.ignore": {
                         "markdownDescription": "List of `cargo check` (or other command specified in `check.command`) diagnostics to ignore.\n\nFor example for `cargo check`: `dead_code`, `unused_imports`, `unused_variables`,...",
@@ -1086,7 +1174,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.invocationStrategy": {
                         "markdownDescription": "Specifies the invocation strategy to use when running the check command.\nIf `per_workspace` is set, the command will be executed for each workspace.\nIf `once` is set, the command will be executed once.\nThis config only has an effect when `#rust-analyzer.check.overrideCommand#`\nis set.",
@@ -1104,7 +1192,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.noDefaultFeatures": {
                         "markdownDescription": "Whether to pass `--no-default-features` to Cargo. Defaults to\n`#rust-analyzer.cargo.noDefaultFeatures#`.",
@@ -1117,10 +1205,10 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.overrideCommand": {
-                        "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option\n(if your client supports the `colorDiagnosticOutput` experimental\ncapability, you can use `--message-format=json-diagnostic-rendered-ansi`).\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.check.invocationStrategy#`.\n\nIf `$saved_file` is part of the command, rust-analyzer will pass\nthe absolute path of the saved file to the provided command. This is\nintended to be used with non-Cargo build systems.\nNote that `$saved_file` is experimental and may be removed in the future.\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n.",
+                        "markdownDescription": "Override the command rust-analyzer uses instead of `cargo check` for\ndiagnostics on save. The command is required to output json and\nshould therefore include `--message-format=json` or a similar option\n(if your client supports the `colorDiagnosticOutput` experimental\ncapability, you can use `--message-format=json-diagnostic-rendered-ansi`).\n\nIf you're changing this because you're using some tool wrapping\nCargo, you might also want to change\n`#rust-analyzer.cargo.buildScripts.overrideCommand#`.\n\nIf there are multiple linked projects/workspaces, this command is invoked for\neach of them, with the working directory being the workspace root\n(i.e., the folder containing the `Cargo.toml`). This can be overwritten\nby changing `#rust-analyzer.check.invocationStrategy#`.\n\nIf `$saved_file` is part of the command, rust-analyzer will pass\nthe absolute path of the saved file to the provided command. This is\nintended to be used with non-Cargo build systems.\nNote that `$saved_file` is experimental and may be removed in the future.\n\nAn example command would be:\n\n```bash\ncargo check --workspace --message-format=json --all-targets\n```\n\nNote: The option must be specified as an array of command line arguments, with\nthe first argument being the name of the command to run.",
                         "default": null,
                         "type": [
                             "null",
@@ -1133,7 +1221,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.targets": {
                         "markdownDescription": "Check for specific targets. Defaults to `#rust-analyzer.cargo.target#` if empty.\n\nCan be a single target, e.g. `\"x86_64-unknown-linux-gnu\"` or a list of targets, e.g.\n`[\"aarch64-apple-darwin\", \"x86_64-apple-darwin\"]`.\n\nAliased as `\"checkOnSave.targets\"`.",
@@ -1156,7 +1244,7 @@
                 }
             },
             {
-                "title": "check",
+                "title": "Check",
                 "properties": {
                     "rust-analyzer.check.workspace": {
                         "markdownDescription": "Whether `--workspace` should be passed to `cargo check`.\nIf false, `-p ` will be passed instead if applicable. In case it is not, no\ncheck will be performed.",
@@ -1166,50 +1254,50 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.addSemicolonToUnit": {
-                        "markdownDescription": "Whether to automatically add a semicolon when completing unit-returning functions.\n\nIn `match` arms it completes a comma instead.",
+                        "markdownDescription": "Automatically add a semicolon when completing unit-returning functions.\n\nIn `match` arms it completes a comma instead.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.autoAwait.enable": {
-                        "markdownDescription": "Toggles the additional completions that automatically show method calls and field accesses with `await` prefixed to them when completing on a future.",
+                        "markdownDescription": "Show method calls and field accesses completions with `await` prefixed to them when\ncompleting on a future.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.autoIter.enable": {
-                        "markdownDescription": "Toggles the additional completions that automatically show method calls with `iter()` or `into_iter()` prefixed to them when completing on a type that has them.",
+                        "markdownDescription": "Show method call completions with `iter()` or `into_iter()` prefixed to them when\ncompleting on a type that has them.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.autoimport.enable": {
-                        "markdownDescription": "Toggles the additional completions that automatically add imports when completed.\nNote that your client must specify the `additionalTextEdits` LSP client capability to truly have this feature enabled.",
+                        "markdownDescription": "Show completions that automatically add imports when completed.\n\nNote that your client must specify the `additionalTextEdits` LSP client capability to\ntruly have this feature enabled.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.autoimport.exclude": {
-                        "markdownDescription": "A list of full paths to items to exclude from auto-importing completions.\n\nTraits in this list won't have their methods suggested in completions unless the trait\nis in scope.\n\nYou can either specify a string path which defaults to type \"always\" or use the more verbose\nform `{ \"path\": \"path::to::item\", type: \"always\" }`.\n\nFor traits the type \"methods\" can be used to only exclude the methods but not the trait itself.\n\nThis setting also inherits `#rust-analyzer.completion.excludeTraits#`.",
+                        "markdownDescription": "A list of full paths to items to exclude from auto-importing completions.\n\nTraits in this list won't have their methods suggested in completions unless the trait\nis in scope.\n\nYou can either specify a string path which defaults to type \"always\" or use the more\nverbose form `{ \"path\": \"path::to::item\", type: \"always\" }`.\n\nFor traits the type \"methods\" can be used to only exclude the methods but not the trait\nitself.\n\nThis setting also inherits `#rust-analyzer.completion.excludeTraits#`.",
                         "default": [
                             {
                                 "path": "core::borrow::Borrow",
@@ -1251,20 +1339,20 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.autoself.enable": {
-                        "markdownDescription": "Toggles the additional completions that automatically show method calls and field accesses\nwith `self` prefixed to them when inside a method.",
+                        "markdownDescription": "Show method calls and field access completions with `self` prefixed to them when\ninside a method.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.callable.snippets": {
-                        "markdownDescription": "Whether to add parenthesis and argument snippets when completing function.",
+                        "markdownDescription": "Add parenthesis and argument snippets when completing function.",
                         "default": "fill_arguments",
                         "type": "string",
                         "enum": [
@@ -1281,10 +1369,10 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.excludeTraits": {
-                        "markdownDescription": "A list of full paths to traits whose methods to exclude from completion.\n\nMethods from these traits won't be completed, even if the trait is in scope. However, they will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or `T where T: Trait`.\n\nNote that the trait themselves can still be completed.",
+                        "markdownDescription": "A list of full paths to traits whose methods to exclude from completion.\n\nMethods from these traits won't be completed, even if the trait is in scope. However,\nthey will still be suggested on expressions whose type is `dyn Trait`, `impl Trait` or\n`T where T: Trait`.\n\nNote that the trait themselves can still be completed.",
                         "default": [],
                         "type": "array",
                         "items": {
@@ -1294,27 +1382,27 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.fullFunctionSignatures.enable": {
-                        "markdownDescription": "Whether to show full function/method signatures in completion docs.",
+                        "markdownDescription": "Show full function / method signatures in completion docs.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.hideDeprecated": {
-                        "markdownDescription": "Whether to omit deprecated items from autocompletion. By default they are marked as deprecated but not hidden.",
+                        "markdownDescription": "Omit deprecated items from completions. By default they are marked as deprecated but not\nhidden.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.limit": {
                         "markdownDescription": "Maximum number of completions to return. If `None`, the limit is infinite.",
@@ -1328,27 +1416,27 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.postfix.enable": {
-                        "markdownDescription": "Whether to show postfix snippets like `dbg`, `if`, `not`, etc.",
+                        "markdownDescription": "Show postfix snippets like `dbg`, `if`, `not`, etc.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.privateEditable.enable": {
-                        "markdownDescription": "Enables completions of private items and fields that are defined in the current workspace even if they are not visible at the current position.",
+                        "markdownDescription": "Show completions of private items and fields that are defined in the current workspace\neven if they are not visible at the current position.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.snippets.custom": {
                         "markdownDescription": "Custom completion snippets.",
@@ -1398,17 +1486,17 @@
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.termSearch.enable": {
-                        "markdownDescription": "Whether to enable term search based snippets like `Some(foo.bar().baz())`.",
+                        "markdownDescription": "Enable term search based snippets like `Some(foo.bar().baz())`.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "completion",
+                "title": "Completion",
                 "properties": {
                     "rust-analyzer.completion.termSearch.fuel": {
                         "markdownDescription": "Term search fuel in \"units of work\" for autocompletion (Defaults to 1000).",
@@ -1419,7 +1507,7 @@
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.disabled": {
                         "markdownDescription": "List of rust-analyzer diagnostics to disable.",
@@ -1433,50 +1521,50 @@
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.enable": {
-                        "markdownDescription": "Whether to show native rust-analyzer diagnostics.",
+                        "markdownDescription": "Show native rust-analyzer diagnostics.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.experimental.enable": {
-                        "markdownDescription": "Whether to show experimental rust-analyzer diagnostics that might\nhave more false positives than usual.",
+                        "markdownDescription": "Show experimental rust-analyzer diagnostics that might have more false positives than\nusual.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.remapPrefix": {
-                        "markdownDescription": "Map of prefixes to be substituted when parsing diagnostic file paths.\nThis should be the reverse mapping of what is passed to `rustc` as `--remap-path-prefix`.",
+                        "markdownDescription": "Map of prefixes to be substituted when parsing diagnostic file paths. This should be the\nreverse mapping of what is passed to `rustc` as `--remap-path-prefix`.",
                         "default": {},
                         "type": "object"
                     }
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.styleLints.enable": {
-                        "markdownDescription": "Whether to run additional style lints.",
+                        "markdownDescription": "Run additional style lints.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.warningsAsHint": {
-                        "markdownDescription": "List of warnings that should be displayed with hint severity.\n\nThe warnings will be indicated by faded text or three dots in code\nand will not show up in the `Problems Panel`.",
+                        "markdownDescription": "List of warnings that should be displayed with hint severity.\n\nThe warnings will be indicated by faded text or three dots in code and will not show up\nin the `Problems Panel`.",
                         "default": [],
                         "type": "array",
                         "items": {
@@ -1486,10 +1574,10 @@
                 }
             },
             {
-                "title": "diagnostics",
+                "title": "Diagnostics",
                 "properties": {
                     "rust-analyzer.diagnostics.warningsAsInfo": {
-                        "markdownDescription": "List of warnings that should be displayed with info severity.\n\nThe warnings will be indicated by a blue squiggly underline in code\nand a blue icon in the `Problems Panel`.",
+                        "markdownDescription": "List of warnings that should be displayed with info severity.\n\nThe warnings will be indicated by a blue squiggly underline in code and a blue icon in\nthe `Problems Panel`.",
                         "default": [],
                         "type": "array",
                         "items": {
@@ -1499,10 +1587,20 @@
                 }
             },
             {
-                "title": "files",
+                "title": "Document",
+                "properties": {
+                    "rust-analyzer.document.symbol.search.excludeLocals": {
+                        "markdownDescription": "Exclude all locals from document symbol search.",
+                        "default": true,
+                        "type": "boolean"
+                    }
+                }
+            },
+            {
+                "title": "Files",
                 "properties": {
                     "rust-analyzer.files.exclude": {
-                        "markdownDescription": "These paths (file/directories) will be ignored by rust-analyzer. They are\nrelative to the workspace root, and globs are not supported. You may\nalso need to add the folders to Code's `files.watcherExclude`.",
+                        "markdownDescription": "List of files to ignore\n\nThese paths (file/directories) will be ignored by rust-analyzer. They are relative to\nthe workspace root, and globs are not supported. You may also need to add the folders to\nCode's `files.watcherExclude`.",
                         "default": [],
                         "type": "array",
                         "items": {
@@ -1512,7 +1610,7 @@
                 }
             },
             {
-                "title": "files",
+                "title": "Files",
                 "properties": {
                     "rust-analyzer.files.watcher": {
                         "markdownDescription": "Controls file watching implementation.",
@@ -1530,167 +1628,167 @@
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.branchExitPoints.enable": {
-                        "markdownDescription": "Enables highlighting of related return values while the cursor is on any `match`, `if`, or match arm arrow (`=>`).",
+                        "markdownDescription": "Highlight related return values while the cursor is on any `match`, `if`, or match arm\narrow (`=>`).",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.breakPoints.enable": {
-                        "markdownDescription": "Enables highlighting of related references while the cursor is on `break`, `loop`, `while`, or `for` keywords.",
+                        "markdownDescription": "Highlight related references while the cursor is on `break`, `loop`, `while`, or `for`\nkeywords.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.closureCaptures.enable": {
-                        "markdownDescription": "Enables highlighting of all captures of a closure while the cursor is on the `|` or move keyword of a closure.",
+                        "markdownDescription": "Highlight all captures of a closure while the cursor is on the `|` or move keyword of a closure.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.exitPoints.enable": {
-                        "markdownDescription": "Enables highlighting of all exit points while the cursor is on any `return`, `?`, `fn`, or return type arrow (`->`).",
+                        "markdownDescription": "Highlight all exit points while the cursor is on any `return`, `?`, `fn`, or return type\narrow (`->`).",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.references.enable": {
-                        "markdownDescription": "Enables highlighting of related references while the cursor is on any identifier.",
+                        "markdownDescription": "Highlight related references while the cursor is on any identifier.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "highlightRelated",
+                "title": "Highlight Related",
                 "properties": {
                     "rust-analyzer.highlightRelated.yieldPoints.enable": {
-                        "markdownDescription": "Enables highlighting of all break points for a loop or block context while the cursor is on any `async` or `await` keywords.",
+                        "markdownDescription": "Highlight all break points for a loop or block context while the cursor is on any\n`async` or `await` keywords.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.debug.enable": {
-                        "markdownDescription": "Whether to show `Debug` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
+                        "markdownDescription": "Show `Debug` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.enable": {
-                        "markdownDescription": "Whether to show HoverActions in Rust files.",
+                        "markdownDescription": "Show HoverActions in Rust files.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.gotoTypeDef.enable": {
-                        "markdownDescription": "Whether to show `Go to Type Definition` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
+                        "markdownDescription": "Show `Go to Type Definition` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.implementations.enable": {
-                        "markdownDescription": "Whether to show `Implementations` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
+                        "markdownDescription": "Show `Implementations` action. Only applies when `#rust-analyzer.hover.actions.enable#`\nis set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.references.enable": {
-                        "markdownDescription": "Whether to show `References` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
+                        "markdownDescription": "Show `References` action. Only applies when `#rust-analyzer.hover.actions.enable#` is\nset.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.run.enable": {
-                        "markdownDescription": "Whether to show `Run` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` is set.",
+                        "markdownDescription": "Show `Run` action. Only applies when `#rust-analyzer.hover.actions.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.actions.updateTest.enable": {
-                        "markdownDescription": "Whether to show `Update Test` action. Only applies when\n`#rust-analyzer.hover.actions.enable#` and `#rust-analyzer.hover.actions.run.enable#` are set.",
+                        "markdownDescription": "Show `Update Test` action. Only applies when `#rust-analyzer.hover.actions.enable#` and\n`#rust-analyzer.hover.actions.run.enable#` are set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.documentation.enable": {
-                        "markdownDescription": "Whether to show documentation on hover.",
+                        "markdownDescription": "Show documentation on hover.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.documentation.keywords.enable": {
-                        "markdownDescription": "Whether to show keyword hover popups. Only applies when\n`#rust-analyzer.hover.documentation.enable#` is set.",
+                        "markdownDescription": "Show keyword hover popups. Only applies when\n`#rust-analyzer.hover.documentation.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.dropGlue.enable": {
-                        "markdownDescription": "Whether to show drop glue information on hover.",
+                        "markdownDescription": "Show drop glue information on hover.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.links.enable": {
                         "markdownDescription": "Use markdown syntax for links on hover.",
@@ -1700,10 +1798,10 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.maxSubstitutionLength": {
-                        "markdownDescription": "Whether to show what types are used as generic arguments in calls etc. on hover, and what is their max length to show such types, beyond it they will be shown with ellipsis.\n\nThis can take three values: `null` means \"unlimited\", the string `\"hide\"` means to not show generic substitutions at all, and a number means to limit them to X characters.\n\nThe default is 20 characters.",
+                        "markdownDescription": "Show what types are used as generic arguments in calls etc. on hover, and limit the max\nlength to show such types, beyond which they will be shown with ellipsis.\n\nThis can take three values: `null` means \"unlimited\", the string `\"hide\"` means to not\nshow generic substitutions at all, and a number means to limit them to X characters.\n\nThe default is 20 characters.",
                         "default": 20,
                         "anyOf": [
                             {
@@ -1723,7 +1821,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.alignment": {
                         "markdownDescription": "How to render the align information in a memory layout hover.",
@@ -1750,17 +1848,17 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.enable": {
-                        "markdownDescription": "Whether to show memory layout data on hover.",
+                        "markdownDescription": "Show memory layout data on hover.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.niches": {
                         "markdownDescription": "How to render the niche information in a memory layout hover.",
@@ -1773,7 +1871,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.offset": {
                         "markdownDescription": "How to render the offset information in a memory layout hover.",
@@ -1800,7 +1898,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.padding": {
                         "markdownDescription": "How to render the padding information in a memory layout hover.",
@@ -1827,7 +1925,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.memoryLayout.size": {
                         "markdownDescription": "How to render the size information in a memory layout hover.",
@@ -1854,7 +1952,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.show.enumVariants": {
                         "markdownDescription": "How many variants of an enum to display when hovering on. Show none if empty.",
@@ -1868,10 +1966,10 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.show.fields": {
-                        "markdownDescription": "How many fields of a struct, variant or union to display when hovering on. Show none if empty.",
+                        "markdownDescription": "How many fields of a struct, variant or union to display when hovering on. Show none if\nempty.",
                         "default": 5,
                         "type": [
                             "null",
@@ -1882,7 +1980,7 @@
                 }
             },
             {
-                "title": "hover",
+                "title": "Hover",
                 "properties": {
                     "rust-analyzer.hover.show.traitAssocItems": {
                         "markdownDescription": "How many associated items of a trait to display when hovering a trait.",
@@ -1896,61 +1994,72 @@
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.granularity.enforce": {
-                        "markdownDescription": "Whether to enforce the import granularity setting for all files. If set to false rust-analyzer will try to keep import styles consistent per file.",
+                        "markdownDescription": "Enforce the import granularity setting for all files. If set to false rust-analyzer will\ntry to keep import styles consistent per file.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.granularity.group": {
                         "markdownDescription": "How imports should be grouped into use statements.",
                         "default": "crate",
                         "type": "string",
-                        "enum": [
-                            "preserve",
-                            "crate",
-                            "module",
-                            "item",
-                            "one"
-                        ],
-                        "enumDescriptions": [
-                            "Do not change the granularity of any imports and preserve the original structure written by the developer.",
-                            "Merge imports from the same crate into a single use statement. Conversely, imports from different crates are split into separate statements.",
-                            "Merge imports from the same module into a single use statement. Conversely, imports from different modules are split into separate statements.",
-                            "Flatten imports so that each has its own use statement.",
-                            "Merge all imports into a single use statement as long as they have the same visibility and attributes."
+                        "anyOf": [
+                            {
+                                "enum": [
+                                    "crate",
+                                    "module",
+                                    "item",
+                                    "one"
+                                ],
+                                "enumDescriptions": [
+                                    "Merge imports from the same crate into a single use statement. Conversely, imports from different crates are split into separate statements.",
+                                    "Merge imports from the same module into a single use statement. Conversely, imports from different modules are split into separate statements.",
+                                    "Flatten imports so that each has its own use statement.",
+                                    "Merge all imports into a single use statement as long as they have the same visibility and attributes."
+                                ]
+                            },
+                            {
+                                "enum": [
+                                    "preserve"
+                                ],
+                                "enumDescriptions": [
+                                    "Deprecated - unless `enforceGranularity` is `true`, the style of the current file is preferred over this setting. Behaves like `item`."
+                                ],
+                                "deprecated": true
+                            }
                         ]
                     }
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.group.enable": {
-                        "markdownDescription": "Group inserted imports by the [following order](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are separated by newlines.",
+                        "markdownDescription": "Group inserted imports by the [following\norder](https://rust-analyzer.github.io/book/features.html#auto-import). Groups are\nseparated by newlines.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.merge.glob": {
-                        "markdownDescription": "Whether to allow import insertion to merge new imports into single path glob imports like `use std::fmt::*;`.",
+                        "markdownDescription": "Allow import insertion to merge new imports into single path glob imports like `use\nstd::fmt::*;`.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.preferNoStd": {
                         "markdownDescription": "Prefer to unconditionally use imports of the core and alloc crate, over the std crate.",
@@ -1960,17 +2069,17 @@
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.preferPrelude": {
-                        "markdownDescription": "Whether to prefer import paths containing a `prelude` module.",
+                        "markdownDescription": "Prefer import paths containing a `prelude` module.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.prefix": {
                         "markdownDescription": "The path structure for newly inserted paths to use.",
@@ -1990,47 +2099,47 @@
                 }
             },
             {
-                "title": "imports",
+                "title": "Imports",
                 "properties": {
                     "rust-analyzer.imports.prefixExternPrelude": {
-                        "markdownDescription": "Whether to prefix external (including std, core) crate imports with `::`. e.g. \"use ::std::io::Read;\".",
+                        "markdownDescription": "Prefix external (including std, core) crate imports with `::`.\n\nE.g. `use ::std::io::Read;`.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.bindingModeHints.enable": {
-                        "markdownDescription": "Whether to show inlay type hints for binding modes.",
+                        "markdownDescription": "Show inlay type hints for binding modes.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.chainingHints.enable": {
-                        "markdownDescription": "Whether to show inlay type hints for method chains.",
+                        "markdownDescription": "Show inlay type hints for method chains.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.closingBraceHints.enable": {
-                        "markdownDescription": "Whether to show inlay hints after a closing `}` to indicate what item it belongs to.",
+                        "markdownDescription": "Show inlay hints after a closing `}` to indicate what item it belongs to.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.closingBraceHints.minLines": {
                         "markdownDescription": "Minimum number of lines required before the `}` until the hint is shown (set to 0 or 1\nto always show them).",
@@ -2041,20 +2150,20 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.closureCaptureHints.enable": {
-                        "markdownDescription": "Whether to show inlay hints for closure captures.",
+                        "markdownDescription": "Show inlay hints for closure captures.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.closureReturnTypeHints.enable": {
-                        "markdownDescription": "Whether to show inlay type hints for return types of closures.",
+                        "markdownDescription": "Show inlay type hints for return types of closures.",
                         "default": "never",
                         "type": "string",
                         "enum": [
@@ -2071,7 +2180,7 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.closureStyle": {
                         "markdownDescription": "Closure notation in type and chaining inlay hints.",
@@ -2093,10 +2202,10 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.discriminantHints.enable": {
-                        "markdownDescription": "Whether to show enum variant discriminant hints.",
+                        "markdownDescription": "Show enum variant discriminant hints.",
                         "default": "never",
                         "type": "string",
                         "enum": [
@@ -2113,10 +2222,20 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
+                "properties": {
+                    "rust-analyzer.inlayHints.expressionAdjustmentHints.disableReborrows": {
+                        "markdownDescription": "Disable reborrows in expression adjustments inlay hints.\n\nReborrows are a pair of a builtin deref then borrow, i.e. `&*`. They are inserted by the compiler but are mostly useless to the programmer.\n\nNote: if the deref is not builtin (an overloaded deref), or the borrow is `&raw const`/`&raw mut`, they are not removed.",
+                        "default": true,
+                        "type": "boolean"
+                    }
+                }
+            },
+            {
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.expressionAdjustmentHints.enable": {
-                        "markdownDescription": "Whether to show inlay hints for type adjustments.",
+                        "markdownDescription": "Show inlay hints for type adjustments.",
                         "default": "never",
                         "type": "string",
                         "enum": [
@@ -2133,20 +2252,20 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.expressionAdjustmentHints.hideOutsideUnsafe": {
-                        "markdownDescription": "Whether to hide inlay hints for type adjustments outside of `unsafe` blocks.",
+                        "markdownDescription": "Hide inlay hints for type adjustments outside of `unsafe` blocks.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.expressionAdjustmentHints.mode": {
-                        "markdownDescription": "Whether to show inlay hints as postfix ops (`.*` instead of `*`, etc).",
+                        "markdownDescription": "Show inlay hints as postfix ops (`.*` instead of `*`, etc).",
                         "default": "prefix",
                         "type": "string",
                         "enum": [
@@ -2165,60 +2284,60 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.genericParameterHints.const.enable": {
-                        "markdownDescription": "Whether to show const generic parameter name inlay hints.",
+                        "markdownDescription": "Show const generic parameter name inlay hints.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.genericParameterHints.lifetime.enable": {
-                        "markdownDescription": "Whether to show generic lifetime parameter name inlay hints.",
+                        "markdownDescription": "Show generic lifetime parameter name inlay hints.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.genericParameterHints.type.enable": {
-                        "markdownDescription": "Whether to show generic type parameter name inlay hints.",
+                        "markdownDescription": "Show generic type parameter name inlay hints.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.implicitDrops.enable": {
-                        "markdownDescription": "Whether to show implicit drop hints.",
+                        "markdownDescription": "Show implicit drop hints.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.implicitSizedBoundHints.enable": {
-                        "markdownDescription": "Whether to show inlay hints for the implied type parameter `Sized` bound.",
+                        "markdownDescription": "Show inlay hints for the implied type parameter `Sized` bound.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.lifetimeElisionHints.enable": {
-                        "markdownDescription": "Whether to show inlay type hints for elided lifetimes in function signatures.",
+                        "markdownDescription": "Show inlay type hints for elided lifetimes in function signatures.",
                         "default": "never",
                         "type": "string",
                         "enum": [
@@ -2235,20 +2354,20 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.lifetimeElisionHints.useParameterNames": {
-                        "markdownDescription": "Whether to prefer using parameter names as the name for elided lifetime hints if possible.",
+                        "markdownDescription": "Prefer using parameter names as the name for elided lifetime hints if possible.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.maxLength": {
-                        "markdownDescription": "Maximum length for inlay hints. Set to null to have an unlimited length.",
+                        "markdownDescription": "Maximum length for inlay hints. Set to null to have an unlimited length.\n\n**Note:** This is mostly a hint, and we don't guarantee to strictly follow the limit.",
                         "default": 25,
                         "type": [
                             "null",
@@ -2259,30 +2378,30 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.parameterHints.enable": {
-                        "markdownDescription": "Whether to show function parameter name inlay hints at the call\nsite.",
+                        "markdownDescription": "Show function parameter name inlay hints at the call site.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.rangeExclusiveHints.enable": {
-                        "markdownDescription": "Whether to show exclusive range inlay hints.",
+                        "markdownDescription": "Show exclusive range inlay hints.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.reborrowHints.enable": {
-                        "markdownDescription": "Whether to show inlay hints for compiler inserted reborrows.\nThis setting is deprecated in favor of #rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.",
+                        "markdownDescription": "Show inlay hints for compiler inserted reborrows.\n\nThis setting is deprecated in favor of\n#rust-analyzer.inlayHints.expressionAdjustmentHints.enable#.",
                         "default": "never",
                         "type": "string",
                         "enum": [
@@ -2299,7 +2418,7 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.renderColons": {
                         "markdownDescription": "Whether to render leading colons for type hints, and trailing colons for parameter hints.",
@@ -2309,57 +2428,57 @@
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.typeHints.enable": {
-                        "markdownDescription": "Whether to show inlay type hints for variables.",
+                        "markdownDescription": "Show inlay type hints for variables.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.typeHints.hideClosureInitialization": {
-                        "markdownDescription": "Whether to hide inlay type hints for `let` statements that initialize to a closure.\nOnly applies to closures with blocks, same as `#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.",
+                        "markdownDescription": "Hide inlay type hints for `let` statements that initialize to a closure.\n\nOnly applies to closures with blocks, same as\n`#rust-analyzer.inlayHints.closureReturnTypeHints.enable#`.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.typeHints.hideClosureParameter": {
-                        "markdownDescription": "Whether to hide inlay parameter type hints for closures.",
+                        "markdownDescription": "Hide inlay parameter type hints for closures.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "inlayHints",
+                "title": "Inlay Hints",
                 "properties": {
                     "rust-analyzer.inlayHints.typeHints.hideNamedConstructor": {
-                        "markdownDescription": "Whether to hide inlay type hints for constructors.",
+                        "markdownDescription": "Hide inlay type hints for constructors.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "interpret",
+                "title": "Interpret",
                 "properties": {
                     "rust-analyzer.interpret.tests": {
-                        "markdownDescription": "Enables the experimental support for interpreting tests.",
+                        "markdownDescription": "Enable the experimental support for interpreting tests.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "joinLines",
+                "title": "Join Lines",
                 "properties": {
                     "rust-analyzer.joinLines.joinAssignments": {
                         "markdownDescription": "Join lines merges consecutive declaration and initialization of an assignment.",
@@ -2369,7 +2488,7 @@
                 }
             },
             {
-                "title": "joinLines",
+                "title": "Join Lines",
                 "properties": {
                     "rust-analyzer.joinLines.joinElseIf": {
                         "markdownDescription": "Join lines inserts else between consecutive ifs.",
@@ -2379,7 +2498,7 @@
                 }
             },
             {
-                "title": "joinLines",
+                "title": "Join Lines",
                 "properties": {
                     "rust-analyzer.joinLines.removeTrailingComma": {
                         "markdownDescription": "Join lines removes trailing commas.",
@@ -2389,7 +2508,7 @@
                 }
             },
             {
-                "title": "joinLines",
+                "title": "Join Lines",
                 "properties": {
                     "rust-analyzer.joinLines.unwrapTrivialBlock": {
                         "markdownDescription": "Join lines unwraps trivial blocks.",
@@ -2399,37 +2518,37 @@
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.debug.enable": {
-                        "markdownDescription": "Whether to show `Debug` lens. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `Debug` lens. Only applies when `#rust-analyzer.lens.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.enable": {
-                        "markdownDescription": "Whether to show CodeLens in Rust files.",
+                        "markdownDescription": "Show CodeLens in Rust files.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.implementations.enable": {
-                        "markdownDescription": "Whether to show `Implementations` lens. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `Implementations` lens. Only applies when `#rust-analyzer.lens.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.location": {
                         "markdownDescription": "Where to render annotations.",
@@ -2447,70 +2566,70 @@
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.references.adt.enable": {
-                        "markdownDescription": "Whether to show `References` lens for Struct, Enum, and Union.\nOnly applies when `#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `References` lens for Struct, Enum, and Union. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.references.enumVariant.enable": {
-                        "markdownDescription": "Whether to show `References` lens for Enum Variants.\nOnly applies when `#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `References` lens for Enum Variants. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.references.method.enable": {
-                        "markdownDescription": "Whether to show `Method References` lens. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `Method References` lens. Only applies when `#rust-analyzer.lens.enable#` is set.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.references.trait.enable": {
-                        "markdownDescription": "Whether to show `References` lens for Trait.\nOnly applies when `#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `References` lens for Trait. Only applies when `#rust-analyzer.lens.enable#` is\nset.",
                         "default": false,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.run.enable": {
-                        "markdownDescription": "Whether to show `Run` lens. Only applies when\n`#rust-analyzer.lens.enable#` is set.",
+                        "markdownDescription": "Show `Run` lens. Only applies when `#rust-analyzer.lens.enable#` is set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "lens",
+                "title": "Lens",
                 "properties": {
                     "rust-analyzer.lens.updateTest.enable": {
-                        "markdownDescription": "Whether to show `Update Test` lens. Only applies when\n`#rust-analyzer.lens.enable#` and `#rust-analyzer.lens.run.enable#` are set.",
+                        "markdownDescription": "Show `Update Test` lens. Only applies when `#rust-analyzer.lens.enable#` and\n`#rust-analyzer.lens.run.enable#` are set.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "general",
+                "title": "rust-analyzer",
                 "properties": {
                     "rust-analyzer.linkedProjects": {
-                        "markdownDescription": "Disable project auto-discovery in favor of explicitly specified set\nof projects.\n\nElements must be paths pointing to `Cargo.toml`,\n`rust-project.json`, `.rs` files (which will be treated as standalone files) or JSON\nobjects in `rust-project.json` format.",
+                        "markdownDescription": "Disable project auto-discovery in favor of explicitly specified set of projects.\n\nElements must be paths pointing to `Cargo.toml`, `rust-project.json`, `.rs` files (which\nwill be treated as standalone files) or JSON objects in `rust-project.json` format.",
                         "default": [],
                         "type": "array",
                         "items": {
@@ -2523,7 +2642,7 @@
                 }
             },
             {
-                "title": "lru",
+                "title": "Lru",
                 "properties": {
                     "rust-analyzer.lru.capacity": {
                         "markdownDescription": "Number of syntax trees rust-analyzer keeps in memory. Defaults to 128.",
@@ -2538,30 +2657,30 @@
                 }
             },
             {
-                "title": "lru",
+                "title": "Lru",
                 "properties": {
                     "rust-analyzer.lru.query.capacities": {
-                        "markdownDescription": "Sets the LRU capacity of the specified queries.",
+                        "markdownDescription": "The LRU capacity of the specified queries.",
                         "default": {},
                         "type": "object"
                     }
                 }
             },
             {
-                "title": "notifications",
+                "title": "Notifications",
                 "properties": {
                     "rust-analyzer.notifications.cargoTomlNotFound": {
-                        "markdownDescription": "Whether to show `can't find Cargo.toml` error message.",
+                        "markdownDescription": "Show `can't find Cargo.toml` error message.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "general",
+                "title": "rust-analyzer",
                 "properties": {
                     "rust-analyzer.numThreads": {
-                        "markdownDescription": "How many worker threads in the main loop. The default `null` means to pick automatically.",
+                        "markdownDescription": "The number of worker threads in the main loop. The default `null` means to pick\nautomatically.",
                         "default": null,
                         "anyOf": [
                             {
@@ -2588,7 +2707,7 @@
                 }
             },
             {
-                "title": "procMacro",
+                "title": "Proc Macro",
                 "properties": {
                     "rust-analyzer.procMacro.attributes.enable": {
                         "markdownDescription": "Expand attribute macros. Requires `#rust-analyzer.procMacro.enable#` to be set.",
@@ -2598,7 +2717,7 @@
                 }
             },
             {
-                "title": "procMacro",
+                "title": "Proc Macro",
                 "properties": {
                     "rust-analyzer.procMacro.enable": {
                         "markdownDescription": "Enable support for procedural macros, implies `#rust-analyzer.cargo.buildScripts.enable#`.",
@@ -2608,7 +2727,7 @@
                 }
             },
             {
-                "title": "procMacro",
+                "title": "Proc Macro",
                 "properties": {
                     "rust-analyzer.procMacro.ignored": {
                         "markdownDescription": "These proc-macros will be ignored when trying to expand them.\n\nThis config takes a map of crate names with the exported proc-macro names to ignore as values.",
@@ -2618,7 +2737,7 @@
                 }
             },
             {
-                "title": "procMacro",
+                "title": "Proc Macro",
                 "properties": {
                     "rust-analyzer.procMacro.server": {
                         "markdownDescription": "Internal config, path to proc-macro server executable.",
@@ -2631,7 +2750,7 @@
                 }
             },
             {
-                "title": "references",
+                "title": "References",
                 "properties": {
                     "rust-analyzer.references.excludeImports": {
                         "markdownDescription": "Exclude imports from find-all-references.",
@@ -2641,7 +2760,7 @@
                 }
             },
             {
-                "title": "references",
+                "title": "References",
                 "properties": {
                     "rust-analyzer.references.excludeTests": {
                         "markdownDescription": "Exclude tests from find-all-references and call-hierarchy.",
@@ -2651,7 +2770,7 @@
                 }
             },
             {
-                "title": "runnables",
+                "title": "Runnables",
                 "properties": {
                     "rust-analyzer.runnables.command": {
                         "markdownDescription": "Command to be executed instead of 'cargo' for runnables.",
@@ -2664,7 +2783,7 @@
                 }
             },
             {
-                "title": "runnables",
+                "title": "Runnables",
                 "properties": {
                     "rust-analyzer.runnables.extraArgs": {
                         "markdownDescription": "Additional arguments to be passed to cargo for runnables such as\ntests or binaries. For example, it may be `--release`.",
@@ -2677,12 +2796,12 @@
                 }
             },
             {
-                "title": "runnables",
+                "title": "Runnables",
                 "properties": {
                     "rust-analyzer.runnables.extraTestBinaryArgs": {
                         "markdownDescription": "Additional arguments to be passed through Cargo to launched tests, benchmarks, or\ndoc-tests.\n\nUnless the launched target uses a\n[custom test harness](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-harness-field),\nthey will end up being interpreted as options to\n[`rustc`’s built-in test harness (“libtest”)](https://doc.rust-lang.org/rustc/tests/index.html#cli-arguments).",
                         "default": [
-                            "--show-output"
+                            "--nocapture"
                         ],
                         "type": "array",
                         "items": {
@@ -2692,7 +2811,7 @@
                 }
             },
             {
-                "title": "rustc",
+                "title": "Rustc",
                 "properties": {
                     "rust-analyzer.rustc.source": {
                         "markdownDescription": "Path to the Cargo.toml of the rust compiler workspace, for usage in rustc_private\nprojects, or \"discover\" to try to automatically find it if the `rustc-dev` component\nis installed.\n\nAny project which uses rust-analyzer with the rustcPrivate\ncrates must set `[package.metadata.rust-analyzer] rustc_private=true` to use it.\n\nThis option does not take effect until rust-analyzer is restarted.",
@@ -2705,7 +2824,7 @@
                 }
             },
             {
-                "title": "rustfmt",
+                "title": "Rustfmt",
                 "properties": {
                     "rust-analyzer.rustfmt.extraArgs": {
                         "markdownDescription": "Additional arguments to `rustfmt`.",
@@ -2718,10 +2837,10 @@
                 }
             },
             {
-                "title": "rustfmt",
+                "title": "Rustfmt",
                 "properties": {
                     "rust-analyzer.rustfmt.overrideCommand": {
-                        "markdownDescription": "Advanced option, fully override the command rust-analyzer uses for\nformatting. This should be the equivalent of `rustfmt` here, and\nnot that of `cargo fmt`. The file contents will be passed on the\nstandard input and the formatted result will be read from the\nstandard output.",
+                        "markdownDescription": "Advanced option, fully override the command rust-analyzer uses for\nformatting. This should be the equivalent of `rustfmt` here, and\nnot that of `cargo fmt`. The file contents will be passed on the\nstandard input and the formatted result will be read from the\nstandard output.\n\nNote: The option must be specified as an array of command line arguments, with\nthe first argument being the name of the command to run.",
                         "default": null,
                         "type": [
                             "null",
@@ -2734,7 +2853,7 @@
                 }
             },
             {
-                "title": "rustfmt",
+                "title": "Rustfmt",
                 "properties": {
                     "rust-analyzer.rustfmt.rangeFormatting.enable": {
                         "markdownDescription": "Enables the use of rustfmt's unstable range formatting command for the\n`textDocument/rangeFormatting` request. The rustfmt option is unstable and only\navailable on a nightly build.",
@@ -2744,7 +2863,17 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
+                "properties": {
+                    "rust-analyzer.semanticHighlighting.comments.enable": {
+                        "markdownDescription": "Use semantic tokens for comments.\n\nIn some editors (e.g. vscode) semantic tokens override other highlighting grammars.\nBy disabling semantic tokens for comments, other grammars can be used to highlight\ntheir contents.",
+                        "default": true,
+                        "type": "boolean"
+                    }
+                }
+            },
+            {
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.doc.comment.inject.enable": {
                         "markdownDescription": "Inject additional highlighting into doc comments.\n\nWhen enabled, rust-analyzer will highlight rust source in doc comments as well as intra\ndoc links.",
@@ -2754,17 +2883,17 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.nonStandardTokens": {
-                        "markdownDescription": "Whether the server is allowed to emit non-standard tokens and modifiers.",
+                        "markdownDescription": "Emit non-standard tokens and modifiers\n\nWhen enabled, rust-analyzer will emit tokens and modifiers that are not part of the\nstandard set of semantic tokens.",
                         "default": true,
                         "type": "boolean"
                     }
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.operator.enable": {
                         "markdownDescription": "Use semantic tokens for operators.\n\nWhen disabled, rust-analyzer will emit semantic tokens only for operator tokens when\nthey are tagged with modifiers.",
@@ -2774,7 +2903,7 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.operator.specialization.enable": {
                         "markdownDescription": "Use specialized semantic tokens for operators.\n\nWhen enabled, rust-analyzer will emit special token types for operator tokens instead\nof the generic `operator` token type.",
@@ -2784,7 +2913,7 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.punctuation.enable": {
                         "markdownDescription": "Use semantic tokens for punctuation.\n\nWhen disabled, rust-analyzer will emit semantic tokens only for punctuation tokens when\nthey are tagged with modifiers or have a special role.",
@@ -2794,7 +2923,7 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.punctuation.separate.macro.bang": {
                         "markdownDescription": "When enabled, rust-analyzer will emit a punctuation semantic token for the `!` of macro\ncalls.",
@@ -2804,7 +2933,7 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.punctuation.specialization.enable": {
                         "markdownDescription": "Use specialized semantic tokens for punctuation.\n\nWhen enabled, rust-analyzer will emit special token types for punctuation tokens instead\nof the generic `punctuation` token type.",
@@ -2814,7 +2943,7 @@
                 }
             },
             {
-                "title": "semanticHighlighting",
+                "title": "Semantic Highlighting",
                 "properties": {
                     "rust-analyzer.semanticHighlighting.strings.enable": {
                         "markdownDescription": "Use semantic tokens for strings.\n\nIn some editors (e.g. vscode) semantic tokens override other highlighting grammars.\nBy disabling semantic tokens for strings, other grammars can be used to highlight\ntheir contents.",
@@ -2824,7 +2953,7 @@
                 }
             },
             {
-                "title": "signatureInfo",
+                "title": "Signature Info",
                 "properties": {
                     "rust-analyzer.signatureInfo.detail": {
                         "markdownDescription": "Show full signature of the callable. Only shows parameters if disabled.",
@@ -2842,7 +2971,7 @@
                 }
             },
             {
-                "title": "signatureInfo",
+                "title": "Signature Info",
                 "properties": {
                     "rust-analyzer.signatureInfo.documentation.enable": {
                         "markdownDescription": "Show documentation.",
@@ -2852,10 +2981,10 @@
                 }
             },
             {
-                "title": "typing",
+                "title": "Typing",
                 "properties": {
                     "rust-analyzer.typing.triggerChars": {
-                        "markdownDescription": "Specify the characters allowed to invoke special on typing triggers.\n- typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing expression\n- typing `=` between two expressions adds `;` when in statement position\n- typing `=` to turn an assignment into an equality comparison removes `;` when in expression position\n- typing `.` in a chain method call auto-indents\n- typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the expression\n- typing `{` in a use item adds a closing `}` in the right place\n- typing `>` to complete a return type `->` will insert a whitespace after it\n- typing `<` in a path or type position inserts a closing `>` after the path or type.",
+                        "markdownDescription": "Specify the characters allowed to invoke special on typing triggers.\n\n- typing `=` after `let` tries to smartly add `;` if `=` is followed by an existing\n    expression\n- typing `=` between two expressions adds `;` when in statement position\n- typing `=` to turn an assignment into an equality comparison removes `;` when in\n    expression position\n- typing `.` in a chain method call auto-indents\n- typing `{` or `(` in front of an expression inserts a closing `}` or `)` after the\n    expression\n- typing `{` in a use item adds a closing `}` in the right place\n- typing `>` to complete a return type `->` will insert a whitespace after it\n- typing `<` in a path or type position inserts a closing `>` after the path or type.",
                         "default": "=.",
                         "type": [
                             "null",
@@ -2865,7 +2994,7 @@
                 }
             },
             {
-                "title": "vfs",
+                "title": "Vfs",
                 "properties": {
                     "rust-analyzer.vfs.extraIncludes": {
                         "markdownDescription": "Additional paths to include in the VFS. Generally for code that is\ngenerated or otherwise managed by a build system outside of Cargo,\nthough Cargo might be the eventual consumer.",
@@ -2878,10 +3007,10 @@
                 }
             },
             {
-                "title": "workspace",
+                "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.discoverConfig": {
-                        "markdownDescription": "Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].\n\n[`DiscoverWorkspaceConfig`] also requires setting `progress_label` and `files_to_watch`.\n`progress_label` is used for the title in progress indicators, whereas `files_to_watch`\nis used to determine which build system-specific files should be watched in order to\nreload rust-analyzer.\n\nBelow is an example of a valid configuration:\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n        \"command\": [\n                \"rust-project\",\n                \"develop-json\"\n        ],\n        \"progressLabel\": \"rust-analyzer\",\n        \"filesToWatch\": [\n                \"BUCK\"\n        ]\n}\n```\n\n## On `DiscoverWorkspaceConfig::command`\n\n**Warning**: This format is provisional and subject to change.\n\n[`DiscoverWorkspaceConfig::command`] *must* return a JSON object\ncorresponding to `DiscoverProjectData::Finished`:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n        Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n        Error { error: String, source: Option },\n        Progress { message: String },\n}\n```\n\nAs JSON, `DiscoverProjectData::Finished` is:\n\n```json\n{\n        // the internally-tagged representation of the enum.\n        \"kind\": \"finished\",\n        // the file used by a non-Cargo build system to define\n        // a package or target.\n        \"buildfile\": \"rust-analyzer/BUILD\",\n        // the contents of a rust-project.json, elided for brevity\n        \"project\": {\n                \"sysroot\": \"foo\",\n                \"crates\": []\n        }\n}\n```\n\nIt is encouraged, but not required, to use the other variants on\n`DiscoverProjectData` to provide a more polished end-user experience.\n\n`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`,\nwhich will be substituted with the JSON-serialized form of the following\nenum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n     Path(AbsPathBuf),\n     Buildfile(AbsPathBuf),\n}\n```\n\nThe JSON representation of `DiscoverArgument::Path` is:\n\n```json\n{\n        \"path\": \"src/main.rs\"\n}\n```\n\nSimilarly, the JSON representation of `DiscoverArgument::Buildfile` is:\n\n```json\n{\n        \"buildfile\": \"BUILD\"\n}\n```\n\n`DiscoverArgument::Path` is used to find and generate a `rust-project.json`,\nand therefore, a workspace, whereas `DiscoverArgument::buildfile` is used to\nto update an existing workspace. As a reference for implementors,\nbuck2's `rust-project` will likely be useful:\nhttps://github.com/facebook/buck2/tree/main/integrations/rust-project.",
+                        "markdownDescription": "Enables automatic discovery of projects using [`DiscoverWorkspaceConfig::command`].\n\n[`DiscoverWorkspaceConfig`] also requires setting `progress_label` and `files_to_watch`.\n`progress_label` is used for the title in progress indicators, whereas `files_to_watch`\nis used to determine which build system-specific files should be watched in order to\nreload rust-analyzer.\n\nBelow is an example of a valid configuration:\n```json\n\"rust-analyzer.workspace.discoverConfig\": {\n        \"command\": [\n                \"rust-project\",\n                \"develop-json\"\n        ],\n        \"progressLabel\": \"rust-analyzer\",\n        \"filesToWatch\": [\n                \"BUCK\"\n        ]\n}\n```\n\n## On `DiscoverWorkspaceConfig::command`\n\n**Warning**: This format is provisional and subject to change.\n\n[`DiscoverWorkspaceConfig::command`] *must* return a JSON object corresponding to\n`DiscoverProjectData::Finished`:\n\n```norun\n#[derive(Debug, Clone, Deserialize, Serialize)]\n#[serde(tag = \"kind\")]\n#[serde(rename_all = \"snake_case\")]\nenum DiscoverProjectData {\n        Finished { buildfile: Utf8PathBuf, project: ProjectJsonData },\n        Error { error: String, source: Option },\n        Progress { message: String },\n}\n```\n\nAs JSON, `DiscoverProjectData::Finished` is:\n\n```json\n{\n        // the internally-tagged representation of the enum.\n        \"kind\": \"finished\",\n        // the file used by a non-Cargo build system to define\n        // a package or target.\n        \"buildfile\": \"rust-analyzer/BUILD\",\n        // the contents of a rust-project.json, elided for brevity\n        \"project\": {\n                \"sysroot\": \"foo\",\n                \"crates\": []\n        }\n}\n```\n\nIt is encouraged, but not required, to use the other variants on `DiscoverProjectData`\nto provide a more polished end-user experience.\n\n`DiscoverWorkspaceConfig::command` may *optionally* include an `{arg}`, which will be\nsubstituted with the JSON-serialized form of the following enum:\n\n```norun\n#[derive(PartialEq, Clone, Debug, Serialize)]\n#[serde(rename_all = \"camelCase\")]\npub enum DiscoverArgument {\n     Path(AbsPathBuf),\n     Buildfile(AbsPathBuf),\n}\n```\n\nThe JSON representation of `DiscoverArgument::Path` is:\n\n```json\n{\n        \"path\": \"src/main.rs\"\n}\n```\n\nSimilarly, the JSON representation of `DiscoverArgument::Buildfile` is:\n\n```json\n{\n        \"buildfile\": \"BUILD\"\n}\n```\n\n`DiscoverArgument::Path` is used to find and generate a `rust-project.json`, and\ntherefore, a workspace, whereas `DiscoverArgument::buildfile` is used to to update an\nexisting workspace. As a reference for implementors, buck2's `rust-project` will likely\nbe useful: https://github.com/facebook/buck2/tree/main/integrations/rust-project.",
                         "default": null,
                         "anyOf": [
                             {
@@ -2912,7 +3041,7 @@
                 }
             },
             {
-                "title": "workspace",
+                "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.symbol.search.excludeImports": {
                         "markdownDescription": "Exclude all imports from workspace symbol search.\n\nIn addition to regular imports (which are always excluded),\nthis option removes public imports (better known as re-exports)\nand removes imports that rename the imported symbol.",
@@ -2922,7 +3051,7 @@
                 }
             },
             {
-                "title": "workspace",
+                "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.symbol.search.kind": {
                         "markdownDescription": "Workspace symbol search kind.",
@@ -2940,7 +3069,7 @@
                 }
             },
             {
-                "title": "workspace",
+                "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.symbol.search.limit": {
                         "markdownDescription": "Limits the number of items returned from a workspace symbol search (Defaults to 128).\nSome clients like vs-code issue new searches on result filtering and don't require all results to be returned in the initial search.\nOther clients requires all results upfront and might require a higher limit.",
@@ -2951,7 +3080,7 @@
                 }
             },
             {
-                "title": "workspace",
+                "title": "Workspace",
                 "properties": {
                     "rust-analyzer.workspace.symbol.search.scope": {
                         "markdownDescription": "Workspace symbol search scope.",
diff --git a/src/tools/rust-analyzer/editors/code/src/client.ts b/src/tools/rust-analyzer/editors/code/src/client.ts
index 073ff2f4703f..cb71a01138b3 100644
--- a/src/tools/rust-analyzer/editors/code/src/client.ts
+++ b/src/tools/rust-analyzer/editors/code/src/client.ts
@@ -13,7 +13,7 @@ import { RaLanguageClient } from "./lang_client";
 export async function createClient(
     traceOutputChannel: vscode.OutputChannel,
     outputChannel: vscode.OutputChannel,
-    initializationOptions: vscode.WorkspaceConfiguration,
+    initializationOptions: lc.LanguageClientOptions["initializationOptions"],
     serverOptions: lc.ServerOptions,
     config: Config,
     unlinkedFiles: vscode.Uri[],
diff --git a/src/tools/rust-analyzer/editors/code/src/config.ts b/src/tools/rust-analyzer/editors/code/src/config.ts
index 3b1b0768d3cf..5dc2c419efa8 100644
--- a/src/tools/rust-analyzer/editors/code/src/config.ts
+++ b/src/tools/rust-analyzer/editors/code/src/config.ts
@@ -4,7 +4,7 @@ import * as path from "path";
 import * as vscode from "vscode";
 import { expectNotUndefined, log, normalizeDriveLetter, unwrapUndefinable } from "./util";
 import type { Env } from "./util";
-import type { Disposable } from "vscode";
+import { cloneDeep, get, pickBy, set } from "lodash";
 
 export type RunnableEnvCfgItem = {
     mask?: string;
@@ -12,13 +12,25 @@ export type RunnableEnvCfgItem = {
     platform?: string | string[];
 };
 
+export type ConfigurationTree = { [key: string]: ConfigurationValue };
+export type ConfigurationValue =
+    | undefined
+    | null
+    | boolean
+    | number
+    | string
+    | ConfigurationValue[]
+    | ConfigurationTree;
+
 type ShowStatusBar = "always" | "never" | { documentSelector: vscode.DocumentSelector };
 
 export class Config {
     readonly extensionId = "rust-lang.rust-analyzer";
-    configureLang: vscode.Disposable | undefined;
 
-    readonly rootSection = "rust-analyzer";
+    configureLang: vscode.Disposable | undefined;
+    workspaceState: vscode.Memento;
+
+    private readonly rootSection = "rust-analyzer";
     private readonly requiresServerReloadOpts = ["server", "files", "showSyntaxTree"].map(
         (opt) => `${this.rootSection}.${opt}`,
     );
@@ -27,8 +39,13 @@ export class Config {
         (opt) => `${this.rootSection}.${opt}`,
     );
 
-    constructor(disposables: Disposable[]) {
-        vscode.workspace.onDidChangeConfiguration(this.onDidChangeConfiguration, this, disposables);
+    constructor(ctx: vscode.ExtensionContext) {
+        this.workspaceState = ctx.workspaceState;
+        vscode.workspace.onDidChangeConfiguration(
+            this.onDidChangeConfiguration,
+            this,
+            ctx.subscriptions,
+        );
         this.refreshLogging();
         this.configureLanguage();
     }
@@ -37,6 +54,44 @@ export class Config {
         this.configureLang?.dispose();
     }
 
+    private readonly extensionConfigurationStateKey = "extensionConfigurations";
+
+    /// Returns the rust-analyzer-specific workspace configuration, incl. any
+    /// configuration items overridden by (present) extensions.
+    get extensionConfigurations(): Record> {
+        return pickBy(
+            this.workspaceState.get>(
+                "extensionConfigurations",
+                {},
+            ),
+            // ignore configurations from disabled/removed extensions
+            (_, extensionId) => vscode.extensions.getExtension(extensionId) !== undefined,
+        );
+    }
+
+    async addExtensionConfiguration(
+        extensionId: string,
+        configuration: Record,
+    ): Promise {
+        const oldConfiguration = this.cfg;
+
+        const extCfgs = this.extensionConfigurations;
+        extCfgs[extensionId] = configuration;
+        await this.workspaceState.update(this.extensionConfigurationStateKey, extCfgs);
+
+        const newConfiguration = this.cfg;
+        const prefix = `${this.rootSection}.`;
+        await this.onDidChangeConfiguration({
+            affectsConfiguration(section: string, _scope?: vscode.ConfigurationScope): boolean {
+                return (
+                    section.startsWith(prefix) &&
+                    get(oldConfiguration, section.slice(prefix.length)) !==
+                        get(newConfiguration, section.slice(prefix.length))
+                );
+            },
+        });
+    }
+
     private refreshLogging() {
         log.info(
             "Extension version:",
@@ -176,10 +231,36 @@ export class Config {
     // We don't do runtime config validation here for simplicity. More on stackoverflow:
     // https://stackoverflow.com/questions/60135780/what-is-the-best-way-to-type-check-the-configuration-for-vscode-extension
 
-    private get cfg(): vscode.WorkspaceConfiguration {
+    // Returns the raw configuration for rust-analyzer as returned by vscode. This
+    // should only be used when modifications to the user/workspace configuration
+    // are required.
+    private get rawCfg(): vscode.WorkspaceConfiguration {
         return vscode.workspace.getConfiguration(this.rootSection);
     }
 
+    // Returns the final configuration to use, with extension configuration overrides merged in.
+    public get cfg(): ConfigurationTree {
+        const finalConfig = cloneDeep(this.rawCfg);
+        for (const [extensionId, items] of Object.entries(this.extensionConfigurations)) {
+            for (const [k, v] of Object.entries(items)) {
+                const i = this.rawCfg.inspect(k);
+                if (
+                    i?.workspaceValue !== undefined ||
+                    i?.workspaceFolderValue !== undefined ||
+                    i?.globalValue !== undefined
+                ) {
+                    log.trace(
+                        `Ignoring configuration override for ${k} from extension ${extensionId}`,
+                    );
+                    continue;
+                }
+                log.trace(`Extension ${extensionId} overrides configuration ${k} to `, v);
+                set(finalConfig, k, v);
+            }
+        }
+        return finalConfig;
+    }
+
     /**
      * Beware that postfix `!` operator erases both `null` and `undefined`.
      * This is why the following doesn't work as expected:
@@ -187,7 +268,6 @@ export class Config {
      * ```ts
      * const nullableNum = vscode
      *  .workspace
-     *  .getConfiguration
      *  .getConfiguration("rust-analyzer")
      *  .get(path)!;
      *
@@ -197,7 +277,7 @@ export class Config {
      * So this getter handles this quirk by not requiring the caller to use postfix `!`
      */
     private get(path: string): T | undefined {
-        return prepareVSCodeConfig(this.cfg.get(path));
+        return prepareVSCodeConfig(get(this.cfg, path)) as T;
     }
 
     get serverPath() {
@@ -223,7 +303,7 @@ export class Config {
     }
 
     async toggleCheckOnSave() {
-        const config = this.cfg.inspect("checkOnSave") ?? { key: "checkOnSave" };
+        const config = this.rawCfg.inspect("checkOnSave") ?? { key: "checkOnSave" };
         let overrideInLanguage;
         let target;
         let value;
@@ -249,7 +329,12 @@ export class Config {
             overrideInLanguage = config.defaultLanguageValue;
             value = config.defaultValue || config.defaultLanguageValue;
         }
-        await this.cfg.update("checkOnSave", !(value || false), target || null, overrideInLanguage);
+        await this.rawCfg.update(
+            "checkOnSave",
+            !(value || false),
+            target || null,
+            overrideInLanguage,
+        );
     }
 
     get problemMatcher(): string[] {
@@ -367,26 +452,24 @@ export class Config {
     }
 
     async setAskBeforeUpdateTest(value: boolean) {
-        await this.cfg.update("runnables.askBeforeUpdateTest", value, true);
+        await this.rawCfg.update("runnables.askBeforeUpdateTest", value, true);
     }
 }
 
-export function prepareVSCodeConfig(resp: T): T {
+export function prepareVSCodeConfig(resp: ConfigurationValue): ConfigurationValue {
     if (Is.string(resp)) {
-        return substituteVSCodeVariableInString(resp) as T;
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    } else if (resp && Is.array(resp)) {
+        return substituteVSCodeVariableInString(resp);
+    } else if (resp && Is.array(resp)) {
         return resp.map((val) => {
             return prepareVSCodeConfig(val);
-        }) as T;
+        });
     } else if (resp && typeof resp === "object") {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const res: { [key: string]: any } = {};
+        const res: ConfigurationTree = {};
         for (const key in resp) {
             const val = resp[key];
             res[key] = prepareVSCodeConfig(val);
         }
-        return res as T;
+        return res;
     }
     return resp;
 }
diff --git a/src/tools/rust-analyzer/editors/code/src/ctx.ts b/src/tools/rust-analyzer/editors/code/src/ctx.ts
index e55754fb9f04..a7b7be03b5d8 100644
--- a/src/tools/rust-analyzer/editors/code/src/ctx.ts
+++ b/src/tools/rust-analyzer/editors/code/src/ctx.ts
@@ -125,7 +125,7 @@ export class Ctx implements RustAnalyzerExtensionApi {
         extCtx.subscriptions.push(this);
         this.version = extCtx.extension.packageJSON.version ?? "";
         this._serverVersion = "";
-        this.config = new Config(extCtx.subscriptions);
+        this.config = new Config(extCtx);
         this.statusBar = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left);
         this.updateStatusBarVisibility(vscode.window.activeTextEditor);
         this.statusBarActiveEditorListener = vscode.window.onDidChangeActiveTextEditor((editor) =>
@@ -150,6 +150,13 @@ export class Ctx implements RustAnalyzerExtensionApi {
         });
     }
 
+    async addConfiguration(
+        extensionId: string,
+        configuration: Record,
+    ): Promise {
+        await this.config.addExtensionConfiguration(extensionId, configuration);
+    }
+
     dispose() {
         this.config.dispose();
         this.statusBar.dispose();
@@ -230,7 +237,7 @@ export class Ctx implements RustAnalyzerExtensionApi {
                 debug: run,
             };
 
-            let rawInitializationOptions = vscode.workspace.getConfiguration("rust-analyzer");
+            let rawInitializationOptions = this.config.cfg;
 
             if (this.workspace.kind === "Detached Files") {
                 rawInitializationOptions = {
diff --git a/src/tools/rust-analyzer/editors/code/src/main.ts b/src/tools/rust-analyzer/editors/code/src/main.ts
index 996298524f11..190f5866d0ea 100644
--- a/src/tools/rust-analyzer/editors/code/src/main.ts
+++ b/src/tools/rust-analyzer/editors/code/src/main.ts
@@ -13,6 +13,12 @@ const RUST_PROJECT_CONTEXT_NAME = "inRustProject";
 export interface RustAnalyzerExtensionApi {
     // FIXME: this should be non-optional
     readonly client?: lc.LanguageClient;
+
+    // Allows adding a configuration override from another extension.
+    // `extensionId` is used to only merge configuration override from present
+    // extensions. `configuration` is map of rust-analyzer-specific setting
+    // overrides, e.g., `{"cargo.cfgs": ["foo", "bar"]}`.
+    addConfiguration(extensionId: string, configuration: Record): Promise;
 }
 
 export async function deactivate() {
diff --git a/src/tools/rust-analyzer/josh-sync.toml b/src/tools/rust-analyzer/josh-sync.toml
index 51ff0d71e71a..a12cfa1b5cbd 100644
--- a/src/tools/rust-analyzer/josh-sync.toml
+++ b/src/tools/rust-analyzer/josh-sync.toml
@@ -1,2 +1,6 @@
 repo = "rust-analyzer"
 filter = ":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer"
+
+[[post-pull]]
+cmd = ["cargo", "fmt"]
+commit-message = "Format code"
diff --git a/src/tools/rust-analyzer/lib/lsp-server/Cargo.toml b/src/tools/rust-analyzer/lib/lsp-server/Cargo.toml
index 1fc1da50a0a0..f56a0de61637 100644
--- a/src/tools/rust-analyzer/lib/lsp-server/Cargo.toml
+++ b/src/tools/rust-analyzer/lib/lsp-server/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lsp-server"
-version = "0.7.8"
+version = "0.7.9"
 description = "Generic LSP server scaffold."
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rust-lang/rust-analyzer/tree/master/lib/lsp-server"
@@ -16,9 +16,9 @@ crossbeam-channel.workspace = true
 [dev-dependencies]
 lsp-types = "=0.95"
 ctrlc = "3.4.7"
-anyhow.workspace     = true
+anyhow.workspace = true
 rustc-hash.workspace = true
-toolchain.workspace  = true
+toolchain.workspace = true
 
 [lints]
 workspace = true
diff --git a/src/tools/rust-analyzer/lib/lsp-server/src/msg.rs b/src/tools/rust-analyzer/lib/lsp-server/src/msg.rs
index 399d674e41d2..305008e69ae1 100644
--- a/src/tools/rust-analyzer/lib/lsp-server/src/msg.rs
+++ b/src/tools/rust-analyzer/lib/lsp-server/src/msg.rs
@@ -84,9 +84,9 @@ pub struct Response {
     // request id. We fail deserialization in that case, so we just
     // make this field mandatory.
     pub id: RequestId,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none", default)]
     pub result: Option,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none", default)]
     pub error: Option,
 }
 
@@ -94,7 +94,7 @@ pub struct Response {
 pub struct ResponseError {
     pub code: i32,
     pub message: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(skip_serializing_if = "Option::is_none", default)]
     pub data: Option,
 }
 
@@ -175,7 +175,7 @@ impl Message {
         let msg = match serde_json::from_str(&text) {
             Ok(msg) => msg,
             Err(e) => {
-                return Err(invalid_data!("malformed LSP payload: {:?}", e));
+                return Err(invalid_data!("malformed LSP payload `{e:?}`: {text:?}"));
             }
         };
 
diff --git a/src/tools/rust-analyzer/rust-version b/src/tools/rust-analyzer/rust-version
index 2178caf63968..c9529fde2363 100644
--- a/src/tools/rust-analyzer/rust-version
+++ b/src/tools/rust-analyzer/rust-version
@@ -1 +1 @@
-733dab558992d902d6d17576de1da768094e2cf3
+fb24b04b096a980bffd80154f6aba22fd07cb3d9
diff --git a/src/tools/rust-analyzer/rustfmt.toml b/src/tools/rust-analyzer/rustfmt.toml
index 20bf59547b86..3ee7fdabdc21 100644
--- a/src/tools/rust-analyzer/rustfmt.toml
+++ b/src/tools/rust-analyzer/rustfmt.toml
@@ -1,2 +1,3 @@
+edition = "2024"
 reorder_modules = true
 use_small_heuristics = "Max"
diff --git a/src/tools/rust-analyzer/triagebot.toml b/src/tools/rust-analyzer/triagebot.toml
index 27fdb672455b..c9862495bc0c 100644
--- a/src/tools/rust-analyzer/triagebot.toml
+++ b/src/tools/rust-analyzer/triagebot.toml
@@ -28,6 +28,3 @@ labels = ["has-merge-commits", "S-waiting-on-author"]
 
 # Prevents mentions in commits to avoid users being spammed
 [no-mentions]
-
-# Automatically close and reopen PRs made by bots to run CI on them
-[bot-pull-requests]
diff --git a/src/tools/rust-analyzer/xtask/src/codegen.rs b/src/tools/rust-analyzer/xtask/src/codegen.rs
index 19ca62e8a329..bc7eb88f3a84 100644
--- a/src/tools/rust-analyzer/xtask/src/codegen.rs
+++ b/src/tools/rust-analyzer/xtask/src/codegen.rs
@@ -173,11 +173,11 @@ fn add_preamble(cg: CodegenType, mut text: String) -> String {
 #[allow(clippy::print_stderr)]
 fn ensure_file_contents(cg: CodegenType, file: &Path, contents: &str, check: bool) -> bool {
     let contents = normalize_newlines(contents);
-    if let Ok(old_contents) = fs::read_to_string(file) {
-        if normalize_newlines(&old_contents) == contents {
-            // File is already up to date.
-            return false;
-        }
+    if let Ok(old_contents) = fs::read_to_string(file)
+        && normalize_newlines(&old_contents) == contents
+    {
+        // File is already up to date.
+        return false;
     }
 
     let display_path = file.strip_prefix(project_root()).unwrap_or(file);
diff --git a/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
index 824b38fc872d..9bd87a7ef5fe 100644
--- a/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
+++ b/src/tools/rust-analyzer/xtask/src/codegen/grammar.rs
@@ -1081,7 +1081,6 @@ fn extract_struct_traits(ast: &mut AstSrc) {
         "Enum",
         "Variant",
         "Trait",
-        "TraitAlias",
         "Module",
         "Static",
         "Const",
diff --git a/src/tools/rust-analyzer/xtask/src/flags.rs b/src/tools/rust-analyzer/xtask/src/flags.rs
index 72f6215d4c3f..8f70a1861893 100644
--- a/src/tools/rust-analyzer/xtask/src/flags.rs
+++ b/src/tools/rust-analyzer/xtask/src/flags.rs
@@ -49,6 +49,9 @@ xflags::xflags! {
             /// build in release with debug info set to 2.
             optional --dev-rel
 
+            /// Make `never!()`, `always!()` etc. panic instead of just logging an error.
+            optional --force-always-assert
+
             /// Apply PGO optimizations
             optional --pgo pgo: PgoTrainingCrate
         }
@@ -124,6 +127,7 @@ pub struct Install {
     pub jemalloc: bool,
     pub proc_macro_server: bool,
     pub dev_rel: bool,
+    pub force_always_assert: bool,
     pub pgo: Option,
 }
 
@@ -300,7 +304,12 @@ impl Install {
         } else {
             Malloc::System
         };
-        Some(ServerOpt { malloc, dev_rel: self.dev_rel, pgo: self.pgo.clone() })
+        Some(ServerOpt {
+            malloc,
+            dev_rel: self.dev_rel,
+            pgo: self.pgo.clone(),
+            force_always_assert: self.force_always_assert,
+        })
     }
     pub(crate) fn proc_macro_server(&self) -> Option {
         if !self.proc_macro_server {
diff --git a/src/tools/rust-analyzer/xtask/src/install.rs b/src/tools/rust-analyzer/xtask/src/install.rs
index f0cc445dfa23..975e361ba50b 100644
--- a/src/tools/rust-analyzer/xtask/src/install.rs
+++ b/src/tools/rust-analyzer/xtask/src/install.rs
@@ -39,6 +39,18 @@ pub(crate) struct ServerOpt {
     pub(crate) malloc: Malloc,
     pub(crate) dev_rel: bool,
     pub(crate) pgo: Option,
+    pub(crate) force_always_assert: bool,
+}
+
+impl ServerOpt {
+    fn to_features(&self) -> Vec<&'static str> {
+        let mut features = Vec::new();
+        features.extend(self.malloc.to_features());
+        if self.force_always_assert {
+            features.extend(["--features", "force-always-assert"]);
+        }
+        features
+    }
 }
 
 pub(crate) struct ProcMacroServerOpt {
@@ -136,7 +148,7 @@ fn install_client(sh: &Shell, client_opt: ClientOpt) -> anyhow::Result<()> {
 }
 
 fn install_server(sh: &Shell, opts: ServerOpt) -> anyhow::Result<()> {
-    let features = opts.malloc.to_features();
+    let features = &opts.to_features();
     let profile = if opts.dev_rel { "dev-rel" } else { "release" };
 
     let mut install_cmd = cmd!(
@@ -145,12 +157,12 @@ fn install_server(sh: &Shell, opts: ServerOpt) -> anyhow::Result<()> {
     );
 
     if let Some(train_crate) = opts.pgo {
+        let target = detect_target(sh);
         let build_cmd = cmd!(
             sh,
-            "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --profile={profile} --locked --features force-always-assert {features...}"
+            "cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --target {target} --profile={profile} --locked {features...}"
         );
 
-        let target = detect_target(sh);
         let profile = crate::pgo::gather_pgo_profile(sh, build_cmd, &target, train_crate)?;
         install_cmd = crate::pgo::apply_pgo_to_cmd(install_cmd, &profile);
     }
diff --git a/src/tools/rust-analyzer/xtask/src/metrics.rs b/src/tools/rust-analyzer/xtask/src/metrics.rs
index 6ff6a1b15310..fd4b600b0347 100644
--- a/src/tools/rust-analyzer/xtask/src/metrics.rs
+++ b/src/tools/rust-analyzer/xtask/src/metrics.rs
@@ -16,13 +16,16 @@ type Unit = String;
 impl flags::Metrics {
     pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
         let mut metrics = Metrics::new(sh)?;
-        if !Path::new("./target/rustc-perf").exists() {
-            sh.create_dir("./target/rustc-perf")?;
-            cmd!(sh, "git clone https://github.com/rust-lang/rustc-perf.git ./target/rustc-perf")
-                .run()?;
+        if !Path::new("./target/metrics/rustc-perf").exists() {
+            sh.create_dir("./target/metrics/rustc-perf")?;
+            cmd!(
+                sh,
+                "git clone https://github.com/rust-lang/rustc-perf.git ./target/metrics/rustc-perf"
+            )
+            .run()?;
         }
         {
-            let _d = sh.push_dir("./target/rustc-perf");
+            let _d = sh.push_dir("./target/metrics/rustc-perf");
             let revision = &metrics.perf_revision;
             cmd!(sh, "git reset --hard {revision}").run()?;
         }
@@ -88,11 +91,12 @@ impl Metrics {
 
         cmd!(
             sh,
-            "git clone --depth=1 --branch 1.76.0 https://github.com/rust-lang/rust.git --single-branch"
+            "git clone --depth=1 --branch 1.76.0 https://github.com/rust-lang/rust.git --single-branch ./target/metrics/rust"
         )
         .run()?;
 
-        let output = cmd!(sh, "./target/release/rust-analyzer rustc-tests ./rust").read()?;
+        let output =
+            cmd!(sh, "./target/release/rust-analyzer rustc-tests ./target/metrics/rust").read()?;
         for (metric, value, unit) in parse_metrics(&output) {
             self.report(metric, value, unit.into());
         }
@@ -106,7 +110,7 @@ impl Metrics {
         self.measure_analysis_stats_path(
             sh,
             bench,
-            &format!("./target/rustc-perf/collector/compile-benchmarks/{bench}"),
+            &format!("./target/metrics/rustc-perf/collector/compile-benchmarks/{bench}"),
         )
     }
     fn measure_analysis_stats_path(
@@ -156,7 +160,7 @@ struct Host {
 
 impl Metrics {
     fn new(sh: &Shell) -> anyhow::Result {
-        let host = Host::new(sh)?;
+        let host = Host::new(sh).unwrap_or_else(|_| Host::unknown());
         let timestamp = SystemTime::now();
         let revision = cmd!(sh, "git rev-parse HEAD").read()?;
         let perf_revision = "a584462e145a0c04760fd9391daefb4f6bd13a99".into();
@@ -187,9 +191,13 @@ impl Metrics {
 }
 
 impl Host {
+    fn unknown() -> Host {
+        Host { os: "unknown".into(), cpu: "unknown".into(), mem: "unknown".into() }
+    }
+
     fn new(sh: &Shell) -> anyhow::Result {
         if cfg!(not(target_os = "linux")) {
-            return Ok(Host { os: "unknown".into(), cpu: "unknown".into(), mem: "unknown".into() });
+            return Ok(Host::unknown());
         }
 
         let os = read_field(sh, "/etc/os-release", "PRETTY_NAME=")?.trim_matches('"').to_owned();
diff --git a/src/tools/rust-analyzer/xtask/src/pgo.rs b/src/tools/rust-analyzer/xtask/src/pgo.rs
index 7f7b3311d962..0c6f499811a2 100644
--- a/src/tools/rust-analyzer/xtask/src/pgo.rs
+++ b/src/tools/rust-analyzer/xtask/src/pgo.rs
@@ -53,14 +53,23 @@ pub(crate) fn gather_pgo_profile<'a>(
 
     // Merge profiles into a single file
     let merged_profile = pgo_dir.join("merged.profdata");
-    let profile_files = std::fs::read_dir(pgo_dir)?.filter_map(|entry| {
-        let entry = entry.ok()?;
-        if entry.path().extension() == Some(OsStr::new("profraw")) {
-            Some(entry.path().to_str().unwrap().to_owned())
-        } else {
-            None
-        }
-    });
+    let profile_files = std::fs::read_dir(pgo_dir)?
+        .filter_map(|entry| {
+            let entry = entry.ok()?;
+            if entry.path().extension() == Some(OsStr::new("profraw")) {
+                Some(entry.path().to_str().unwrap().to_owned())
+            } else {
+                None
+            }
+        })
+        .collect::>();
+
+    if profile_files.is_empty() {
+        anyhow::bail!(
+            "rust-analyzer analysis-stats produced no pgo files. This is a bug in rust-analyzer; please file an issue."
+        );
+    }
+
     cmd!(sh, "{llvm_profdata} merge {profile_files...} -o {merged_profile}").run().context(
         "cannot merge PGO profiles. Do you have the rustup `llvm-tools` component installed?",
     )?;
diff --git a/src/tools/rust-analyzer/xtask/src/publish/notes.rs b/src/tools/rust-analyzer/xtask/src/publish/notes.rs
index 93592d4986f8..8d36fcb61b44 100644
--- a/src/tools/rust-analyzer/xtask/src/publish/notes.rs
+++ b/src/tools/rust-analyzer/xtask/src/publish/notes.rs
@@ -72,13 +72,13 @@ impl<'a, 'b, R: BufRead> Converter<'a, 'b, R> {
     }
 
     fn process_document_title(&mut self) -> anyhow::Result<()> {
-        if let Some(Ok(line)) = self.iter.next() {
-            if let Some((level, title)) = get_title(&line) {
-                let title = process_inline_macros(title)?;
-                if level == 1 {
-                    self.write_title(level, &title);
-                    return Ok(());
-                }
+        if let Some(Ok(line)) = self.iter.next()
+            && let Some((level, title)) = get_title(&line)
+        {
+            let title = process_inline_macros(title)?;
+            if level == 1 {
+                self.write_title(level, &title);
+                return Ok(());
             }
         }
         bail!("document title not found")
@@ -141,39 +141,39 @@ impl<'a, 'b, R: BufRead> Converter<'a, 'b, R> {
     }
 
     fn process_source_code_block(&mut self, level: usize) -> anyhow::Result<()> {
-        if let Some(Ok(line)) = self.iter.next() {
-            if let Some(styles) = line.strip_prefix("[source").and_then(|s| s.strip_suffix(']')) {
-                let mut styles = styles.split(',');
-                if !styles.next().unwrap().is_empty() {
-                    bail!("not a source code block");
-                }
-                let language = styles.next();
-                return self.process_listing_block(language, level);
+        if let Some(Ok(line)) = self.iter.next()
+            && let Some(styles) = line.strip_prefix("[source").and_then(|s| s.strip_suffix(']'))
+        {
+            let mut styles = styles.split(',');
+            if !styles.next().unwrap().is_empty() {
+                bail!("not a source code block");
             }
+            let language = styles.next();
+            return self.process_listing_block(language, level);
         }
         bail!("not a source code block")
     }
 
     fn process_listing_block(&mut self, style: Option<&str>, level: usize) -> anyhow::Result<()> {
-        if let Some(Ok(line)) = self.iter.next() {
-            if line == LISTING_DELIMITER {
-                self.write_indent(level);
-                self.output.push_str("```");
-                if let Some(style) = style {
-                    self.output.push_str(style);
-                }
-                self.output.push('\n');
-                while let Some(line) = self.iter.next() {
-                    let line = line?;
-                    if line == LISTING_DELIMITER {
-                        self.write_line("```", level);
-                        return Ok(());
-                    } else {
-                        self.write_line(&line, level);
-                    }
-                }
-                bail!("listing block is not terminated")
+        if let Some(Ok(line)) = self.iter.next()
+            && line == LISTING_DELIMITER
+        {
+            self.write_indent(level);
+            self.output.push_str("```");
+            if let Some(style) = style {
+                self.output.push_str(style);
             }
+            self.output.push('\n');
+            while let Some(line) = self.iter.next() {
+                let line = line?;
+                if line == LISTING_DELIMITER {
+                    self.write_line("```", level);
+                    return Ok(());
+                } else {
+                    self.write_line(&line, level);
+                }
+            }
+            bail!("listing block is not terminated")
         }
         bail!("not a listing block")
     }
@@ -200,49 +200,48 @@ impl<'a, 'b, R: BufRead> Converter<'a, 'b, R> {
     }
 
     fn process_image_block(&mut self, caption: Option<&str>, level: usize) -> anyhow::Result<()> {
-        if let Some(Ok(line)) = self.iter.next() {
-            if let Some((url, attrs)) = parse_media_block(&line, IMAGE_BLOCK_PREFIX) {
-                let alt = if let Some(stripped) =
-                    attrs.strip_prefix('"').and_then(|s| s.strip_suffix('"'))
-                {
+        if let Some(Ok(line)) = self.iter.next()
+            && let Some((url, attrs)) = parse_media_block(&line, IMAGE_BLOCK_PREFIX)
+        {
+            let alt =
+                if let Some(stripped) = attrs.strip_prefix('"').and_then(|s| s.strip_suffix('"')) {
                     stripped
                 } else {
                     attrs
                 };
-                if let Some(caption) = caption {
-                    self.write_caption_line(caption, level);
-                }
-                self.write_indent(level);
-                self.output.push_str("![");
-                self.output.push_str(alt);
-                self.output.push_str("](");
-                self.output.push_str(url);
-                self.output.push_str(")\n");
-                return Ok(());
+            if let Some(caption) = caption {
+                self.write_caption_line(caption, level);
             }
+            self.write_indent(level);
+            self.output.push_str("![");
+            self.output.push_str(alt);
+            self.output.push_str("](");
+            self.output.push_str(url);
+            self.output.push_str(")\n");
+            return Ok(());
         }
         bail!("not a image block")
     }
 
     fn process_video_block(&mut self, caption: Option<&str>, level: usize) -> anyhow::Result<()> {
-        if let Some(Ok(line)) = self.iter.next() {
-            if let Some((url, attrs)) = parse_media_block(&line, VIDEO_BLOCK_PREFIX) {
-                let html_attrs = match attrs {
-                    "options=loop" => "controls loop",
-                    r#"options="autoplay,loop""# => "autoplay controls loop",
-                    _ => bail!("unsupported video syntax"),
-                };
-                if let Some(caption) = caption {
-                    self.write_caption_line(caption, level);
-                }
-                self.write_indent(level);
-                self.output.push_str(r#"\n");
-                return Ok(());
+        if let Some(Ok(line)) = self.iter.next()
+            && let Some((url, attrs)) = parse_media_block(&line, VIDEO_BLOCK_PREFIX)
+        {
+            let html_attrs = match attrs {
+                "options=loop" => "controls loop",
+                r#"options="autoplay,loop""# => "autoplay controls loop",
+                _ => bail!("unsupported video syntax"),
+            };
+            if let Some(caption) = caption {
+                self.write_caption_line(caption, level);
             }
+            self.write_indent(level);
+            self.output.push_str(r#"\n");
+            return Ok(());
         }
         bail!("not a video block")
     }
@@ -371,12 +370,11 @@ fn strip_prefix_symbol(line: &str, symbol: char) -> Option<(usize, &str)> {
 }
 
 fn parse_media_block<'a>(line: &'a str, prefix: &str) -> Option<(&'a str, &'a str)> {
-    if let Some(line) = line.strip_prefix(prefix) {
-        if let Some((url, rest)) = line.split_once('[') {
-            if let Some(attrs) = rest.strip_suffix(']') {
-                return Some((url, attrs));
-            }
-        }
+    if let Some(line) = line.strip_prefix(prefix)
+        && let Some((url, rest)) = line.split_once('[')
+        && let Some(attrs) = rest.strip_suffix(']')
+    {
+        return Some((url, attrs));
     }
     None
 }
diff --git a/src/tools/rust-analyzer/xtask/src/release.rs b/src/tools/rust-analyzer/xtask/src/release.rs
index d06a25c8929b..13cb44ebed5a 100644
--- a/src/tools/rust-analyzer/xtask/src/release.rs
+++ b/src/tools/rust-analyzer/xtask/src/release.rs
@@ -43,7 +43,7 @@ impl flags::Release {
             .unwrap_or_default();
 
         let tags = cmd!(sh, "git tag --list").read()?;
-        let prev_tag = tags.lines().filter(|line| is_release_tag(line)).next_back().unwrap();
+        let prev_tag = tags.lines().rfind(|line| is_release_tag(line)).unwrap();
 
         let contents = changelog::get_changelog(sh, changelog_n, &commit, prev_tag, &today)?;
         let path = changelog_dir.join(format!("{today}-changelog-{changelog_n}.adoc"));
diff --git a/src/tools/rust-analyzer/xtask/src/tidy.rs b/src/tools/rust-analyzer/xtask/src/tidy.rs
index f91192b0076b..40997eb93d35 100644
--- a/src/tools/rust-analyzer/xtask/src/tidy.rs
+++ b/src/tools/rust-analyzer/xtask/src/tidy.rs
@@ -235,6 +235,10 @@ impl TidyDocs {
             return;
         }
 
+        if is_ported_from_rustc(path, &["crates/hir-ty/src/next_solver"]) {
+            return;
+        }
+
         let first_line = match text.lines().next() {
             Some(it) => it,
             None => return,
@@ -255,7 +259,7 @@ impl TidyDocs {
         }
 
         fn is_exclude_file(d: &Path) -> bool {
-            let file_names = ["tests.rs", "famous_defs_fixture.rs"];
+            let file_names = ["tests.rs", "famous_defs_fixture.rs", "frontmatter.rs"];
 
             d.file_name()
                 .unwrap_or_default()
@@ -290,6 +294,11 @@ fn is_exclude_dir(p: &Path, dirs_to_exclude: &[&str]) -> bool {
         .any(|it| dirs_to_exclude.contains(&it))
 }
 
+fn is_ported_from_rustc(p: &Path, dirs_to_exclude: &[&str]) -> bool {
+    let p = p.strip_prefix(project_root()).unwrap();
+    dirs_to_exclude.iter().any(|exclude| p.starts_with(exclude))
+}
+
 #[derive(Default)]
 struct TidyMarks {
     hits: HashSet,
diff --git a/src/tools/rustbook/Cargo.lock b/src/tools/rustbook/Cargo.lock
index 5f30c75732c7..5788fef82d70 100644
--- a/src/tools/rustbook/Cargo.lock
+++ b/src/tools/rustbook/Cargo.lock
@@ -47,9 +47,9 @@ dependencies = [
 
 [[package]]
 name = "anstream"
-version = "0.6.19"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
+checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -77,29 +77,29 @@ dependencies = [
 
 [[package]]
 name = "anstyle-query"
-version = "1.1.3"
+version = "1.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
+checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.9"
+version = "3.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
+checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
 dependencies = [
  "anstyle",
  "once_cell_polyfill",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
 name = "anyhow"
-version = "1.0.98"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
 
 [[package]]
 name = "autocfg"
@@ -124,9 +124,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.9.1"
+version = "2.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
+checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29"
 
 [[package]]
 name = "block-buffer"
@@ -156,9 +156,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
 name = "cc"
-version = "1.2.30"
+version = "1.2.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7"
+checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f"
 dependencies = [
  "shlex",
 ]
@@ -185,9 +185,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.41"
+version = "4.5.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9"
+checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -195,9 +195,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.41"
+version = "4.5.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d"
+checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8"
 dependencies = [
  "anstream",
  "anstyle",
@@ -208,18 +208,18 @@ dependencies = [
 
 [[package]]
 name = "clap_complete"
-version = "4.5.55"
+version = "4.5.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5abde44486daf70c5be8b8f8f1b66c49f86236edf6fa2abadb4d961c4c6229a"
+checksum = "4d9501bd3f5f09f7bbee01da9a511073ed30a80cd7a509f1214bb74eadea71ad"
 dependencies = [
  "clap",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.5.41"
+version = "4.5.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
+checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -464,7 +464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
 dependencies = [
  "libc",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -559,14 +559,14 @@ dependencies = [
  "pest_derive",
  "serde",
  "serde_json",
- "thiserror 2.0.12",
+ "thiserror 2.0.15",
 ]
 
 [[package]]
 name = "hashbrown"
-version = "0.15.4"
+version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
 
 [[package]]
 name = "heck"
@@ -807,9 +807,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
-version = "0.2.174"
+version = "0.2.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
+checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
 
 [[package]]
 name = "linereader"
@@ -822,9 +822,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.9.4"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
 
 [[package]]
 name = "litemap"
@@ -1035,7 +1035,7 @@ version = "6.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
  "libc",
  "once_cell",
  "onig_sys",
@@ -1104,7 +1104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
 dependencies = [
  "memchr",
- "thiserror 2.0.12",
+ "thiserror 2.0.15",
  "ucd-trie",
 ]
 
@@ -1240,9 +1240,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.95"
+version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
+checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
 dependencies = [
  "unicode-ident",
 ]
@@ -1253,7 +1253,7 @@ version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
  "memchr",
  "pulldown-cmark-escape 0.10.1",
  "unicase",
@@ -1265,7 +1265,7 @@ version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
  "getopts",
  "memchr",
  "pulldown-cmark-escape 0.11.0",
@@ -1343,11 +1343,11 @@ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.16"
+version = "0.5.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7251471db004e509f4e75a62cca9435365b5ec7bcdff530d612ac7c87c44a792"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
 ]
 
 [[package]]
@@ -1385,6 +1385,7 @@ version = "0.1.0"
 dependencies = [
  "clap",
  "env_logger",
+ "libc",
  "mdbook",
  "mdbook-i18n-helpers",
  "mdbook-spec",
@@ -1393,22 +1394,22 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "1.0.8"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
+checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.60.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "rustversion"
-version = "1.0.21"
+version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
 
 [[package]]
 name = "ryu"
@@ -1459,9 +1460,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.141"
+version = "1.0.142"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3"
+checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
 dependencies = [
  "itoa",
  "memchr",
@@ -1546,9 +1547,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "syn"
-version = "2.0.104"
+version = "2.0.106"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
+checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1612,12 +1613,12 @@ dependencies = [
 
 [[package]]
 name = "terminal_size"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45c6481c4829e4cc63825e62c49186a34538b7b2750b73b266581ffb612fb5ed"
+checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"
 dependencies = [
  "rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
 ]
 
 [[package]]
@@ -1637,11 +1638,11 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "2.0.12"
+version = "2.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
+checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850"
 dependencies = [
- "thiserror-impl 2.0.12",
+ "thiserror-impl 2.0.15",
 ]
 
 [[package]]
@@ -1657,9 +1658,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "2.0.12"
+version = "2.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
+checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1969,7 +1970,7 @@ version = "0.60.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
 dependencies = [
- "windows-targets 0.53.2",
+ "windows-targets 0.53.3",
 ]
 
 [[package]]
@@ -1990,10 +1991,11 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.53.2"
+version = "0.53.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef"
+checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
 dependencies = [
+ "windows-link",
  "windows_aarch64_gnullvm 0.53.0",
  "windows_aarch64_msvc 0.53.0",
  "windows_i686_gnu 0.53.0",
@@ -2115,7 +2117,7 @@ version = "0.39.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
 dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.9.2",
 ]
 
 [[package]]
@@ -2182,9 +2184,9 @@ dependencies = [
 
 [[package]]
 name = "zerovec"
-version = "0.11.2"
+version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
+checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b"
 dependencies = [
  "yoke",
  "zerofrom",
diff --git a/src/tools/rustbook/Cargo.toml b/src/tools/rustbook/Cargo.toml
index c7c6e39f1577..1a5b2c29d20b 100644
--- a/src/tools/rustbook/Cargo.toml
+++ b/src/tools/rustbook/Cargo.toml
@@ -10,6 +10,7 @@ edition = "2021"
 [dependencies]
 clap = "4.0.32"
 env_logger = "0.11"
+libc = "0.2"
 mdbook-trpl = { path = "../../doc/book/packages/mdbook-trpl" }
 mdbook-i18n-helpers = "0.3.3"
 mdbook-spec = { path = "../../doc/reference/mdbook-spec" }
diff --git a/src/tools/rustc-perf b/src/tools/rustc-perf
index dde879cf1087..c0301bc44d17 160000
--- a/src/tools/rustc-perf
+++ b/src/tools/rustc-perf
@@ -1 +1 @@
-Subproject commit dde879cf1087cb34a32287bd8ccc4d545bb9fee5
+Subproject commit c0301bc44d175b9b2c5442b25049475c39d7700c
diff --git a/src/tools/rustdoc-gui-test/Cargo.toml b/src/tools/rustdoc-gui-test/Cargo.toml
index 8d958ac94f30..f7384a98f856 100644
--- a/src/tools/rustdoc-gui-test/Cargo.toml
+++ b/src/tools/rustdoc-gui-test/Cargo.toml
@@ -5,7 +5,6 @@ edition = "2021"
 
 [dependencies]
 build_helper = { path = "../../build_helper" }
-camino = "1"
 compiletest = { path = "../compiletest" }
 getopts = "0.2"
 walkdir = "2"
diff --git a/src/tools/rustdoc-gui-test/src/main.rs b/src/tools/rustdoc-gui-test/src/main.rs
index 42feae8c2080..cd10882fc226 100644
--- a/src/tools/rustdoc-gui-test/src/main.rs
+++ b/src/tools/rustdoc-gui-test/src/main.rs
@@ -5,7 +5,7 @@ use std::sync::Arc;
 
 use build_helper::npm;
 use build_helper::util::try_run;
-use compiletest::directives::TestProps;
+use compiletest::rustdoc_gui_test::RustdocGuiTestProps;
 use config::Config;
 
 mod config;
@@ -43,13 +43,7 @@ fn main() -> Result<(), ()> {
                 .current_dir(path);
 
             if let Some(librs) = find_librs(entry.path()) {
-                let compiletest_c = compiletest::common::Config::incomplete_for_rustdoc_gui_test();
-
-                let test_props = TestProps::from_file(
-                    &camino::Utf8PathBuf::try_from(librs).unwrap(),
-                    None,
-                    &compiletest_c,
-                );
+                let test_props = RustdocGuiTestProps::from_file(&librs);
 
                 if !test_props.compile_flags.is_empty() {
                     cargo.env("RUSTDOCFLAGS", test_props.compile_flags.join(" "));
@@ -71,22 +65,28 @@ fn main() -> Result<(), ()> {
     let mut command = Command::new(&config.nodejs);
 
     command
-        .arg(config.rust_src.join("src/tools/rustdoc-gui/tester.js"))
+        .arg(local_node_modules.join(".bin/browser-ui-test"))
         .arg("--jobs")
         .arg(&config.jobs)
-        .arg("--doc-folder")
+        .arg("--variable")
+        .arg("DOC_PATH")
         .arg(config.out_dir.join("doc"))
-        .arg("--tests-folder")
-        .arg(config.rust_src.join("tests/rustdoc-gui"));
+        .arg("--allow-file-access-from-files")
+        .arg("--display-format")
+        .arg("compact");
 
     if local_node_modules.exists() {
-        // Link the local node_modules if exists.
+        // Link the local node_modules if it exists.
         // This is useful when we run rustdoc-gui-test from outside of the source root.
         command.env("NODE_PATH", local_node_modules);
     }
 
-    for file in &config.goml_files {
-        command.arg("--file").arg(file);
+    if config.goml_files.is_empty() {
+        command.arg("--test-folder").arg(config.rust_src.join("tests/rustdoc-gui"));
+    } else {
+        for file in &config.goml_files {
+            command.arg("--test-file").arg(config.rust_src.join("tests/rustdoc-gui").join(file));
+        }
     }
 
     command.args(&config.test_args);
diff --git a/src/tools/rustdoc-gui/.eslintrc.js b/src/tools/rustdoc-gui/.eslintrc.js
deleted file mode 100644
index 3eccbc74cb95..000000000000
--- a/src/tools/rustdoc-gui/.eslintrc.js
+++ /dev/null
@@ -1,96 +0,0 @@
-module.exports = {
-    "env": {
-        "browser": true,
-        "node": true,
-        "es6": true
-    },
-    "extends": "eslint:recommended",
-    "parserOptions": {
-        "ecmaVersion": 2019,
-        "sourceType": "module"
-    },
-    "rules": {
-        "linebreak-style": [
-            "error",
-            "unix"
-        ],
-        "semi": [
-            "error",
-            "always"
-        ],
-        "quotes": [
-            "error",
-            "double"
-        ],
-        "linebreak-style": [
-            "error",
-            "unix"
-        ],
-        "no-trailing-spaces": "error",
-        "no-var": ["error"],
-        "prefer-const": ["error"],
-        "prefer-arrow-callback": ["error"],
-        "brace-style": [
-            "error",
-            "1tbs",
-            { "allowSingleLine": false }
-        ],
-        "keyword-spacing": [
-            "error",
-            { "before": true, "after": true }
-        ],
-        "arrow-spacing": [
-            "error",
-            { "before": true, "after": true }
-        ],
-        "key-spacing": [
-            "error",
-            { "beforeColon": false, "afterColon": true, "mode": "strict" }
-        ],
-        "func-call-spacing": ["error", "never"],
-        "space-infix-ops": "error",
-        "space-before-function-paren": ["error", "never"],
-        "space-before-blocks": "error",
-        "comma-dangle": ["error", "always-multiline"],
-        "comma-style": ["error", "last"],
-        "max-len": ["error", { "code": 100, "tabWidth": 4 }],
-        "eol-last": ["error", "always"],
-        "arrow-parens": ["error", "as-needed"],
-        "no-unused-vars": [
-            "error",
-            {
-                "argsIgnorePattern": "^_",
-                "varsIgnorePattern": "^_"
-            }
-        ],
-        "eqeqeq": "error",
-        "no-const-assign": "error",
-        "no-debugger": "error",
-        "no-dupe-args": "error",
-        "no-dupe-else-if": "error",
-        "no-dupe-keys": "error",
-        "no-duplicate-case": "error",
-        "no-ex-assign": "error",
-        "no-fallthrough": "error",
-        "no-invalid-regexp": "error",
-        "no-import-assign": "error",
-        "no-self-compare": "error",
-        "no-template-curly-in-string": "error",
-        "block-scoped-var": "error",
-        "guard-for-in": "error",
-        "no-alert": "error",
-        "no-confusing-arrow": "error",
-        "no-div-regex": "error",
-        "no-floating-decimal": "error",
-        "no-implicit-globals": "error",
-        "no-implied-eval": "error",
-        "no-label-var": "error",
-        "no-lonely-if": "error",
-        "no-mixed-operators": "error",
-        "no-multi-assign": "error",
-        "no-return-assign": "error",
-        "no-script-url": "error",
-        "no-sequences": "error",
-        "no-div-regex": "error",
-    }
-};
diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js
deleted file mode 100644
index 9dc32f335a89..000000000000
--- a/src/tools/rustdoc-gui/tester.js
+++ /dev/null
@@ -1,309 +0,0 @@
-// This package needs to be install:
-//
-// ```
-// npm install browser-ui-test
-// ```
-
-const fs = require("fs");
-const path = require("path");
-const os = require("os");
-const {Options, runTest} = require("browser-ui-test");
-
-// If a test fails or errors, we will retry it two more times in case it was a flaky failure.
-const NB_RETRY = 3;
-
-function showHelp() {
-    console.log("rustdoc-js options:");
-    console.log("  --doc-folder [PATH]        : location of the generated doc folder");
-    console.log("  --file [PATH]              : file to run (can be repeated)");
-    console.log("  --debug                    : show extra information about script run");
-    console.log("  --show-text                : render font in pages");
-    console.log("  --no-headless              : disable headless mode");
-    console.log("  --help                     : show this message then quit");
-    console.log("  --tests-folder [PATH]      : location of the .GOML tests folder");
-    console.log("  --jobs [NUMBER]            : number of threads to run tests on");
-    console.log("  --executable-path [PATH]   : path of the browser's executable to be used");
-}
-
-function isNumeric(s) {
-    return /^\d+$/.test(s);
-}
-
-function parseOptions(args) {
-    const opts = {
-        "doc_folder": "",
-        "tests_folder": "",
-        "files": [],
-        "debug": false,
-        "show_text": false,
-        "no_headless": false,
-        "jobs": -1,
-        "executable_path": null,
-    };
-    const correspondences = {
-        "--doc-folder": "doc_folder",
-        "--tests-folder": "tests_folder",
-        "--debug": "debug",
-        "--show-text": "show_text",
-        "--no-headless": "no_headless",
-        "--executable-path": "executable_path",
-    };
-
-    for (let i = 0; i < args.length; ++i) {
-        const arg = args[i];
-        if (arg === "--doc-folder"
-            || arg === "--tests-folder"
-            || arg === "--file"
-            || arg === "--jobs"
-            || arg === "--executable-path") {
-            i += 1;
-            if (i >= args.length) {
-                console.log("Missing argument after `" + arg + "` option.");
-                return null;
-            }
-            const arg_value = args[i];
-            if (arg === "--jobs") {
-                if (!isNumeric(arg_value)) {
-                    console.log(
-                        "`--jobs` option expects a positive number, found `" + arg_value + "`");
-                    return null;
-                }
-                opts["jobs"] = parseInt(arg_value);
-            } else if (arg !== "--file") {
-                opts[correspondences[arg]] = arg_value;
-            } else {
-                opts["files"].push(arg_value);
-            }
-        } else if (arg === "--help") {
-            showHelp();
-            process.exit(0);
-        } else if (correspondences[arg]) {
-            opts[correspondences[arg]] = true;
-        } else {
-            console.log("Unknown option `" + arg + "`.");
-            console.log("Use `--help` to see the list of options");
-            return null;
-        }
-    }
-    if (opts["tests_folder"].length < 1) {
-        console.log("Missing `--tests-folder` option.");
-    } else if (opts["doc_folder"].length < 1) {
-        console.log("Missing `--doc-folder` option.");
-    } else {
-        return opts;
-    }
-    return null;
-}
-
-/// Print single char status information without \n
-function char_printer(n_tests) {
-    const max_per_line = 10;
-    let current = 0;
-    return {
-        successful: function() {
-            current += 1;
-            if (current % max_per_line === 0) {
-                process.stdout.write(`. (${current}/${n_tests})${os.EOL}`);
-            } else {
-                process.stdout.write(".");
-            }
-        },
-        erroneous: function() {
-            current += 1;
-            if (current % max_per_line === 0) {
-                process.stderr.write(`F (${current}/${n_tests})${os.EOL}`);
-            } else {
-                process.stderr.write("F");
-            }
-        },
-        finish: function() {
-            if (current % max_per_line === 0) {
-                // Don't output if we are already at a matching line end
-                console.log("");
-            } else {
-                const spaces = " ".repeat(max_per_line - (current % max_per_line));
-                process.stdout.write(`${spaces} (${current}/${n_tests})${os.EOL}${os.EOL}`);
-            }
-        },
-    };
-}
-
-// Sort array by .file_name property
-function by_filename(a, b) {
-    return a.file_name - b.file_name;
-}
-
-async function runTests(opts, framework_options, files, results, status_bar, showTestFailures) {
-    const tests_queue = [];
-
-    for (const testPath of files) {
-        const callback = runTest(testPath, {"options": framework_options})
-            .then(out => {
-                const [output, nb_failures] = out;
-                results[nb_failures === 0 ? "successful" : "failed"].push({
-                    file_name: testPath,
-                    output: output,
-                });
-                if (nb_failures === 0) {
-                    status_bar.successful();
-                } else if (showTestFailures) {
-                    status_bar.erroneous();
-                }
-            })
-            .catch(err => {
-                results.errored.push({
-                    file_name: testPath,
-                    output: err,
-                });
-                if (showTestFailures) {
-                    status_bar.erroneous();
-                }
-            })
-            .finally(() => {
-                // We now remove the promise from the tests_queue.
-                tests_queue.splice(tests_queue.indexOf(callback), 1);
-            });
-        tests_queue.push(callback);
-        if (opts["jobs"] > 0 && tests_queue.length >= opts["jobs"]) {
-            await Promise.race(tests_queue);
-        }
-    }
-    if (tests_queue.length > 0) {
-        await Promise.all(tests_queue);
-    }
-}
-
-function createEmptyResults() {
-    return {
-        successful: [],
-        failed: [],
-        errored: [],
-    };
-}
-
-async function main(argv) {
-    const opts = parseOptions(argv.slice(2));
-    if (opts === null) {
-        process.exit(1);
-    }
-
-    // Print successful tests too
-    let debug = false;
-    // Run tests in sequentially
-    let headless = true;
-    const framework_options = new Options();
-    try {
-        // This is more convenient that setting fields one by one.
-        const args = [
-            "--variable", "DOC_PATH", opts["doc_folder"].split("\\").join("/"),
-            "--allow-file-access-from-files",
-        ];
-        if (opts["debug"]) {
-            debug = true;
-            args.push("--debug");
-        }
-        if (opts["show_text"]) {
-            args.push("--show-text");
-        }
-        if (opts["no_headless"]) {
-            args.push("--no-headless");
-            headless = false;
-        }
-        if (opts["executable_path"] !== null) {
-            args.push("--executable-path");
-            args.push(opts["executable_path"]);
-        }
-        framework_options.parseArguments(args);
-    } catch (error) {
-        console.error(`invalid argument: ${error}`);
-        process.exit(1);
-    }
-
-    let files;
-    if (opts["files"].length === 0) {
-        files = fs.readdirSync(opts["tests_folder"]);
-    } else {
-        files = opts["files"];
-    }
-    files = files.filter(file => path.extname(file) === ".goml");
-    if (files.length === 0) {
-        console.error("rustdoc-gui: No test selected");
-        process.exit(2);
-    }
-    files.forEach((file_name, index) => {
-        files[index] = path.join(opts["tests_folder"], file_name);
-    });
-    files.sort();
-
-    if (!headless) {
-        opts["jobs"] = 1;
-        console.log("`--no-headless` option is active, disabling concurrency for running tests.");
-    }
-
-    if (opts["jobs"] < 1) {
-        const len = files.length;
-        console.log(
-            `Running ${len} rustdoc-gui (UNBOUNDED concurrency; use "-j#" for a limit) ...`,
-        );
-        process.setMaxListeners(files.length + 1);
-    } else if (headless) {
-        console.log(`Running ${files.length} rustdoc-gui (${opts["jobs"]} concurrently) ...`);
-        process.setMaxListeners(opts["jobs"] + 1);
-    } else {
-        console.log(`Running ${files.length} rustdoc-gui ...`);
-    }
-
-    const originalFilesLen = files.length;
-    const results = createEmptyResults();
-    const status_bar = char_printer(files.length);
-
-    let new_results;
-    for (let it = 0; it < NB_RETRY && files.length > 0; ++it) {
-        new_results = createEmptyResults();
-        await runTests(opts, framework_options, files, new_results, status_bar, it + 1 >= NB_RETRY);
-        Array.prototype.push.apply(results.successful, new_results.successful);
-        // We generate the new list of files with the previously failing tests.
-        files = Array.prototype.concat(new_results.failed, new_results.errored).map(
-            f => f["file_name"]);
-        if (files.length > originalFilesLen / 2) {
-            // If we have too many failing tests, it's very likely not flaky failures anymore so
-            // no need to retry.
-            break;
-        }
-    }
-
-    status_bar.finish();
-
-    Array.prototype.push.apply(results.failed, new_results.failed);
-    Array.prototype.push.apply(results.errored, new_results.errored);
-
-    if (debug) {
-        results.successful.sort(by_filename);
-        results.successful.forEach(r => {
-            console.log(r.output);
-        });
-    }
-
-    if (results.failed.length > 0) {
-        console.log("");
-        results.failed.sort(by_filename);
-        results.failed.forEach(r => {
-            console.log(r.file_name, r.output);
-        });
-    }
-    if (results.errored.length > 0) {
-        console.log(os.EOL);
-        // print run errors on the bottom so developers see them better
-        results.errored.sort(by_filename);
-        results.errored.forEach(r => {
-            console.error(r.file_name, r.output);
-        });
-    }
-
-    if (results.failed.length > 0 || results.errored.length > 0) {
-        process.exit(1);
-    }
-    process.exit(0);
-}
-
-main(process.argv);
diff --git a/src/tools/rustdoc-js/tester.js b/src/tools/rustdoc-js/tester.js
index 0baa179e16b2..124839efdd02 100644
--- a/src/tools/rustdoc-js/tester.js
+++ b/src/tools/rustdoc-js/tester.js
@@ -1,7 +1,8 @@
 /* global globalThis */
+
 const fs = require("fs");
 const path = require("path");
-
+const { isGeneratorObject } = require("util/types");
 
 function arrayToCode(array) {
     return array.map((value, index) => {
@@ -45,23 +46,16 @@ function shouldIgnoreField(fieldName) {
 }
 
 function valueMapper(key, testOutput) {
-    const isAlias = testOutput["is_alias"];
     let value = testOutput[key];
     // To make our life easier, if there is a "parent" type, we add it to the path.
     if (key === "path") {
-        if (testOutput["parent"] !== undefined) {
+        if (testOutput["parent"]) {
             if (value.length > 0) {
                 value += "::" + testOutput["parent"]["name"];
             } else {
                 value = testOutput["parent"]["name"];
             }
-        } else if (testOutput["is_alias"]) {
-            value = valueMapper(key, testOutput["original"]);
         }
-    } else if (isAlias && key === "alias") {
-        value = testOutput["name"];
-    } else if (isAlias && ["name"].includes(key)) {
-        value = testOutput["original"][key];
     }
     return value;
 }
@@ -237,7 +231,7 @@ async function runSearch(query, expected, doSearch, loadedFile, queryName) {
     const ignore_order = loadedFile.ignore_order;
     const exact_check = loadedFile.exact_check;
 
-    const results = await doSearch(query, loadedFile.FILTER_CRATE);
+    const { resultsTable } = await doSearch(query, loadedFile.FILTER_CRATE);
     const error_text = [];
 
     for (const key in expected) {
@@ -247,37 +241,38 @@ async function runSearch(query, expected, doSearch, loadedFile, queryName) {
         if (!Object.prototype.hasOwnProperty.call(expected, key)) {
             continue;
         }
-        if (!Object.prototype.hasOwnProperty.call(results, key)) {
+        if (!Object.prototype.hasOwnProperty.call(resultsTable, key)) {
             error_text.push("==> Unknown key \"" + key + "\"");
             break;
         }
         const entry = expected[key];
 
-        if (exact_check && entry.length !== results[key].length) {
+        if (exact_check && entry.length !== resultsTable[key].length) {
             error_text.push(queryName + "==> Expected exactly " + entry.length +
-                            " results but found " + results[key].length + " in '" + key + "'");
+                            " results but found " + resultsTable[key].length + " in '" + key + "'");
         }
 
         let prev_pos = -1;
         for (const [index, elem] of entry.entries()) {
-            const entry_pos = lookForEntry(elem, results[key]);
+            const entry_pos = lookForEntry(elem, resultsTable[key]);
             if (entry_pos === -1) {
                 error_text.push(queryName + "==> Result not found in '" + key + "': '" +
                                 JSON.stringify(elem) + "'");
                 // By default, we just compare the two first items.
                 let item_to_diff = 0;
-                if ((!ignore_order || exact_check) && index < results[key].length) {
+                if ((!ignore_order || exact_check) && index < resultsTable[key].length) {
                     item_to_diff = index;
                 }
                 error_text.push("Diff of first error:\n" +
-                    betterLookingDiff(elem, results[key][item_to_diff]));
+                    betterLookingDiff(elem, resultsTable[key][item_to_diff]));
             } else if (exact_check === true && prev_pos + 1 !== entry_pos) {
                 error_text.push(queryName + "==> Exact check failed at position " + (prev_pos + 1) +
                                 ": expected '" + JSON.stringify(elem) + "' but found '" +
-                                JSON.stringify(results[key][index]) + "'");
+                                JSON.stringify(resultsTable[key][index]) + "'");
             } else if (ignore_order === false && entry_pos < prev_pos) {
-                error_text.push(queryName + "==> '" + JSON.stringify(elem) + "' was supposed " +
-                                "to be before '" + JSON.stringify(results[key][prev_pos]) + "'");
+                error_text.push(queryName + "==> '" +
+                                JSON.stringify(elem) + "' was supposed to be before '" +
+                                JSON.stringify(resultsTable[key][prev_pos]) + "'");
             } else {
                 prev_pos = entry_pos;
             }
@@ -286,19 +281,20 @@ async function runSearch(query, expected, doSearch, loadedFile, queryName) {
     return error_text;
 }
 
-async function runCorrections(query, corrections, getCorrections, loadedFile) {
-    const qc = await getCorrections(query, loadedFile.FILTER_CRATE);
+async function runCorrections(query, corrections, doSearch, loadedFile) {
+    const { parsedQuery } = await doSearch(query, loadedFile.FILTER_CRATE);
+    const qc = parsedQuery.correction;
     const error_text = [];
 
     if (corrections === null) {
         if (qc !== null) {
-            error_text.push(`==> expected = null, found = ${qc}`);
+            error_text.push(`==> [correction] expected = null, found = ${qc}`);
         }
         return error_text;
     }
 
-    if (qc !== corrections.toLowerCase()) {
-        error_text.push(`==> expected = ${corrections}, found = ${qc}`);
+    if (qc.toLowerCase() !== corrections.toLowerCase()) {
+        error_text.push(`==> [correction] expected = ${corrections}, found = ${qc}`);
     }
 
     return error_text;
@@ -320,7 +316,7 @@ function checkResult(error_text, loadedFile, displaySuccess) {
     return 1;
 }
 
-async function runCheckInner(callback, loadedFile, entry, getCorrections, extra) {
+async function runCheckInner(callback, loadedFile, entry, extra, doSearch) {
     if (typeof entry.query !== "string") {
         console.log("FAILED");
         console.log("==> Missing `query` field");
@@ -338,7 +334,7 @@ async function runCheckInner(callback, loadedFile, entry, getCorrections, extra)
         error_text = await runCorrections(
             entry.query,
             entry.correction,
-            getCorrections,
+            doSearch,
             loadedFile,
         );
         if (checkResult(error_text, loadedFile, false) !== 0) {
@@ -348,16 +344,16 @@ async function runCheckInner(callback, loadedFile, entry, getCorrections, extra)
     return true;
 }
 
-async function runCheck(loadedFile, key, getCorrections, callback) {
+async function runCheck(loadedFile, key, doSearch, callback) {
     const expected = loadedFile[key];
 
     if (Array.isArray(expected)) {
         for (const entry of expected) {
-            if (!await runCheckInner(callback, loadedFile, entry, getCorrections, true)) {
+            if (!await runCheckInner(callback, loadedFile, entry, true, doSearch)) {
                 return 1;
             }
         }
-    } else if (!await runCheckInner(callback, loadedFile, expected, getCorrections, false)) {
+    } else if (!await runCheckInner(callback, loadedFile, expected, false, doSearch)) {
         return 1;
     }
     console.log("OK");
@@ -368,10 +364,10 @@ function hasCheck(content, checkName) {
     return content.startsWith(`const ${checkName}`) || content.includes(`\nconst ${checkName}`);
 }
 
-async function runChecks(testFile, doSearch, parseQuery, getCorrections) {
+async function runChecks(testFile, doSearch, parseQuery, revision) {
     let checkExpected = false;
     let checkParsed = false;
-    let testFileContent = readFile(testFile);
+    let testFileContent = `const REVISION = "${revision}";\n${readFile(testFile)}`;
 
     if (testFileContent.indexOf("FILTER_CRATE") !== -1) {
         testFileContent += "exports.FILTER_CRATE = FILTER_CRATE;";
@@ -397,18 +393,36 @@ async function runChecks(testFile, doSearch, parseQuery, getCorrections) {
     let res = 0;
 
     if (checkExpected) {
-        res += await runCheck(loadedFile, "EXPECTED", getCorrections, (query, expected, text) => {
+        res += await runCheck(loadedFile, "EXPECTED", doSearch, (query, expected, text) => {
             return runSearch(query, expected, doSearch, loadedFile, text);
         });
     }
     if (checkParsed) {
-        res += await runCheck(loadedFile, "PARSED", getCorrections, (query, expected, text) => {
+        res += await runCheck(loadedFile, "PARSED", doSearch, (query, expected, text) => {
             return runParser(query, expected, parseQuery, text);
         });
     }
     return res;
 }
 
+function mostRecentMatch(staticFiles, regex) {
+    const matchingEntries = fs.readdirSync(staticFiles)
+        .filter(f => f.match(regex))
+        .map(f => {
+            const stats = fs.statSync(path.join(staticFiles, f));
+            return {
+                path: f,
+                time: stats.mtimeMs,
+            };
+        });
+    if (matchingEntries.length === 0) {
+        throw "No static file matching regex";
+    }
+    // We sort entries in descending order.
+    matchingEntries.sort((a, b) => b.time - a.time);
+    return matchingEntries[0].path;
+}
+
 /**
  * Load searchNNN.js and search-indexNNN.js.
  *
@@ -416,71 +430,89 @@ async function runChecks(testFile, doSearch, parseQuery, getCorrections) {
  * @param {string} resource_suffix - Version number between filename and .js, e.g. "1.59.0"
  * @returns {Object}               - Object containing keys: `doSearch`, which runs a search
  *   with the loaded index and returns a table of results; `parseQuery`, which is the
- *   `parseQuery` function exported from the search module; and `getCorrections`, which runs
+ *   `parseQuery` function exported from the search module, which runs
  *   a search but returns type name corrections instead of results.
  */
-function loadSearchJS(doc_folder, resource_suffix) {
-    const searchIndexJs = path.join(doc_folder, "search-index" + resource_suffix + ".js");
-    const searchIndex = require(searchIndexJs);
-
-    globalThis.searchState = {
-        descShards: new Map(),
-        loadDesc: async function({descShard, descIndex}) {
-            if (descShard.promise === null) {
-                descShard.promise = new Promise((resolve, reject) => {
-                    descShard.resolve = resolve;
-                    const ds = descShard;
-                    const fname = `${ds.crate}-desc-${ds.shard}-${resource_suffix}.js`;
-                    fs.readFile(
-                        `${doc_folder}/search.desc/${descShard.crate}/${fname}`,
-                        (err, data) => {
-                            if (err) {
-                                reject(err);
-                            } else {
-                                eval(data.toString("utf8"));
-                            }
-                        },
-                    );
-                });
-            }
-            const list = await descShard.promise;
-            return list[descIndex];
-        },
-        loadedDescShard: function(crate, shard, data) {
-            this.descShards.get(crate)[shard].resolve(data.split("\n"));
-        },
-    };
-
+async function loadSearchJS(doc_folder, resource_suffix) {
     const staticFiles = path.join(doc_folder, "static.files");
-    const searchJs = fs.readdirSync(staticFiles).find(f => f.match(/search.*\.js$/));
+    const stringdexJs = mostRecentMatch(staticFiles, /stringdex.*\.js$/);
+    const stringdexModule = require(path.join(staticFiles, stringdexJs));
+    const searchJs = mostRecentMatch(staticFiles, /search-[0-9a-f]{8}.*\.js$/);
     const searchModule = require(path.join(staticFiles, searchJs));
-    searchModule.initSearch(searchIndex.searchIndex);
-    const docSearch = searchModule.docSearch;
+    globalThis.nonnull = (x, msg) => {
+        if (x === null) {
+            throw (msg || "unexpected null value!");
+        } else {
+            return x;
+        }
+    };
+    const { docSearch, DocSearch } = await searchModule.initSearch(
+        stringdexModule.Stringdex,
+        stringdexModule.RoaringBitmap,
+        {
+            loadRoot: callbacks => {
+                for (const key in callbacks) {
+                    if (Object.hasOwn(callbacks, key)) {
+                        globalThis[key] = callbacks[key];
+                    }
+                }
+                const rootJs = readFile(path.join(doc_folder, "search.index/root" +
+                    resource_suffix + ".js"));
+                eval(rootJs);
+            },
+            loadTreeByHash: hashHex => {
+                const shardJs = readFile(path.join(doc_folder, "search.index/" + hashHex + ".js"));
+                eval(shardJs);
+            },
+            loadDataByNameAndHash: (name, hashHex) => {
+                const shardJs = readFile(path.join(doc_folder, "search.index/" + name + "/" +
+                    hashHex + ".js"));
+                eval(shardJs);
+            },
+        },
+    );
     return {
         doSearch: async function(queryStr, filterCrate, currentCrate) {
-            const result = await docSearch.execQuery(searchModule.parseQuery(queryStr),
-                filterCrate, currentCrate);
+            const parsedQuery = DocSearch.parseQuery(queryStr);
+            const result = await docSearch.execQuery(parsedQuery, filterCrate, currentCrate);
+            const resultsTable = {};
             for (const tab in result) {
                 if (!Object.prototype.hasOwnProperty.call(result, tab)) {
                     continue;
                 }
-                if (!(result[tab] instanceof Array)) {
+                if (!isGeneratorObject(result[tab])) {
                     continue;
                 }
-                for (const entry of result[tab]) {
+                resultsTable[tab] = [];
+                for await (const entry of result[tab]) {
+                    const flatEntry = Object.assign({
+                        crate: entry.item.crate,
+                        name: entry.item.name,
+                        path: entry.item.modulePath,
+                        exactPath: entry.item.exactModulePath,
+                        ty: entry.item.ty,
+                    }, entry);
                     for (const key in entry) {
                         if (!Object.prototype.hasOwnProperty.call(entry, key)) {
                             continue;
                         }
-                        if (key === "displayTypeSignature" && entry.displayTypeSignature !== null) {
-                            const {type, mappedNames, whereClause} =
-                                await entry.displayTypeSignature;
-                            entry.displayType = arrayToCode(type);
-                            entry.displayMappedNames = [...mappedNames.entries()]
+                        if (key === "desc" && entry.desc !== null) {
+                            flatEntry.desc = await entry.desc;
+                        } else if (key === "displayTypeSignature" &&
+                            entry.displayTypeSignature !== null
+                        ) {
+                            flatEntry.displayTypeSignature = await entry.displayTypeSignature;
+                            const {
+                                type,
+                                mappedNames,
+                                whereClause,
+                            } = flatEntry.displayTypeSignature;
+                            flatEntry.displayType = arrayToCode(type);
+                            flatEntry.displayMappedNames = [...mappedNames.entries()]
                                 .map(([name, qname]) => {
                                     return `${name} = ${qname}`;
                                 }).join(", ");
-                            entry.displayWhereClause = [...whereClause.entries()]
+                            flatEntry.displayWhereClause = [...whereClause.entries()]
                                 .flatMap(([name, value]) => {
                                     if (value.length === 0) {
                                         return [];
@@ -489,16 +521,12 @@ function loadSearchJS(doc_folder, resource_suffix) {
                                 }).join(", ");
                         }
                     }
+                    resultsTable[tab].push(flatEntry);
                 }
             }
-            return result;
+            return { resultsTable, parsedQuery };
         },
-        getCorrections: function(queryStr, filterCrate, currentCrate) {
-            const parsedQuery = searchModule.parseQuery(queryStr);
-            docSearch.execQuery(parsedQuery, filterCrate, currentCrate);
-            return parsedQuery.correction;
-        },
-        parseQuery: searchModule.parseQuery,
+        parseQuery: DocSearch.parseQuery,
     };
 }
 
@@ -520,6 +548,7 @@ function parseOptions(args) {
         "doc_folder": "",
         "test_folder": "",
         "test_file": [],
+        "revision": "",
     };
     const correspondences = {
         "--resource-suffix": "resource_suffix",
@@ -527,6 +556,7 @@ function parseOptions(args) {
         "--test-folder": "test_folder",
         "--test-file": "test_file",
         "--crate-name": "crate_name",
+        "--revision": "revision",
     };
 
     for (let i = 0; i < args.length; ++i) {
@@ -570,7 +600,7 @@ async function main(argv) {
         return 1;
     }
 
-    const parseAndSearch = loadSearchJS(
+    const parseAndSearch = await loadSearchJS(
         opts["doc_folder"],
         opts["resource_suffix"],
     );
@@ -579,14 +609,11 @@ async function main(argv) {
     const doSearch = function(queryStr, filterCrate) {
         return parseAndSearch.doSearch(queryStr, filterCrate, opts["crate_name"]);
     };
-    const getCorrections = function(queryStr, filterCrate) {
-        return parseAndSearch.getCorrections(queryStr, filterCrate, opts["crate_name"]);
-    };
 
     if (opts["test_file"].length !== 0) {
         for (const file of opts["test_file"]) {
             process.stdout.write(`Testing ${file} ... `);
-            errors += await runChecks(file, doSearch, parseAndSearch.parseQuery, getCorrections);
+            errors += await runChecks(file, doSearch, parseAndSearch.parseQuery, opts.revision);
         }
     } else if (opts["test_folder"].length !== 0) {
         for (const file of fs.readdirSync(opts["test_folder"])) {
@@ -594,8 +621,8 @@ async function main(argv) {
                 continue;
             }
             process.stdout.write(`Testing ${file} ... `);
-            errors += await runChecks(path.join(opts["test_folder"], file), doSearch,
-                    parseAndSearch.parseQuery, getCorrections);
+            errors += await runChecks(path.join(opts["test_folder"], file, ""), doSearch,
+                    parseAndSearch.parseQuery);
         }
     }
     return errors > 0 ? 1 : 0;
diff --git a/src/tools/rustfmt/Cargo.toml b/src/tools/rustfmt/Cargo.toml
index e497b7923420..6392ffbe409f 100644
--- a/src/tools/rustfmt/Cargo.toml
+++ b/src/tools/rustfmt/Cargo.toml
@@ -40,7 +40,7 @@ cargo_metadata = "0.18"
 clap = { version = "4.4.2", features = ["derive"] }
 clap-cargo = "0.12.0"
 diff = "0.1"
-dirs = "5.0"
+dirs = "6.0"
 getopts = "0.2"
 ignore = "0.4"
 itertools = "0.12"
diff --git a/src/tools/rustfmt/src/chains.rs b/src/tools/rustfmt/src/chains.rs
index 034ecde068a9..2f388197ea10 100644
--- a/src/tools/rustfmt/src/chains.rs
+++ b/src/tools/rustfmt/src/chains.rs
@@ -58,7 +58,7 @@
 use std::borrow::Cow;
 use std::cmp::min;
 
-use rustc_ast::{ast, ptr};
+use rustc_ast::ast;
 use rustc_span::{BytePos, Span, symbol};
 use tracing::debug;
 
@@ -187,7 +187,7 @@ enum ChainItemKind {
     MethodCall(
         ast::PathSegment,
         Vec,
-        ThinVec>,
+        ThinVec>,
     ),
     StructField(symbol::Ident),
     TupleField(symbol::Ident, bool),
@@ -343,7 +343,7 @@ impl ChainItem {
     fn rewrite_method_call(
         method_name: symbol::Ident,
         types: &[ast::GenericArg],
-        args: &[ptr::P],
+        args: &[Box],
         span: Span,
         context: &RewriteContext<'_>,
         shape: Shape,
diff --git a/src/tools/rustfmt/src/closures.rs b/src/tools/rustfmt/src/closures.rs
index 61e148cdf188..bb10a7946b81 100644
--- a/src/tools/rustfmt/src/closures.rs
+++ b/src/tools/rustfmt/src/closures.rs
@@ -1,4 +1,4 @@
-use rustc_ast::{ast, ptr};
+use rustc_ast::ast;
 use rustc_span::Span;
 use thin_vec::thin_vec;
 use tracing::debug;
@@ -165,7 +165,7 @@ fn rewrite_closure_with_block(
     let block = ast::Block {
         stmts: thin_vec![ast::Stmt {
             id: ast::NodeId::root(),
-            kind: ast::StmtKind::Expr(ptr::P(body.clone())),
+            kind: ast::StmtKind::Expr(Box::new(body.clone())),
             span: body.span,
         }],
         id: ast::NodeId::root(),
diff --git a/src/tools/rustfmt/src/config/mod.rs b/src/tools/rustfmt/src/config/mod.rs
index 6b63108c037e..525953bf4458 100644
--- a/src/tools/rustfmt/src/config/mod.rs
+++ b/src/tools/rustfmt/src/config/mod.rs
@@ -516,7 +516,6 @@ mod test {
     #[allow(dead_code)]
     mod mock {
         use super::super::*;
-        use crate::config_option_with_style_edition_default;
         use rustfmt_config_proc_macro::config_type;
 
         #[config_type]
diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs
index 08aedff2b20d..975f9be44e4d 100644
--- a/src/tools/rustfmt/src/expr.rs
+++ b/src/tools/rustfmt/src/expr.rs
@@ -3,7 +3,7 @@ use std::cmp::min;
 
 use itertools::Itertools;
 use rustc_ast::token::{Delimiter, Lit, LitKind};
-use rustc_ast::{ForLoopKind, MatchKind, ast, ptr, token};
+use rustc_ast::{ForLoopKind, MatchKind, ast, token};
 use rustc_span::{BytePos, Span};
 use tracing::debug;
 
@@ -1368,7 +1368,7 @@ fn choose_separator_tactic(context: &RewriteContext<'_>, span: Span) -> Option,
     callee: &str,
-    args: &[ptr::P],
+    args: &[Box],
     span: Span,
     shape: Shape,
 ) -> RewriteResult {
@@ -1634,7 +1634,7 @@ fn struct_lit_can_be_aligned(fields: &[ast::ExprField], has_base: bool) -> bool
 fn rewrite_struct_lit<'a>(
     context: &RewriteContext<'_>,
     path: &ast::Path,
-    qself: &Option>,
+    qself: &Option>,
     fields: &'a [ast::ExprField],
     struct_rest: &ast::StructRest,
     attrs: &[ast::Attribute],
diff --git a/src/tools/rustfmt/src/items.rs b/src/tools/rustfmt/src/items.rs
index 7084639aca9a..ecaa4670f61d 100644
--- a/src/tools/rustfmt/src/items.rs
+++ b/src/tools/rustfmt/src/items.rs
@@ -4,8 +4,8 @@ use std::borrow::Cow;
 use std::cmp::{Ordering, max, min};
 
 use regex::Regex;
+use rustc_ast::ast;
 use rustc_ast::visit;
-use rustc_ast::{ast, ptr};
 use rustc_span::{BytePos, DUMMY_SP, Ident, Span, symbol};
 use tracing::debug;
 
@@ -725,9 +725,9 @@ impl<'a> FmtVisitor<'a> {
             .ok()
     }
 
-    fn visit_impl_items(&mut self, items: &[ptr::P]) {
+    fn visit_impl_items(&mut self, items: &[Box]) {
         if self.get_context().config.reorder_impl_items() {
-            type TyOpt = Option>;
+            type TyOpt = Option>;
             use crate::ast::AssocItemKind::*;
             let is_type = |ty: &TyOpt| opaque_ty(ty).is_none();
             let is_opaque = |ty: &TyOpt| opaque_ty(ty).is_some();
@@ -836,8 +836,7 @@ pub(crate) fn format_impl(
     let where_span_end = context.snippet_provider.opt_span_before(missing_span, "{");
     let where_clause_str = rewrite_where_clause(
         context,
-        &generics.where_clause.predicates,
-        generics.where_clause.span,
+        &generics.where_clause,
         context.config.brace_style(),
         Shape::legacy(where_budget, offset.block_only()),
         false,
@@ -934,7 +933,7 @@ pub(crate) fn format_impl(
 
 fn is_impl_single_line(
     context: &RewriteContext<'_>,
-    items: &[ptr::P],
+    items: &[Box],
     result: &str,
     where_clause_str: &str,
     item: &ast::Item,
@@ -958,20 +957,19 @@ fn format_impl_ref_and_type(
     offset: Indent,
 ) -> Option {
     let ast::Impl {
-        safety,
-        polarity,
-        defaultness,
-        constness,
-        ref generics,
-        of_trait: ref trait_ref,
-        ref self_ty,
-        ..
-    } = *iimpl;
+        generics,
+        of_trait,
+        self_ty,
+        items: _,
+    } = iimpl;
     let mut result = String::with_capacity(128);
 
     result.push_str(&format_visibility(context, &item.vis));
-    result.push_str(format_defaultness(defaultness));
-    result.push_str(format_safety(safety));
+
+    if let Some(of_trait) = of_trait.as_deref() {
+        result.push_str(format_defaultness(of_trait.defaultness));
+        result.push_str(format_safety(of_trait.safety));
+    }
 
     let shape = if context.config.style_edition() >= StyleEdition::Edition2024 {
         Shape::indented(offset + last_line_width(&result), context.config)
@@ -984,28 +982,24 @@ fn format_impl_ref_and_type(
     };
     let generics_str = rewrite_generics(context, "impl", generics, shape).ok()?;
     result.push_str(&generics_str);
-    result.push_str(format_constness_right(constness));
 
-    let polarity_str = match polarity {
-        ast::ImplPolarity::Negative(_) => "!",
-        ast::ImplPolarity::Positive => "",
-    };
-
-    let polarity_overhead;
     let trait_ref_overhead;
-    if let Some(ref trait_ref) = *trait_ref {
+    if let Some(of_trait) = of_trait.as_deref() {
+        result.push_str(format_constness_right(of_trait.constness));
+        let polarity_str = match of_trait.polarity {
+            ast::ImplPolarity::Negative(_) => "!",
+            ast::ImplPolarity::Positive => "",
+        };
         let result_len = last_line_width(&result);
         result.push_str(&rewrite_trait_ref(
             context,
-            trait_ref,
+            &of_trait.trait_ref,
             offset,
             polarity_str,
             result_len,
         )?);
-        polarity_overhead = 0; // already written
         trait_ref_overhead = " for".len();
     } else {
-        polarity_overhead = polarity_str.len();
         trait_ref_overhead = 0;
     }
 
@@ -1020,17 +1014,15 @@ fn format_impl_ref_and_type(
     } else {
         0
     };
-    let used_space =
-        last_line_width(&result) + polarity_overhead + trait_ref_overhead + curly_brace_overhead;
+    let used_space = last_line_width(&result) + trait_ref_overhead + curly_brace_overhead;
     // 1 = space before the type.
     let budget = context.budget(used_space + 1);
     if let Some(self_ty_str) = self_ty.rewrite(context, Shape::legacy(budget, offset)) {
         if !self_ty_str.contains('\n') {
-            if trait_ref.is_some() {
+            if of_trait.is_some() {
                 result.push_str(" for ");
             } else {
                 result.push(' ');
-                result.push_str(polarity_str);
             }
             result.push_str(&self_ty_str);
             return Some(result);
@@ -1042,12 +1034,10 @@ fn format_impl_ref_and_type(
     // Add indentation of one additional tab.
     let new_line_offset = offset.block_indent(context.config);
     result.push_str(&new_line_offset.to_string(context.config));
-    if trait_ref.is_some() {
+    if of_trait.is_some() {
         result.push_str("for ");
-    } else {
-        result.push_str(polarity_str);
     }
-    let budget = context.budget(last_line_width(&result) + polarity_overhead);
+    let budget = context.budget(last_line_width(&result));
     let type_offset = match context.config.indent_style() {
         IndentStyle::Visual => new_line_offset + trait_ref_overhead,
         IndentStyle::Block => new_line_offset,
@@ -1184,8 +1174,8 @@ pub(crate) fn format_trait(
     let mut result = String::with_capacity(128);
     let header = format!(
         "{}{}{}{}trait ",
-        format_constness(constness),
         format_visibility(context, &item.vis),
+        format_constness(constness),
         format_safety(safety),
         format_auto(is_auto),
     );
@@ -1233,8 +1223,7 @@ pub(crate) fn format_trait(
         let option = WhereClauseOption::snuggled(&generics_str);
         let where_clause_str = rewrite_where_clause(
             context,
-            &generics.where_clause.predicates,
-            generics.where_clause.span,
+            &generics.where_clause,
             context.config.brace_style(),
             Shape::legacy(where_budget, offset.block_only()),
             where_on_new_line,
@@ -1359,8 +1348,7 @@ impl<'a> Rewrite for TraitAliasBounds<'a> {
 
         let where_str = rewrite_where_clause(
             context,
-            &self.generics.where_clause.predicates,
-            self.generics.where_clause.span,
+            &self.generics.where_clause,
             context.config.brace_style(),
             shape,
             false,
@@ -1387,22 +1375,21 @@ impl<'a> Rewrite for TraitAliasBounds<'a> {
 
 pub(crate) fn format_trait_alias(
     context: &RewriteContext<'_>,
-    ident: symbol::Ident,
+    ta: &ast::TraitAlias,
     vis: &ast::Visibility,
-    generics: &ast::Generics,
-    generic_bounds: &ast::GenericBounds,
     shape: Shape,
 ) -> Option {
-    let alias = rewrite_ident(context, ident);
+    let alias = rewrite_ident(context, ta.ident);
     // 6 = "trait ", 2 = " ="
     let g_shape = shape.offset_left(6)?.sub_width(2)?;
-    let generics_str = rewrite_generics(context, alias, generics, g_shape).ok()?;
+    let generics_str = rewrite_generics(context, alias, &ta.generics, g_shape).ok()?;
     let vis_str = format_visibility(context, vis);
-    let lhs = format!("{vis_str}trait {generics_str} =");
+    let constness = format_constness(ta.constness);
+    let lhs = format!("{vis_str}{constness}trait {generics_str} =");
     // 1 = ";"
     let trait_alias_bounds = TraitAliasBounds {
-        generic_bounds,
-        generics,
+        generic_bounds: &ta.bounds,
+        generics: &ta.generics,
     };
     rewrite_assign_rhs(
         context,
@@ -1630,8 +1617,7 @@ fn format_tuple_struct(
             let option = WhereClauseOption::new(true, WhereClauseSpace::Newline);
             rewrite_where_clause(
                 context,
-                &generics.where_clause.predicates,
-                generics.where_clause.span,
+                &generics.where_clause,
                 context.config.brace_style(),
                 Shape::legacy(where_budget, offset.block_only()),
                 false,
@@ -1700,7 +1686,7 @@ struct TyAliasRewriteInfo<'c, 'g>(
     &'c RewriteContext<'c>,
     Indent,
     &'g ast::Generics,
-    ast::TyAliasWhereClauses,
+    &'g ast::WhereClause,
     symbol::Ident,
     Span,
 );
@@ -1721,13 +1707,13 @@ pub(crate) fn rewrite_type_alias<'a>(
         ref generics,
         ref bounds,
         ref ty,
-        where_clauses,
+        ref after_where_clause,
     } = *ty_alias_kind;
     let ty_opt = ty.as_ref();
     let rhs_hi = ty
         .as_ref()
-        .map_or(where_clauses.before.span.hi(), |ty| ty.span.hi());
-    let rw_info = &TyAliasRewriteInfo(context, indent, generics, where_clauses, ident, span);
+        .map_or(generics.where_clause.span.hi(), |ty| ty.span.hi());
+    let rw_info = &TyAliasRewriteInfo(context, indent, generics, after_where_clause, ident, span);
     let op_ty = opaque_ty(ty);
     // Type Aliases are formatted slightly differently depending on the context
     // in which they appear, whether they are opaque, and whether they are associated.
@@ -1771,11 +1757,7 @@ fn rewrite_ty(
     vis: &ast::Visibility,
 ) -> RewriteResult {
     let mut result = String::with_capacity(128);
-    let TyAliasRewriteInfo(context, indent, generics, where_clauses, ident, span) = *rw_info;
-    let (before_where_predicates, after_where_predicates) = generics
-        .where_clause
-        .predicates
-        .split_at(where_clauses.split);
+    let TyAliasRewriteInfo(context, indent, generics, after_where_clause, ident, span) = *rw_info;
     result.push_str(&format!("{}type ", format_visibility(context, vis)));
     let ident_str = rewrite_ident(context, ident);
 
@@ -1813,8 +1795,7 @@ fn rewrite_ty(
     }
     let before_where_clause_str = rewrite_where_clause(
         context,
-        before_where_predicates,
-        where_clauses.before.span,
+        &generics.where_clause,
         context.config.brace_style(),
         Shape::legacy(where_budget, indent),
         false,
@@ -1829,9 +1810,9 @@ fn rewrite_ty(
         // If there are any where clauses, add a newline before the assignment.
         // If there is a before where clause, do not indent, but if there is
         // only an after where clause, additionally indent the type.
-        if !before_where_predicates.is_empty() {
+        if !generics.where_clause.predicates.is_empty() {
             result.push_str(&indent.to_string_with_newline(context.config));
-        } else if !after_where_predicates.is_empty() {
+        } else if !after_where_clause.predicates.is_empty() {
             result.push_str(
                 &indent
                     .block_indent(context.config)
@@ -1844,7 +1825,7 @@ fn rewrite_ty(
         let comment_span = context
             .snippet_provider
             .opt_span_before(span, "=")
-            .map(|op_lo| mk_sp(where_clauses.before.span.hi(), op_lo));
+            .map(|op_lo| mk_sp(generics.where_clause.span.hi(), op_lo));
 
         let lhs = match comment_span {
             Some(comment_span)
@@ -1855,7 +1836,7 @@ fn rewrite_ty(
                         .unknown_error()?,
                 ) =>
             {
-                let comment_shape = if !before_where_predicates.is_empty() {
+                let comment_shape = if !generics.where_clause.predicates.is_empty() {
                     Shape::indented(indent, context.config)
                 } else {
                     let shape = Shape::indented(indent, context.config);
@@ -1878,7 +1859,7 @@ fn rewrite_ty(
 
         // 1 = `;` unless there's a trailing where clause
         let shape = Shape::indented(indent, context.config);
-        let shape = if after_where_predicates.is_empty() {
+        let shape = if after_where_clause.predicates.is_empty() {
             Shape::indented(indent, context.config)
                 .sub_width(1)
                 .max_width_error(shape.width, span)?
@@ -1890,12 +1871,11 @@ fn rewrite_ty(
         result
     };
 
-    if !after_where_predicates.is_empty() {
+    if !after_where_clause.predicates.is_empty() {
         let option = WhereClauseOption::new(true, WhereClauseSpace::Newline);
         let after_where_clause_str = rewrite_where_clause(
             context,
-            after_where_predicates,
-            where_clauses.after.span,
+            &after_where_clause,
             context.config.brace_style(),
             Shape::indented(indent, context.config),
             false,
@@ -2024,7 +2004,7 @@ pub(crate) struct StaticParts<'a> {
     generics: Option<&'a ast::Generics>,
     ty: &'a ast::Ty,
     mutability: ast::Mutability,
-    expr_opt: Option<&'a ptr::P>,
+    expr_opt: Option<&'a Box>,
     defaultness: Option,
     span: Span,
 }
@@ -2737,8 +2717,7 @@ fn rewrite_fn_base(
     }
     let where_clause_str = rewrite_where_clause(
         context,
-        &where_clause.predicates,
-        where_clause.span,
+        &where_clause,
         context.config.brace_style(),
         Shape::indented(indent, context.config),
         true,
@@ -3167,8 +3146,7 @@ fn rewrite_bounds_on_where_clause(
 
 fn rewrite_where_clause(
     context: &RewriteContext<'_>,
-    predicates: &[ast::WherePredicate],
-    where_span: Span,
+    where_clause: &ast::WhereClause,
     brace_style: BraceStyle,
     shape: Shape,
     on_new_line: bool,
@@ -3177,6 +3155,12 @@ fn rewrite_where_clause(
     span_end_before_where: BytePos,
     where_clause_option: WhereClauseOption,
 ) -> RewriteResult {
+    let ast::WhereClause {
+        ref predicates,
+        span: where_span,
+        has_where_token: _,
+    } = *where_clause;
+
     if predicates.is_empty() {
         return Ok(String::new());
     }
@@ -3363,8 +3347,7 @@ fn format_generics(
         }
         let where_clause_str = rewrite_where_clause(
             context,
-            &generics.where_clause.predicates,
-            generics.where_clause.span,
+            &generics.where_clause,
             brace_style,
             Shape::legacy(budget, offset.block_only()),
             true,
@@ -3609,7 +3592,7 @@ pub(crate) fn rewrite_extern_crate(
 pub(crate) fn is_mod_decl(item: &ast::Item) -> bool {
     !matches!(
         item.kind,
-        ast::ItemKind::Mod(_, _, ast::ModKind::Loaded(_, ast::Inline::Yes, _, _))
+        ast::ItemKind::Mod(_, _, ast::ModKind::Loaded(_, ast::Inline::Yes, _))
     )
 }
 
diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs
index 0ff0aad7a2d0..2e7ac90f596c 100644
--- a/src/tools/rustfmt/src/macros.rs
+++ b/src/tools/rustfmt/src/macros.rs
@@ -12,9 +12,9 @@
 use std::collections::HashMap;
 use std::panic::{AssertUnwindSafe, catch_unwind};
 
+use rustc_ast::ast;
 use rustc_ast::token::{Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{TokenStream, TokenStreamIter, TokenTree};
-use rustc_ast::{ast, ptr};
 use rustc_ast_pretty::pprust;
 use rustc_span::{BytePos, DUMMY_SP, Ident, Span, Symbol};
 use tracing::debug;
@@ -53,10 +53,10 @@ pub(crate) enum MacroPosition {
 
 #[derive(Debug)]
 pub(crate) enum MacroArg {
-    Expr(ptr::P),
-    Ty(ptr::P),
-    Pat(ptr::P),
-    Item(ptr::P),
+    Expr(Box),
+    Ty(Box),
+    Pat(Box),
+    Item(Box),
     Keyword(Ident, Span),
 }
 
diff --git a/src/tools/rustfmt/src/matches.rs b/src/tools/rustfmt/src/matches.rs
index 8f62648e576b..209cb2f92878 100644
--- a/src/tools/rustfmt/src/matches.rs
+++ b/src/tools/rustfmt/src/matches.rs
@@ -2,7 +2,7 @@
 
 use std::iter::repeat;
 
-use rustc_ast::{MatchKind, ast, ptr};
+use rustc_ast::{MatchKind, ast};
 use rustc_span::{BytePos, Span};
 use tracing::debug;
 
@@ -394,7 +394,7 @@ fn flatten_arm_body<'a>(
 
 fn rewrite_match_body(
     context: &RewriteContext<'_>,
-    body: &ptr::P,
+    body: &Box,
     pats_str: &str,
     shape: Shape,
     has_guard: bool,
@@ -569,7 +569,7 @@ fn rewrite_match_body(
 // The `if ...` guard on a match arm.
 fn rewrite_guard(
     context: &RewriteContext<'_>,
-    guard: &Option>,
+    guard: &Option>,
     shape: Shape,
     // The amount of space used up on this line for the pattern in
     // the arm (excludes offset).
diff --git a/src/tools/rustfmt/src/modules.rs b/src/tools/rustfmt/src/modules.rs
index 44c8123517c5..af9feccb32e2 100644
--- a/src/tools/rustfmt/src/modules.rs
+++ b/src/tools/rustfmt/src/modules.rs
@@ -26,7 +26,7 @@ type FileModMap<'ast> = BTreeMap>;
 #[derive(Debug, Clone)]
 pub(crate) struct Module<'a> {
     ast_mod_kind: Option>,
-    pub(crate) items: Cow<'a, ThinVec>>,
+    pub(crate) items: Cow<'a, ThinVec>>,
     inner_attr: ast::AttrVec,
     pub(crate) span: Span,
 }
@@ -35,7 +35,7 @@ impl<'a> Module<'a> {
     pub(crate) fn new(
         mod_span: Span,
         ast_mod_kind: Option>,
-        mod_items: Cow<'a, ThinVec>>,
+        mod_items: Cow<'a, ThinVec>>,
         mod_attrs: Cow<'a, ast::AttrVec>,
     ) -> Self {
         let inner_attr = mod_attrs
@@ -170,7 +170,7 @@ impl<'ast, 'psess, 'c> ModResolver<'ast, 'psess> {
     /// Visit modules defined inside macro calls.
     fn visit_mod_outside_ast(
         &mut self,
-        items: ThinVec>,
+        items: ThinVec>,
     ) -> Result<(), ModuleResolutionError> {
         for item in items {
             if is_cfg_if(&item) {
@@ -197,7 +197,7 @@ impl<'ast, 'psess, 'c> ModResolver<'ast, 'psess> {
     /// Visit modules from AST.
     fn visit_mod_from_ast(
         &mut self,
-        items: &'ast [rustc_ast::ptr::P],
+        items: &'ast [Box],
     ) -> Result<(), ModuleResolutionError> {
         for item in items {
             if is_cfg_if(item) {
@@ -316,11 +316,12 @@ impl<'ast, 'psess, 'c> ModResolver<'ast, 'psess> {
             self.directory = directory;
         }
         match (sub_mod.ast_mod_kind, sub_mod.items) {
-            (Some(Cow::Borrowed(ast::ModKind::Loaded(items, _, _, _))), _) => {
+            (Some(Cow::Borrowed(ast::ModKind::Loaded(items, _, _))), _) => {
                 self.visit_mod_from_ast(items)
             }
-            (Some(Cow::Owned(ast::ModKind::Loaded(items, _, _, _))), _)
-            | (_, Cow::Owned(items)) => self.visit_mod_outside_ast(items),
+            (Some(Cow::Owned(ast::ModKind::Loaded(items, _, _))), _) | (_, Cow::Owned(items)) => {
+                self.visit_mod_outside_ast(items)
+            }
             (_, _) => Ok(()),
         }
     }
diff --git a/src/tools/rustfmt/src/overflow.rs b/src/tools/rustfmt/src/overflow.rs
index 728adff2c7d1..bb1ebc87f67d 100644
--- a/src/tools/rustfmt/src/overflow.rs
+++ b/src/tools/rustfmt/src/overflow.rs
@@ -3,8 +3,8 @@
 use std::cmp::min;
 
 use itertools::Itertools;
+use rustc_ast::ast;
 use rustc_ast::token::Delimiter;
-use rustc_ast::{ast, ptr};
 use rustc_span::Span;
 use tracing::debug;
 
@@ -219,7 +219,7 @@ pub(crate) trait IntoOverflowableItem<'a>: Rewrite + Spanned {
     fn into_overflowable_item(&'a self) -> OverflowableItem<'a>;
 }
 
-impl<'a, T: 'a + IntoOverflowableItem<'a>> IntoOverflowableItem<'a> for ptr::P {
+impl<'a, T: 'a + IntoOverflowableItem<'a>> IntoOverflowableItem<'a> for Box {
     fn into_overflowable_item(&'a self) -> OverflowableItem<'a> {
         (**self).into_overflowable_item()
     }
diff --git a/src/tools/rustfmt/src/parse/macros/lazy_static.rs b/src/tools/rustfmt/src/parse/macros/lazy_static.rs
index cbe81004e226..8611615d1237 100644
--- a/src/tools/rustfmt/src/parse/macros/lazy_static.rs
+++ b/src/tools/rustfmt/src/parse/macros/lazy_static.rs
@@ -1,5 +1,4 @@
 use rustc_ast::ast;
-use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_parse::exp;
@@ -10,7 +9,7 @@ use crate::rewrite::RewriteContext;
 pub(crate) fn parse_lazy_static(
     context: &RewriteContext<'_>,
     ts: TokenStream,
-) -> Option, P)>> {
+) -> Option, Box)>> {
     let mut result = vec![];
     let mut parser = super::build_parser(context, ts);
     macro_rules! parse_or {
diff --git a/src/tools/rustfmt/src/parse/macros/mod.rs b/src/tools/rustfmt/src/parse/macros/mod.rs
index d7964484b261..c063990dfa0b 100644
--- a/src/tools/rustfmt/src/parse/macros/mod.rs
+++ b/src/tools/rustfmt/src/parse/macros/mod.rs
@@ -1,6 +1,6 @@
+use rustc_ast::ast;
 use rustc_ast::token::{Delimiter, NonterminalKind, NtExprKind::*, NtPatKind::*, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
-use rustc_ast::{ast, ptr};
 use rustc_parse::MACRO_ARGUMENTS;
 use rustc_parse::parser::{ForceCollect, Parser, Recovery};
 use rustc_session::parse::ParseSess;
@@ -49,26 +49,26 @@ fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option {
         Expr,
         NonterminalKind::Expr(Expr),
         |parser: &mut Parser<'b>| parser.parse_expr(),
-        |x: ptr::P| Some(x)
+        |x: Box| Some(x)
     );
     parse_macro_arg!(
         Ty,
         NonterminalKind::Ty,
         |parser: &mut Parser<'b>| parser.parse_ty(),
-        |x: ptr::P| Some(x)
+        |x: Box| Some(x)
     );
     parse_macro_arg!(
         Pat,
         NonterminalKind::Pat(PatParam { inferred: false }),
         |parser: &mut Parser<'b>| parser.parse_pat_no_top_alt(None, None),
-        |x: ptr::P| Some(x)
+        |x: ast::Pat| Some(Box::new(x))
     );
-    // `parse_item` returns `Option>`.
+    // `parse_item` returns `Option>`.
     parse_macro_arg!(
         Item,
         NonterminalKind::Item,
         |parser: &mut Parser<'b>| parser.parse_item(ForceCollect::No),
-        |x: Option>| x
+        |x: Option>| x
     );
 
     None
@@ -164,7 +164,7 @@ pub(crate) fn parse_macro_args(
 pub(crate) fn parse_expr(
     context: &RewriteContext<'_>,
     tokens: TokenStream,
-) -> Option> {
+) -> Option> {
     let mut parser = build_parser(context, tokens);
     parser.parse_expr().ok()
 }
diff --git a/src/tools/rustfmt/src/parse/parser.rs b/src/tools/rustfmt/src/parse/parser.rs
index f357aed66c2b..63c6c8c99d0d 100644
--- a/src/tools/rustfmt/src/parse/parser.rs
+++ b/src/tools/rustfmt/src/parse/parser.rs
@@ -1,8 +1,9 @@
 use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::path::{Path, PathBuf};
 
-use rustc_ast::{ast, attr, ptr};
+use rustc_ast::{ast, attr};
 use rustc_errors::Diag;
+use rustc_parse::lexer::StripTokens;
 use rustc_parse::parser::Parser as RawParser;
 use rustc_parse::{exp, new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_span::{Span, sym};
@@ -64,11 +65,14 @@ impl<'a> ParserBuilder<'a> {
         input: Input,
     ) -> Result, Vec>> {
         match input {
-            Input::File(ref file) => new_parser_from_file(psess, file, None),
+            Input::File(ref file) => {
+                new_parser_from_file(psess, file, StripTokens::ShebangAndFrontmatter, None)
+            }
             Input::Text(text) => new_parser_from_source_str(
                 psess,
                 rustc_span::FileName::Custom("stdin".to_owned()),
                 text,
+                StripTokens::ShebangAndFrontmatter,
             ),
         }
     }
@@ -102,10 +106,14 @@ impl<'a> Parser<'a> {
         psess: &'a ParseSess,
         path: &Path,
         span: Span,
-    ) -> Result<(ast::AttrVec, ThinVec>, Span), ParserError> {
+    ) -> Result<(ast::AttrVec, ThinVec>, Span), ParserError> {
         let result = catch_unwind(AssertUnwindSafe(|| {
-            let mut parser =
-                unwrap_or_emit_fatal(new_parser_from_file(psess.inner(), path, Some(span)));
+            let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
+                psess.inner(),
+                path,
+                StripTokens::ShebangAndFrontmatter,
+                Some(span),
+            ));
             match parser.parse_mod(exp!(Eof)) {
                 Ok((a, i, spans)) => Some((a, i, spans.inner_span)),
                 Err(e) => {
diff --git a/src/tools/rustfmt/src/patterns.rs b/src/tools/rustfmt/src/patterns.rs
index cb3879f4be8c..03752c371ae1 100644
--- a/src/tools/rustfmt/src/patterns.rs
+++ b/src/tools/rustfmt/src/patterns.rs
@@ -1,5 +1,4 @@
 use rustc_ast::ast::{self, BindingMode, ByRef, Pat, PatField, PatKind, RangeEnd, RangeSyntax};
-use rustc_ast::ptr;
 use rustc_span::{BytePos, Span};
 
 use crate::comment::{FindUncommented, combine_strs_with_missing_comments};
@@ -77,7 +76,7 @@ fn is_short_pattern_inner(context: &RewriteContext<'_>, pat: &ast::Pat) -> bool
 }
 
 pub(crate) struct RangeOperand<'a, T> {
-    pub operand: &'a Option>,
+    pub operand: &'a Option>,
     pub span: Span,
 }
 
@@ -135,7 +134,8 @@ impl Rewrite for Pat {
                 let mut_prefix = format_mutability(mutability).trim();
 
                 let (ref_kw, mut_infix) = match by_ref {
-                    ByRef::Yes(rmutbl) => ("ref", format_mutability(rmutbl).trim()),
+                    // FIXME(pin_ergonomics): format the pinnedness
+                    ByRef::Yes(_, rmutbl) => ("ref", format_mutability(rmutbl).trim()),
                     ByRef::No => ("", ""),
                 };
                 let id_str = rewrite_ident(context, ident);
@@ -304,7 +304,7 @@ impl Rewrite for Pat {
                 qself,
                 path,
                 fields,
-                rest == ast::PatFieldsRest::Rest,
+                matches!(rest, ast::PatFieldsRest::Rest(_)),
                 self.span,
                 context,
                 shape,
@@ -329,8 +329,8 @@ impl Rewrite for Pat {
 pub fn rewrite_range_pat(
     context: &RewriteContext<'_>,
     shape: Shape,
-    lhs: &Option>,
-    rhs: &Option>,
+    lhs: &Option>,
+    rhs: &Option>,
     end_kind: &rustc_span::source_map::Spanned,
     span: Span,
 ) -> RewriteResult {
@@ -371,7 +371,7 @@ pub fn rewrite_range_pat(
 }
 
 fn rewrite_struct_pat(
-    qself: &Option>,
+    qself: &Option>,
     path: &ast::Path,
     fields: &[ast::PatField],
     ellipsis: bool,
@@ -505,7 +505,7 @@ impl Rewrite for PatField {
 
 #[derive(Debug)]
 pub(crate) enum TuplePatField<'a> {
-    Pat(&'a ptr::P),
+    Pat(&'a ast::Pat),
     Dotdot(Span),
 }
 
@@ -562,7 +562,7 @@ pub(crate) fn can_be_overflowed_pat(
 }
 
 fn rewrite_tuple_pat(
-    pats: &[ptr::P],
+    pats: &[ast::Pat],
     path_str: Option,
     span: Span,
     context: &RewriteContext<'_>,
diff --git a/src/tools/rustfmt/src/rewrite.rs b/src/tools/rustfmt/src/rewrite.rs
index 830207097970..f0accd62d1a9 100644
--- a/src/tools/rustfmt/src/rewrite.rs
+++ b/src/tools/rustfmt/src/rewrite.rs
@@ -3,7 +3,6 @@
 use std::cell::{Cell, RefCell};
 use std::rc::Rc;
 
-use rustc_ast::ptr;
 use rustc_span::Span;
 use thiserror::Error;
 
@@ -24,7 +23,7 @@ pub(crate) trait Rewrite {
     }
 }
 
-impl Rewrite for ptr::P {
+impl Rewrite for Box {
     fn rewrite(&self, context: &RewriteContext<'_>, shape: Shape) -> Option {
         (**self).rewrite(context, shape)
     }
diff --git a/src/tools/rustfmt/src/spanned.rs b/src/tools/rustfmt/src/spanned.rs
index ac132999b62e..020651e2daac 100644
--- a/src/tools/rustfmt/src/spanned.rs
+++ b/src/tools/rustfmt/src/spanned.rs
@@ -1,6 +1,6 @@
 use std::cmp::max;
 
-use rustc_ast::{ast, ptr};
+use rustc_ast::ast;
 use rustc_span::{Span, source_map};
 
 use crate::macros::MacroArg;
@@ -12,7 +12,7 @@ pub(crate) trait Spanned {
     fn span(&self) -> Span;
 }
 
-impl Spanned for ptr::P {
+impl Spanned for Box {
     fn span(&self) -> Span {
         (**self).span()
     }
diff --git a/src/tools/rustfmt/src/types.rs b/src/tools/rustfmt/src/types.rs
index 9ee10d862703..242d8c23113c 100644
--- a/src/tools/rustfmt/src/types.rs
+++ b/src/tools/rustfmt/src/types.rs
@@ -1,7 +1,6 @@
 use std::ops::Deref;
 
 use rustc_ast::ast::{self, FnRetTy, Mutability, Term};
-use rustc_ast::ptr;
 use rustc_span::{BytePos, Pos, Span, symbol::kw};
 use tracing::debug;
 
@@ -39,7 +38,7 @@ pub(crate) enum PathContext {
 pub(crate) fn rewrite_path(
     context: &RewriteContext<'_>,
     path_context: PathContext,
-    qself: &Option>,
+    qself: &Option>,
     path: &ast::Path,
     shape: Shape,
 ) -> RewriteResult {
@@ -1100,7 +1099,7 @@ impl Rewrite for ast::TyPat {
                 }
                 Ok(s)
             }
-            ast::TyPatKind::Err(_) => Err(RewriteError::Unknown),
+            ast::TyPatKind::NotNull | ast::TyPatKind::Err(_) => Err(RewriteError::Unknown),
         }
     }
 }
@@ -1340,7 +1339,7 @@ fn join_bounds_inner(
     }
 }
 
-pub(crate) fn opaque_ty(ty: &Option>) -> Option<&ast::GenericBounds> {
+pub(crate) fn opaque_ty(ty: &Option>) -> Option<&ast::GenericBounds> {
     ty.as_ref().and_then(|t| match &t.kind {
         ast::TyKind::ImplTrait(_, bounds) => Some(bounds),
         _ => None,
diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs
index fcd475b1784f..b9950e94d0c3 100644
--- a/src/tools/rustfmt/src/utils.rs
+++ b/src/tools/rustfmt/src/utils.rs
@@ -1,10 +1,10 @@
 use std::borrow::Cow;
 
+use rustc_ast::YieldKind;
 use rustc_ast::ast::{
     self, Attribute, MetaItem, MetaItemInner, MetaItemKind, NodeId, Path, Visibility,
     VisibilityKind,
 };
-use rustc_ast::{YieldKind, ptr};
 use rustc_ast_pretty::pprust;
 use rustc_span::{BytePos, LocalExpnId, Span, Symbol, SyntaxContext, sym, symbol};
 use unicode_width::UnicodeWidthStr;
@@ -149,8 +149,8 @@ pub(crate) fn format_extern(ext: ast::Extern, explicit_abi: bool) -> Cow<'static
 }
 
 #[inline]
-// Transform `Vec>` into `Vec<&T>`
-pub(crate) fn ptr_vec_to_ref_vec(vec: &[ptr::P]) -> Vec<&T> {
+// Transform `Vec>` into `Vec<&T>`
+pub(crate) fn ptr_vec_to_ref_vec(vec: &[Box]) -> Vec<&T> {
     vec.iter().map(|x| &**x).collect::>()
 }
 
diff --git a/src/tools/rustfmt/src/visitor.rs b/src/tools/rustfmt/src/visitor.rs
index f6a9a3f2cd17..c521b497a2d9 100644
--- a/src/tools/rustfmt/src/visitor.rs
+++ b/src/tools/rustfmt/src/visitor.rs
@@ -497,16 +497,9 @@ impl<'b, 'a: 'b> FmtVisitor<'a> {
                     let rw = self.with_context(|ctx| format_trait(ctx, item, block_indent));
                     self.push_rewrite(item.span, rw);
                 }
-                ast::ItemKind::TraitAlias(ident, ref generics, ref generic_bounds) => {
+                ast::ItemKind::TraitAlias(ref ta) => {
                     let shape = Shape::indented(self.block_indent, self.config);
-                    let rw = format_trait_alias(
-                        &self.get_context(),
-                        ident,
-                        &item.vis,
-                        generics,
-                        generic_bounds,
-                        shape,
-                    );
+                    let rw = format_trait_alias(&self.get_context(), ta, &item.vis, shape);
                     self.push_rewrite(item.span, rw);
                 }
                 ast::ItemKind::ExternCrate(..) => {
@@ -874,7 +867,7 @@ impl<'b, 'a: 'b> FmtVisitor<'a> {
         !is_skip_attr(segments)
     }
 
-    fn walk_mod_items(&mut self, items: &[rustc_ast::ptr::P]) {
+    fn walk_mod_items(&mut self, items: &[Box]) {
         self.visit_items_with_reordering(&ptr_vec_to_ref_vec(items));
     }
 
@@ -942,7 +935,7 @@ impl<'b, 'a: 'b> FmtVisitor<'a> {
         let ident_str = rewrite_ident(&self.get_context(), ident).to_owned();
         self.push_str(&ident_str);
 
-        if let ast::ModKind::Loaded(ref items, ast::Inline::Yes, ref spans, _) = mod_kind {
+        if let ast::ModKind::Loaded(ref items, ast::Inline::Yes, ref spans) = mod_kind {
             let ast::ModSpans {
                 inner_span,
                 inject_use_span: _,
diff --git a/src/tools/rustfmt/tests/rustfmt/main.rs b/src/tools/rustfmt/tests/rustfmt/main.rs
index a9f58b9328e2..4a9399377d17 100644
--- a/src/tools/rustfmt/tests/rustfmt/main.rs
+++ b/src/tools/rustfmt/tests/rustfmt/main.rs
@@ -185,10 +185,11 @@ fn dont_emit_ICE() {
         "tests/target/issue-6105.rs",
     ];
 
+    let panic_re = regex::Regex::new("thread.*panicked").unwrap();
     for file in files {
         let args = [file];
         let (_stdout, stderr) = rustfmt(&args);
-        assert!(!stderr.contains("thread 'main' panicked"));
+        assert!(!panic_re.is_match(&stderr));
     }
 }
 
diff --git a/src/tools/rustfmt/tests/source/frontmatter_compact.rs b/src/tools/rustfmt/tests/source/frontmatter_compact.rs
new file mode 100644
index 000000000000..21d4c6f4b616
--- /dev/null
+++ b/src/tools/rustfmt/tests/source/frontmatter_compact.rs
@@ -0,0 +1,8 @@
+#!/usr/bin/env cargo
+---identifier
+[dependencies]
+regex = "1"
+---
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/source/frontmatter_escaped.rs b/src/tools/rustfmt/tests/source/frontmatter_escaped.rs
new file mode 100644
index 000000000000..0d0263775668
--- /dev/null
+++ b/src/tools/rustfmt/tests/source/frontmatter_escaped.rs
@@ -0,0 +1,13 @@
+#!/usr/bin/env cargo
+------------
+package.description = """
+Header
+-----
+
+Body
+"""
+------------
+
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/source/frontmatter_spaced.rs b/src/tools/rustfmt/tests/source/frontmatter_spaced.rs
new file mode 100644
index 000000000000..ee0bb81705c4
--- /dev/null
+++ b/src/tools/rustfmt/tests/source/frontmatter_spaced.rs
@@ -0,0 +1,16 @@
+#!/usr/bin/env cargo
+
+
+---   identifier
+[dependencies]
+regex = "1"
+
+---
+
+
+
+
+
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/source/negative-impl.rs b/src/tools/rustfmt/tests/source/negative-impl.rs
index da242d4f3dca..e8f9508e6563 100644
--- a/src/tools/rustfmt/tests/source/negative-impl.rs
+++ b/src/tools/rustfmt/tests/source/negative-impl.rs
@@ -1,7 +1,3 @@
 impl ! Display for JoinHandle { }
 
-impl ! Box < JoinHandle > { }
-
 impl ! std :: fmt :: Display for JoinHandle < T : std :: future :: Future + std :: marker :: Send + std :: marker :: Sync > { }
-
-impl ! JoinHandle < T : std :: future :: Future < Output > + std :: marker :: Send + std :: marker :: Sync + 'static > + 'static { }
diff --git a/src/tools/rustfmt/tests/source/trait.rs b/src/tools/rustfmt/tests/source/trait.rs
index b6db9e1590d4..1970646f7770 100644
--- a/src/tools/rustfmt/tests/source/trait.rs
+++ b/src/tools/rustfmt/tests/source/trait.rs
@@ -181,3 +181,5 @@ trait Visible {
     pub fn f();
     pub fn g() {}
 }
+
+const trait Foomp = Hash;
\ No newline at end of file
diff --git a/src/tools/rustfmt/tests/target/const_trait.rs b/src/tools/rustfmt/tests/target/const_trait.rs
new file mode 100644
index 000000000000..c4c88b17fc50
--- /dev/null
+++ b/src/tools/rustfmt/tests/target/const_trait.rs
@@ -0,0 +1,5 @@
+#![feature(trait_alias, const_trait_impl)]
+
+const trait Bar {}
+
+const trait Foo = Bar;
diff --git a/src/tools/rustfmt/tests/target/frontmatter_compact.rs b/src/tools/rustfmt/tests/target/frontmatter_compact.rs
new file mode 100644
index 000000000000..21d4c6f4b616
--- /dev/null
+++ b/src/tools/rustfmt/tests/target/frontmatter_compact.rs
@@ -0,0 +1,8 @@
+#!/usr/bin/env cargo
+---identifier
+[dependencies]
+regex = "1"
+---
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/target/frontmatter_escaped.rs b/src/tools/rustfmt/tests/target/frontmatter_escaped.rs
new file mode 100644
index 000000000000..0d0263775668
--- /dev/null
+++ b/src/tools/rustfmt/tests/target/frontmatter_escaped.rs
@@ -0,0 +1,13 @@
+#!/usr/bin/env cargo
+------------
+package.description = """
+Header
+-----
+
+Body
+"""
+------------
+
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/target/frontmatter_spaced.rs b/src/tools/rustfmt/tests/target/frontmatter_spaced.rs
new file mode 100644
index 000000000000..ee0bb81705c4
--- /dev/null
+++ b/src/tools/rustfmt/tests/target/frontmatter_spaced.rs
@@ -0,0 +1,16 @@
+#!/usr/bin/env cargo
+
+
+---   identifier
+[dependencies]
+regex = "1"
+
+---
+
+
+
+
+
+#![feature(frontmatter)]
+
+fn main() {}
diff --git a/src/tools/rustfmt/tests/target/negative-impl.rs b/src/tools/rustfmt/tests/target/negative-impl.rs
index 16ce7e26a99a..bb53048dbc62 100644
--- a/src/tools/rustfmt/tests/target/negative-impl.rs
+++ b/src/tools/rustfmt/tests/target/negative-impl.rs
@@ -1,14 +1,6 @@
 impl !Display for JoinHandle {}
 
-impl !Box {}
-
 impl !std::fmt::Display
     for JoinHandle
 {
 }
-
-impl
-    !JoinHandle + std::marker::Send + std::marker::Sync + 'static>
-        + 'static
-{
-}
diff --git a/src/tools/rustfmt/tests/target/trait.rs b/src/tools/rustfmt/tests/target/trait.rs
index 7f067991b267..7a65b13d629e 100644
--- a/src/tools/rustfmt/tests/target/trait.rs
+++ b/src/tools/rustfmt/tests/target/trait.rs
@@ -218,3 +218,5 @@ trait Visible {
     pub fn f();
     pub fn g() {}
 }
+
+const trait Foomp = Hash;
diff --git a/src/tools/tidy/Readme.md b/src/tools/tidy/Readme.md
new file mode 100644
index 000000000000..fc9dc6350e92
--- /dev/null
+++ b/src/tools/tidy/Readme.md
@@ -0,0 +1,112 @@
+# Tidy
+Tidy is the Rust project's custom internal linter and a crucial part of our testing and continuous integration (CI) infrastructure. It is designed to enforce a consistent style and formatting across the entire codebase, but its role extends beyond simple linting. Tidy also helps with infrastructure, policy, and documentation, ensuring the project remains organized, functional, and... tidy.
+
+This document will cover how to use tidy, the specific checks tidy performs, and using tidy directives to manage its behavior. By understanding and utilizing tidy, you can help us maintain the high standards of the Rust project.
+## Tidy Checks
+### Style and Code Quality
+These lints focus on enforcing consistent formatting, style, and general code health.
+* [`alphabetical`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/alphabetical/index.html): Checks that lists are sorted alphabetically
+* [`style`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/style/index.html): Check to enforce various stylistic guidelines on the Rust codebase.
+* [`filenames`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/filenames/index.html): Check to prevent invalid characters in file names.
+* [`pal`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/pal/index.html): Check to enforce rules about platform-specific code in std.
+* [`target_policy`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/target_policy/index.html): Check for target tier policy compliance.
+* [`error_codes`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/error_codes/index.html): Check to ensure error codes are properly documented and tested.
+
+### Infrastructure
+These checks focus on the integrity of the project's dependencies, internal tools, and documentation.
+* [`bins`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/bins/index.html): Prevent stray binaries from being checked into the source tree.
+* [`fluent_alphabetical`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/fluent_alphabetical/index.html) / [`fluent_period`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/fluent_period/index.html) / [`fluent_used`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/fluent_used/index.html): Various checks related to [Fluent](https://github.com/projectfluent) for localization and natural language translation.
+* [`deps`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/deps/index.html) / [`extdeps`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/extdeps/index.html): Check for valid licenses and sources for external dependencies.
+* [`gcc_submodule`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/gcc_submodule/index.html): Check that the `src/gcc` submodule version is valid.
+* [`triagebot`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/triagebot/index.html): Check to ensure paths mentioned in `triagebot.toml` exist in the project.
+* [`x_version`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/x_version/index.html): Validates the current version of the `x` tool.
+
+* [`edition`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/edition/index.html) / [`features`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/features/index.html): Check for a valid Rust edition and proper ordering of unstable features.
+* [`rustdoc_css_themes`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/rustdoc_css_themes/index.html) / [`rustdoc_templates`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/rustdoc_templates/index.html): Verify that Rust documentation templates and themes are correct.
+* [`unstable_book`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/unstable_book/index.html): Synchronizes the unstable book with unstable features.
+### Testing
+These checks ensure that tests are correctly structured, cleaned up, and free of common errors.
+* [`tests_placement`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/tests_placement/index.html) / [`unit_tests`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/unit_tests/index.html): Verify that tests are located in the correct directories and are not using improper attributes.
+* [`known_bug`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/known_bug/index.html) / [`unknown_revision`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/unknown_revision/index.html): Ensure that test directives and annotations are used correctly.
+* [`debug_artifacts`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/debug_artifacts/index.html) / [`mir_opt_tests`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/mir_opt_tests/index.html): Prevent unnecessary artifacts and stale files in test directories.
+* [`tests_revision_unpaired_stdout_stderr`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/tests_revision_unpaired_stdout_stderr/index.html) / [`ui_tests`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/ui_tests/index.html): Check for unpaired or stray test output files.
+* [`target_specific_tests`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/target_specific_tests/index.html): Check to ensure that all target specific tests (those that require a --target flag) also require the pre-requisite LLVM components to run.
+* [`rustdoc_gui_tests`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/rustdoc_gui_tests/index.html): Checks that rustdoc gui tests start with a small description
+* [`rustdoc_json`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/rustdoc_json/index.html): Verify that `FORMAT_VERSION` is in sync with `rust-json-types`.
+## Using Tidy
+
+Tidy is used in a number of different ways.
+* Every time `./x test` is used tidy will run automatically.
+
+* On every pull request, tidy will run automatically during CI checks.
+* Optionally, with the use of git-hooks, tidy can run locally on every push. This can be setup with `./x setup`. See the [rustc-dev-guide](https://rustc-dev-guide.rust-lang.org/building/suggested.html#installing-a-pre-push-hook) for more information.
+
+You can run tidy manually with:
+
+`./x test tidy`
+
+To first run the relevant formatter and then run tidy you can add `--bless`.
+
+`./x test tidy --bless`
+### Extra Checks
+[`extra_checks`](https://doc.rust-lang.org/nightly/nightly-rustc/tidy/extra_checks/index.html) are optional checks primarily focused on other file types and programming languages.
+
+Example usage:
+
+`./x test tidy --extra-checks=py,cpp,js,spellcheck`
+
+All options for `--extra-checks`:
+* `cpp`, `cpp:fmt`
+* `py`, `py:lint`, `py:fmt`
+* `js`, `js:lint`, `js:fmt`, `js:typecheck`
+* `shell`, `shell:lint`
+* `spellcheck`
+
+Default values for tidy's `extra-checks` can be set in `bootstrap.toml`. For example, `build.tidy-extra-checks = "js,py"`.
+
+Any argument without a suffix (eg. `py` or `js`) will include all possible checks. For example, `--extra-checks=js` is the same as `extra-checks=js:lint,js:fmt,js:typecheck`.
+
+Any argument can be prefixed with `auto:` to only run if relevant files are modified (eg. `--extra-checks=auto:py`).
+
+A specific configuration file or folder can be passed to tidy after a double dash (`--extra-checks=py -- foo.py`)
+
+## Tidy Directives
+
+Tidy directives are special comments that help tidy operate.
+
+Tidy directives can be used in the following types of comments:
+* `// `
+* `# `
+* `/* {...} */`
+* ``
+
+You might find yourself needing to ignore a specific tidy style check and can do so with:
+* `ignore-tidy-cr`
+* `ignore-tidy-undocumented-unsafe`
+* `ignore-tidy-tab`
+* `ignore-tidy-linelength`
+* `ignore-tidy-filelength`
+
+* `ignore-tidy-end-whitespace`
+* `ignore-tidy-trailing-newlines`
+* `ignore-tidy-leading-newlines`
+* `ignore-tidy-copyright`
+* `ignore-tidy-dbg`
+* `ignore-tidy-odd-backticks`
+* `ignore-tidy-todo`
+
+Some checks, like `alphabetical`, require a tidy directive to use:
+```
+// tidy-alphabetical-start
+fn aaa() {}
+fn eee() {}
+fn z() {}
+// tidy-alphabetical-end
+```
+While not exactly a tidy directive, // TODO will fail tidy and make sure you can't merge a PR with unfinished work.
+
+### Test Specific Directives
+
+`target-specific-tests` can be ignored with `// ignore-tidy-target-specific-tests`
+
+Tidy's `unknown_revision` check can be suppressed by adding the revision name to `//@ unused-revision-names:{revision}` or with `//@ unused-revision-names:*`.
diff --git a/src/tools/tidy/src/alphabetical.rs b/src/tools/tidy/src/alphabetical.rs
index 9ddce7251069..3845e2269e9b 100644
--- a/src/tools/tidy/src/alphabetical.rs
+++ b/src/tools/tidy/src/alphabetical.rs
@@ -24,6 +24,7 @@ use std::fmt::Display;
 use std::iter::Peekable;
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 #[cfg(test)]
@@ -43,8 +44,7 @@ const END_MARKER: &str = "tidy-alphabetical-end";
 fn check_section<'a>(
     file: impl Display,
     lines: impl Iterator,
-    err: &mut dyn FnMut(&str) -> std::io::Result<()>,
-    bad: &mut bool,
+    check: &mut RunningCheck,
 ) {
     let mut prev_line = String::new();
     let mut first_indent = None;
@@ -56,12 +56,10 @@ fn check_section<'a>(
         }
 
         if line.contains(START_MARKER) {
-            tidy_error_ext!(
-                err,
-                bad,
+            check.error(format!(
                 "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`",
                 idx + 1
-            );
+            ));
             return;
         }
 
@@ -104,45 +102,44 @@ fn check_section<'a>(
         let prev_line_trimmed_lowercase = prev_line.trim_start_matches(' ');
 
         if version_sort(trimmed_line, prev_line_trimmed_lowercase).is_lt() {
-            tidy_error_ext!(err, bad, "{file}:{}: line not in alphabetical order", idx + 1);
+            check.error(format!("{file}:{}: line not in alphabetical order", idx + 1));
         }
 
         prev_line = line;
     }
 
-    tidy_error_ext!(err, bad, "{file}: reached end of file expecting `{END_MARKER}`")
+    check.error(format!("{file}: reached end of file expecting `{END_MARKER}`"));
 }
 
 fn check_lines<'a>(
     file: &impl Display,
     mut lines: impl Iterator,
-    err: &mut dyn FnMut(&str) -> std::io::Result<()>,
-    bad: &mut bool,
+    check: &mut RunningCheck,
 ) {
     while let Some((idx, line)) = lines.next() {
         if line.contains(END_MARKER) {
-            tidy_error_ext!(
-                err,
-                bad,
+            check.error(format!(
                 "{file}:{} found `{END_MARKER}` expecting `{START_MARKER}`",
                 idx + 1
-            )
+            ));
         }
 
         if line.contains(START_MARKER) {
-            check_section(file, &mut lines, err, bad);
+            check_section(file, &mut lines, check);
         }
     }
 }
 
-pub fn check(path: &Path, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("alphabetical").path(path));
+
     let skip =
         |path: &_, _is_dir| filter_dirs(path) || path.ends_with("tidy/src/alphabetical/tests.rs");
 
     walk(path, skip, &mut |entry, contents| {
         let file = &entry.path().display();
         let lines = contents.lines().enumerate();
-        check_lines(file, lines, &mut crate::tidy_error, bad)
+        check_lines(file, lines, &mut check)
     });
 }
 
diff --git a/src/tools/tidy/src/alphabetical/tests.rs b/src/tools/tidy/src/alphabetical/tests.rs
index 4d05bc33cedc..3e0dd798ab9d 100644
--- a/src/tools/tidy/src/alphabetical/tests.rs
+++ b/src/tools/tidy/src/alphabetical/tests.rs
@@ -1,19 +1,22 @@
-use std::io::Write;
-use std::str::from_utf8;
+use std::path::Path;
 
-use super::*;
+use crate::alphabetical::check_lines;
+use crate::diagnostics::{TidyCtx, TidyFlags};
 
 #[track_caller]
 fn test(lines: &str, name: &str, expected_msg: &str, expected_bad: bool) {
-    let mut actual_msg = Vec::new();
-    let mut actual_bad = false;
-    let mut err = |args: &_| {
-        write!(&mut actual_msg, "{args}")?;
-        Ok(())
-    };
-    check_lines(&name, lines.lines().enumerate(), &mut err, &mut actual_bad);
-    assert_eq!(expected_msg, from_utf8(&actual_msg).unwrap());
-    assert_eq!(expected_bad, actual_bad);
+    let tidy_ctx = TidyCtx::new(Path::new("/"), false, TidyFlags::default());
+    let mut check = tidy_ctx.start_check("alphabetical-test");
+    check_lines(&name, lines.lines().enumerate(), &mut check);
+
+    assert_eq!(expected_bad, check.is_bad());
+    let errors = check.get_errors();
+    if expected_bad {
+        assert_eq!(errors.len(), 1);
+        assert_eq!(expected_msg, errors[0]);
+    } else {
+        assert!(errors.is_empty());
+    }
 }
 
 #[track_caller]
diff --git a/src/tools/tidy/src/bins.rs b/src/tools/tidy/src/bins.rs
index a18f549844b8..2dcbd8e9b38a 100644
--- a/src/tools/tidy/src/bins.rs
+++ b/src/tools/tidy/src/bins.rs
@@ -12,11 +12,13 @@ pub use os_impl::*;
 mod os_impl {
     use std::path::Path;
 
+    use crate::diagnostics::TidyCtx;
+
     pub fn check_filesystem_support(_sources: &[&Path], _output: &Path) -> bool {
         return false;
     }
 
-    pub fn check(_path: &Path, _bad: &mut bool) {}
+    pub fn check(_path: &Path, _tidy_ctx: TidyCtx) {}
 }
 
 #[cfg(unix)]
@@ -36,6 +38,8 @@ mod os_impl {
 
     use FilesystemSupport::*;
 
+    use crate::diagnostics::TidyCtx;
+
     fn is_executable(path: &Path) -> std::io::Result {
         Ok(path.metadata()?.mode() & 0o111 != 0)
     }
@@ -106,14 +110,16 @@ mod os_impl {
     }
 
     #[cfg(unix)]
-    pub fn check(path: &Path, bad: &mut bool) {
+    pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+        let mut check = tidy_ctx.start_check("bins");
+
         use std::ffi::OsStr;
 
         const ALLOWED: &[&str] = &["configure", "x"];
 
         for p in RI_EXCLUSION_LIST {
             if !path.join(Path::new(p)).exists() {
-                tidy_error!(bad, "rust-installer test bins missed: {p}");
+                check.error(format!("rust-installer test bins missed: {p}"));
             }
         }
 
@@ -153,7 +159,7 @@ mod os_impl {
                         });
                     let path_bytes = rel_path.as_os_str().as_bytes();
                     if output.status.success() && output.stdout.starts_with(path_bytes) {
-                        tidy_error!(bad, "binary checked into source: {}", file.display());
+                        check.error(format!("binary checked into source: {}", file.display()));
                     }
                 }
             },
diff --git a/src/tools/tidy/src/debug_artifacts.rs b/src/tools/tidy/src/debug_artifacts.rs
index 645534cc8273..a1a0f37f7844 100644
--- a/src/tools/tidy/src/debug_artifacts.rs
+++ b/src/tools/tidy/src/debug_artifacts.rs
@@ -2,24 +2,25 @@
 
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::{filter_dirs, filter_not_rust, walk};
 
 const GRAPHVIZ_POSTFLOW_MSG: &str = "`borrowck_graphviz_postflow` attribute in test";
 
-pub fn check(test_dir: &Path, bad: &mut bool) {
+pub fn check(test_dir: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("debug_artifacts").path(test_dir));
+
     walk(
         test_dir,
         |path, _is_dir| filter_dirs(path) || filter_not_rust(path),
         &mut |entry, contents| {
             for (i, line) in contents.lines().enumerate() {
                 if line.contains("borrowck_graphviz_postflow") {
-                    tidy_error!(
-                        bad,
-                        "{}:{}: {}",
+                    check.error(format!(
+                        "{}:{}: {GRAPHVIZ_POSTFLOW_MSG}",
                         entry.path().display(),
-                        i + 1,
-                        GRAPHVIZ_POSTFLOW_MSG
-                    );
+                        i + 1
+                    ));
                 }
             }
         },
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index 858b058cb7d0..6a1721df1642 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -9,6 +9,8 @@ use build_helper::ci::CiEnv;
 use cargo_metadata::semver::Version;
 use cargo_metadata::{Metadata, Package, PackageId};
 
+use crate::diagnostics::{RunningCheck, TidyCtx};
+
 #[path = "../../../bootstrap/src/utils/proc_macro_deps.rs"]
 mod proc_macro_deps;
 
@@ -17,71 +19,165 @@ mod proc_macro_deps;
 #[rustfmt::skip]
 const LICENSES: &[&str] = &[
     // tidy-alphabetical-start
-    "(MIT OR Apache-2.0) AND Unicode-3.0",                 // unicode_ident (1.0.14)
-    "(MIT OR Apache-2.0) AND Unicode-DFS-2016",            // unicode_ident (1.0.12)
     "0BSD OR MIT OR Apache-2.0",                           // adler2 license
-    "0BSD",
     "Apache-2.0 / MIT",
     "Apache-2.0 OR ISC OR MIT",
     "Apache-2.0 OR MIT",
     "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT", // wasi license
-    "Apache-2.0",
     "Apache-2.0/MIT",
     "BSD-2-Clause OR Apache-2.0 OR MIT",                   // zerocopy
+    "BSD-2-Clause OR MIT OR Apache-2.0",
+    "BSD-3-Clause/MIT",
+    "CC0-1.0 OR MIT-0 OR Apache-2.0",
     "ISC",
     "MIT / Apache-2.0",
     "MIT AND (MIT OR Apache-2.0)",
     "MIT AND Apache-2.0 WITH LLVM-exception AND (MIT OR Apache-2.0)", // compiler-builtins
-    "MIT OR Apache-2.0 OR LGPL-2.1-or-later",              // r-efi, r-efi-alloc
+    "MIT OR Apache-2.0 OR BSD-1-Clause",
+    "MIT OR Apache-2.0 OR LGPL-2.1-or-later",              // r-efi, r-efi-alloc; LGPL is not acceptable, but we use it under MIT OR Apache-2.0
     "MIT OR Apache-2.0 OR Zlib",                           // tinyvec_macros
     "MIT OR Apache-2.0",
     "MIT OR Zlib OR Apache-2.0",                           // miniz_oxide
     "MIT",
     "MIT/Apache-2.0",
-    "Unicode-3.0",                                         // icu4x
-    "Unicode-DFS-2016",                                    // tinystr
     "Unlicense OR MIT",
     "Unlicense/MIT",
+    // tidy-alphabetical-end
+];
+
+/// These are licenses that are allowed for rustc, tools, etc. But not for the runtime!
+#[rustfmt::skip]
+const LICENSES_TOOLS: &[&str] = &[
+    // tidy-alphabetical-start
+    "(Apache-2.0 OR MIT) AND BSD-3-Clause",
+    "(MIT OR Apache-2.0) AND Unicode-3.0",                 // unicode_ident (1.0.14)
+    "(MIT OR Apache-2.0) AND Unicode-DFS-2016",            // unicode_ident (1.0.12)
+    "0BSD",
+    "Apache-2.0 AND ISC",
+    "Apache-2.0 OR BSL-1.0",  // BSL is not acceptable, but we use it under Apache-2.0
+    "Apache-2.0 WITH LLVM-exception",
+    "Apache-2.0",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception",
+    "CC0-1.0",
+    "Unicode-3.0",                                         // icu4x
+    "Unicode-DFS-2016",                                    // tinystr
     "Zlib OR Apache-2.0 OR MIT",                           // tinyvec
+    "Zlib",
     // tidy-alphabetical-end
 ];
 
 type ExceptionList = &'static [(&'static str, &'static str)];
 
+#[derive(Clone, Copy)]
+pub(crate) struct WorkspaceInfo<'a> {
+    /// Path to the directory containing the workspace root Cargo.toml file.
+    pub(crate) path: &'a str,
+    /// The list of license exceptions.
+    pub(crate) exceptions: ExceptionList,
+    /// Optionally:
+    /// * A list of crates for which dependencies need to be explicitly allowed.
+    /// * The list of allowed dependencies.
+    /// * The source code location of the allowed dependencies list
+    crates_and_deps: Option<(&'a [&'a str], &'a [&'a str], ListLocation)>,
+    /// Submodules required for the workspace
+    pub(crate) submodules: &'a [&'a str],
+}
+
 /// The workspaces to check for licensing and optionally permitted dependencies.
-///
-/// Each entry consists of a tuple with the following elements:
-///
-/// * The path to the workspace root Cargo.toml file.
-/// * The list of license exceptions.
-/// * Optionally a tuple of:
-///     * A list of crates for which dependencies need to be explicitly allowed.
-///     * The list of allowed dependencies.
-/// * Submodules required for the workspace.
 // FIXME auto detect all cargo workspaces
-pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>, &[&str])] = &[
+pub(crate) const WORKSPACES: &[WorkspaceInfo<'static>] = &[
     // The root workspace has to be first for check_rustfix to work.
-    (".", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES)), &[]),
-    ("library", EXCEPTIONS_STDLIB, Some((&["sysroot"], PERMITTED_STDLIB_DEPENDENCIES)), &[]),
-    // Outside of the alphabetical section because rustfmt formats it using multiple lines.
-    (
-        "compiler/rustc_codegen_cranelift",
-        EXCEPTIONS_CRANELIFT,
-        Some((&["rustc_codegen_cranelift"], PERMITTED_CRANELIFT_DEPENDENCIES)),
-        &[],
-    ),
-    // tidy-alphabetical-start
-    ("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None, &[]),
-    ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None, &[]),
-    ("src/tools/cargo", EXCEPTIONS_CARGO, None, &["src/tools/cargo"]),
-    //("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored
-    //("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored
-    ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None, &[]),
-    ("src/tools/rustbook", EXCEPTIONS_RUSTBOOK, None, &["src/doc/book", "src/doc/reference"]),
-    ("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None, &["src/tools/rustc-perf"]),
-    ("src/tools/test-float-parse", EXCEPTIONS, None, &[]),
-    ("tests/run-make/uefi-qemu/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None, &[]),
-    // tidy-alphabetical-end
+    WorkspaceInfo {
+        path: ".",
+        exceptions: EXCEPTIONS,
+        crates_and_deps: Some((
+            &["rustc-main"],
+            PERMITTED_RUSTC_DEPENDENCIES,
+            PERMITTED_RUSTC_DEPS_LOCATION,
+        )),
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "library",
+        exceptions: EXCEPTIONS_STDLIB,
+        crates_and_deps: Some((
+            &["sysroot"],
+            PERMITTED_STDLIB_DEPENDENCIES,
+            PERMITTED_STDLIB_DEPS_LOCATION,
+        )),
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "compiler/rustc_codegen_cranelift",
+        exceptions: EXCEPTIONS_CRANELIFT,
+        crates_and_deps: Some((
+            &["rustc_codegen_cranelift"],
+            PERMITTED_CRANELIFT_DEPENDENCIES,
+            PERMITTED_CRANELIFT_DEPS_LOCATION,
+        )),
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "compiler/rustc_codegen_gcc",
+        exceptions: EXCEPTIONS_GCC,
+        crates_and_deps: None,
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "src/bootstrap",
+        exceptions: EXCEPTIONS_BOOTSTRAP,
+        crates_and_deps: None,
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "src/tools/cargo",
+        exceptions: EXCEPTIONS_CARGO,
+        crates_and_deps: None,
+        submodules: &["src/tools/cargo"],
+    },
+    // FIXME uncomment once all deps are vendored
+    //  WorkspaceInfo {
+    //      path: "src/tools/miri/test-cargo-miri",
+    //      crates_and_deps: None
+    //      submodules: &[],
+    //  },
+    // WorkspaceInfo {
+    //      path: "src/tools/miri/test_dependencies",
+    //      crates_and_deps: None,
+    //      submodules: &[],
+    //  }
+    WorkspaceInfo {
+        path: "src/tools/rust-analyzer",
+        exceptions: EXCEPTIONS_RUST_ANALYZER,
+        crates_and_deps: None,
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "src/tools/rustbook",
+        exceptions: EXCEPTIONS_RUSTBOOK,
+        crates_and_deps: None,
+        submodules: &["src/doc/book", "src/doc/reference"],
+    },
+    WorkspaceInfo {
+        path: "src/tools/rustc-perf",
+        exceptions: EXCEPTIONS_RUSTC_PERF,
+        crates_and_deps: None,
+        submodules: &["src/tools/rustc-perf"],
+    },
+    WorkspaceInfo {
+        path: "src/tools/test-float-parse",
+        exceptions: EXCEPTIONS,
+        crates_and_deps: None,
+        submodules: &[],
+    },
+    WorkspaceInfo {
+        path: "tests/run-make-cargo/uefi-qemu/uefi_qemu_test",
+        exceptions: EXCEPTIONS_UEFI_QEMU_TEST,
+        crates_and_deps: None,
+        submodules: &[],
+    },
 ];
 
 /// These are exceptions to Rust's permissive licensing policy, and
@@ -91,19 +187,8 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>,
 #[rustfmt::skip]
 const EXCEPTIONS: ExceptionList = &[
     // tidy-alphabetical-start
-    ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
-    ("arrayref", "BSD-2-Clause"),                            // rustc
-    ("blake3", "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception"),  // rustc
     ("colored", "MPL-2.0"),                                  // rustfmt
-    ("constant_time_eq", "CC0-1.0 OR MIT-0 OR Apache-2.0"),  // rustc
-    ("dissimilar", "Apache-2.0"),                            // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
-    ("fluent-langneg", "Apache-2.0"),                        // rustc (fluent translations)
-    ("foldhash", "Zlib"),                                    // rustc
     ("option-ext", "MPL-2.0"),                               // cargo-miri (via `directories`)
-    ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"),     // rustc (license is the same as LLVM uses)
-    ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0                       // cargo/... (because of serde)
-    ("self_cell", "Apache-2.0"),                             // rustc (fluent translations)
-    ("wasi-preview1-component-adapter-provider", "Apache-2.0 WITH LLVM-exception"), // rustc
     // tidy-alphabetical-end
 ];
 
@@ -120,57 +205,22 @@ const EXCEPTIONS_STDLIB: ExceptionList = &[
 
 const EXCEPTIONS_CARGO: ExceptionList = &[
     // tidy-alphabetical-start
-    ("arrayref", "BSD-2-Clause"),
     ("bitmaps", "MPL-2.0+"),
-    ("blake3", "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception"),
-    ("ciborium", "Apache-2.0"),
-    ("ciborium-io", "Apache-2.0"),
-    ("ciborium-ll", "Apache-2.0"),
-    ("constant_time_eq", "CC0-1.0 OR MIT-0 OR Apache-2.0"),
-    ("dunce", "CC0-1.0 OR MIT-0 OR Apache-2.0"),
-    ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"),
-    ("fiat-crypto", "MIT OR Apache-2.0 OR BSD-1-Clause"),
-    ("foldhash", "Zlib"),
     ("im-rc", "MPL-2.0+"),
-    ("libz-rs-sys", "Zlib"),
-    ("normalize-line-endings", "Apache-2.0"),
-    ("openssl", "Apache-2.0"),
-    ("ring", "Apache-2.0 AND ISC"),
-    ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0
-    ("similar", "Apache-2.0"),
     ("sized-chunks", "MPL-2.0+"),
-    ("subtle", "BSD-3-Clause"),
-    ("supports-hyperlinks", "Apache-2.0"),
-    ("unicode-bom", "Apache-2.0"),
-    ("zlib-rs", "Zlib"),
     // tidy-alphabetical-end
 ];
 
 const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[
     // tidy-alphabetical-start
-    ("dissimilar", "Apache-2.0"),
-    ("foldhash", "Zlib"),
-    ("notify", "CC0-1.0"),
     ("option-ext", "MPL-2.0"),
-    ("pulldown-cmark-to-cmark", "Apache-2.0"),
-    ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"),
-    ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0
-    ("scip", "Apache-2.0"),
     // tidy-alphabetical-end
 ];
 
 const EXCEPTIONS_RUSTC_PERF: ExceptionList = &[
     // tidy-alphabetical-start
-    ("alloc-no-stdlib", "BSD-3-Clause"),
-    ("alloc-stdlib", "BSD-3-Clause"),
-    ("brotli", "BSD-3-Clause/MIT"),
-    ("brotli-decompressor", "BSD-3-Clause/MIT"),
-    ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"),
     ("inferno", "CDDL-1.0"),
     ("option-ext", "MPL-2.0"),
-    ("ryu", "Apache-2.0 OR BSL-1.0"),
-    ("snap", "BSD-3-Clause"),
-    ("subtle", "BSD-3-Clause"),
     // tidy-alphabetical-end
 ];
 
@@ -180,36 +230,10 @@ const EXCEPTIONS_RUSTBOOK: ExceptionList = &[
     ("cssparser-macros", "MPL-2.0"),
     ("dtoa-short", "MPL-2.0"),
     ("mdbook", "MPL-2.0"),
-    ("ryu", "Apache-2.0 OR BSL-1.0"),
     // tidy-alphabetical-end
 ];
 
-const EXCEPTIONS_CRANELIFT: ExceptionList = &[
-    // tidy-alphabetical-start
-    ("cranelift-assembler-x64", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-assembler-x64-meta", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-bitset", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-codegen", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-codegen-meta", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-codegen-shared", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-control", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-entity", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-frontend", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-isle", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-jit", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-module", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-native", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-object", "Apache-2.0 WITH LLVM-exception"),
-    ("cranelift-srcgen", "Apache-2.0 WITH LLVM-exception"),
-    ("foldhash", "Zlib"),
-    ("mach2", "BSD-2-Clause OR MIT OR Apache-2.0"),
-    ("regalloc2", "Apache-2.0 WITH LLVM-exception"),
-    ("target-lexicon", "Apache-2.0 WITH LLVM-exception"),
-    ("wasmtime-jit-icache-coherence", "Apache-2.0 WITH LLVM-exception"),
-    ("wasmtime-math", "Apache-2.0 WITH LLVM-exception"),
-    // tidy-alphabetical-end
-];
+const EXCEPTIONS_CRANELIFT: ExceptionList = &[];
 
 const EXCEPTIONS_GCC: ExceptionList = &[
     // tidy-alphabetical-start
@@ -218,15 +242,24 @@ const EXCEPTIONS_GCC: ExceptionList = &[
     // tidy-alphabetical-end
 ];
 
-const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[
-    ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde. BSL is not acceptble, but we use it under Apache-2.0
-];
+const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[];
 
-const EXCEPTIONS_UEFI_QEMU_TEST: ExceptionList = &[
-    ("r-efi", "MIT OR Apache-2.0 OR LGPL-2.1-or-later"), // LGPL is not acceptable, but we use it under MIT OR Apache-2.0
-];
+const EXCEPTIONS_UEFI_QEMU_TEST: ExceptionList = &[];
 
-const PERMITTED_DEPS_LOCATION: &str = concat!(file!(), ":", line!());
+#[derive(Clone, Copy)]
+struct ListLocation {
+    path: &'static str,
+    line: u32,
+}
+
+/// Creates a [`ListLocation`] for the current location (with an additional offset to the actual list start);
+macro_rules! location {
+    (+ $offset:literal) => {
+        ListLocation { path: file!(), line: line!() + $offset }
+    };
+}
+
+const PERMITTED_RUSTC_DEPS_LOCATION: ListLocation = location!(+6);
 
 /// Crates rustc is allowed to depend on. Avoid adding to the list if possible.
 ///
@@ -238,7 +271,11 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "aho-corasick",
     "allocator-api2", // FIXME: only appears in Cargo.lock due to https://github.com/rust-lang/cargo/issues/10801
     "annotate-snippets",
+    "anstream",
     "anstyle",
+    "anstyle-parse",
+    "anstyle-query",
+    "anstyle-wincon",
     "ar_archive_writer",
     "arrayref",
     "arrayvec",
@@ -250,6 +287,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "cc",
     "cfg-if",
     "cfg_aliases",
+    "colorchoice",
     "constant_time_eq",
     "cpufeatures",
     "crc32fast",
@@ -267,6 +305,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "digest",
     "displaydoc",
     "dissimilar",
+    "dyn-clone",
     "either",
     "elsa",
     "ena",
@@ -275,6 +314,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "expect-test",
     "fallible-iterator", // dependency of `thorin`
     "fastrand",
+    "find-msvc-tools",
     "flate2",
     "fluent-bundle",
     "fluent-langneg",
@@ -287,18 +327,17 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "gimli",
     "gsgdt",
     "hashbrown",
+    "icu_collections",
     "icu_list",
-    "icu_list_data",
-    "icu_locid",
-    "icu_locid_transform",
-    "icu_locid_transform_data",
+    "icu_locale",
+    "icu_locale_core",
+    "icu_locale_data",
     "icu_provider",
-    "icu_provider_adapters",
-    "icu_provider_macros",
     "ident_case",
     "indexmap",
     "intl-memoizer",
     "intl_pluralrules",
+    "is_terminal_polyfill",
     "itertools",
     "itoa",
     "jiff",
@@ -323,6 +362,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "object",
     "odht",
     "once_cell",
+    "once_cell_polyfill",
     "overload",
     "parking_lot",
     "parking_lot_core",
@@ -332,6 +372,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "polonius-engine",
     "portable-atomic", // dependency for platforms doesn't support `AtomicU64` in std
     "portable-atomic-util",
+    "potential_utf",
     "ppv-lite86",
     "proc-macro-hack",
     "proc-macro2",
@@ -347,6 +388,8 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "rand_xorshift", // dependency for doc-tests in rustc_thread_pool
     "rand_xoshiro",
     "redox_syscall",
+    "ref-cast",
+    "ref-cast-impl",
     "regex",
     "regex-automata",
     "regex-syntax",
@@ -358,12 +401,15 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "rustix",
     "ruzstd", // via object in thorin-dwp
     "ryu",
+    "schemars",
+    "schemars_derive",
     "scoped-tls",
     "scopeguard",
     "self_cell",
-    "semver",
     "serde",
+    "serde_core",
     "serde_derive",
+    "serde_derive_internals",
     "serde_json",
     "serde_path_to_error",
     "sha1",
@@ -378,7 +424,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "syn",
     "synstructure",
     "tempfile",
-    "termcolor",
     "termize",
     "thin-vec",
     "thiserror",
@@ -410,6 +455,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "unicode-security",
     "unicode-width",
     "unicode-xid",
+    "utf8parse",
     "valuable",
     "version_check",
     "wasi",
@@ -417,7 +463,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "wasmparser",
     "winapi",
     "winapi-i686-pc-windows-gnu",
-    "winapi-util",
     "winapi-x86_64-pc-windows-gnu",
     "windows",
     "windows-collections",
@@ -448,11 +493,14 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "zerocopy-derive",
     "zerofrom",
     "zerofrom-derive",
+    "zerotrie",
     "zerovec",
     "zerovec-derive",
     // tidy-alphabetical-end
 ];
 
+const PERMITTED_STDLIB_DEPS_LOCATION: ListLocation = location!(+2);
+
 const PERMITTED_STDLIB_DEPENDENCIES: &[&str] = &[
     // tidy-alphabetical-start
     "addr2line",
@@ -469,6 +517,7 @@ const PERMITTED_STDLIB_DEPENDENCIES: &[&str] = &[
     "libc",
     "memchr",
     "miniz_oxide",
+    "moto-rt",
     "object",
     "r-efi",
     "r-efi-alloc",
@@ -478,8 +527,8 @@ const PERMITTED_STDLIB_DEPENDENCIES: &[&str] = &[
     "rustc-demangle",
     "rustc-literal-escaper",
     "shlex",
-    "unicode-width",
     "unwinding",
+    "vex-sdk",
     "wasi",
     "windows-sys",
     "windows-targets",
@@ -491,9 +540,12 @@ const PERMITTED_STDLIB_DEPENDENCIES: &[&str] = &[
     "windows_x86_64_gnu",
     "windows_x86_64_gnullvm",
     "windows_x86_64_msvc",
+    "wit-bindgen",
     // tidy-alphabetical-end
 ];
 
+const PERMITTED_CRANELIFT_DEPS_LOCATION: ListLocation = location!(+2);
+
 const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[
     // tidy-alphabetical-start
     "allocator-api2",
@@ -563,35 +615,48 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[
 ///
 /// `root` is path to the directory with the root `Cargo.toml` (for the workspace). `cargo` is path
 /// to the cargo executable.
-pub fn check(root: &Path, cargo: &Path, bless: bool, bad: &mut bool) {
+pub fn check(root: &Path, cargo: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("deps");
+    let bless = tidy_ctx.is_bless_enabled();
+
     let mut checked_runtime_licenses = false;
 
-    check_proc_macro_dep_list(root, cargo, bless, bad);
+    check_proc_macro_dep_list(root, cargo, bless, &mut check);
 
-    for &(workspace, exceptions, permitted_deps, submodules) in WORKSPACES {
+    for &WorkspaceInfo { path, exceptions, crates_and_deps, submodules } in WORKSPACES {
         if has_missing_submodule(root, submodules) {
             continue;
         }
 
-        if !root.join(workspace).join("Cargo.lock").exists() {
-            tidy_error!(bad, "the `{workspace}` workspace doesn't have a Cargo.lock");
+        if !root.join(path).join("Cargo.lock").exists() {
+            check.error(format!("the `{path}` workspace doesn't have a Cargo.lock"));
             continue;
         }
 
         let mut cmd = cargo_metadata::MetadataCommand::new();
         cmd.cargo_path(cargo)
-            .manifest_path(root.join(workspace).join("Cargo.toml"))
+            .manifest_path(root.join(path).join("Cargo.toml"))
             .features(cargo_metadata::CargoOpt::AllFeatures)
             .other_options(vec!["--locked".to_owned()]);
         let metadata = t!(cmd.exec());
 
-        check_license_exceptions(&metadata, workspace, exceptions, bad);
-        if let Some((crates, permitted_deps)) = permitted_deps {
-            check_permitted_dependencies(&metadata, workspace, permitted_deps, crates, bad);
+        check_license_exceptions(&metadata, path, exceptions, &mut check);
+        if let Some((crates, permitted_deps, location)) = crates_and_deps {
+            let descr = crates.get(0).unwrap_or(&path);
+            check_permitted_dependencies(
+                &metadata,
+                descr,
+                permitted_deps,
+                crates,
+                location,
+                &mut check,
+            );
         }
 
-        if workspace == "library" {
-            check_runtime_license_exceptions(&metadata, bad);
+        if path == "library" {
+            check_runtime_license_exceptions(&metadata, &mut check);
+            check_runtime_no_duplicate_dependencies(&metadata, &mut check);
+            check_runtime_no_proc_macros(&metadata, &mut check);
             checked_runtime_licenses = true;
         }
     }
@@ -602,7 +667,7 @@ pub fn check(root: &Path, cargo: &Path, bless: bool, bad: &mut bool) {
 }
 
 /// Ensure the list of proc-macro crate transitive dependencies is up to date
-fn check_proc_macro_dep_list(root: &Path, cargo: &Path, bless: bool, bad: &mut bool) {
+fn check_proc_macro_dep_list(root: &Path, cargo: &Path, bless: bool, check: &mut RunningCheck) {
     let mut cmd = cargo_metadata::MetadataCommand::new();
     cmd.cargo_path(cargo)
         .manifest_path(root.join("Cargo.toml"))
@@ -649,22 +714,22 @@ pub static CRATES: &[&str] = &[
         )
         .unwrap();
     } else {
-        let old_bad = *bad;
+        let mut error_found = false;
 
         for missing in proc_macro_deps.difference(&expected) {
-            tidy_error!(
-                bad,
+            error_found = true;
+            check.error(format!(
                 "proc-macro crate dependency `{missing}` is not registered in `src/bootstrap/src/utils/proc_macro_deps.rs`",
-            );
+            ));
         }
         for extra in expected.difference(&proc_macro_deps) {
-            tidy_error!(
-                bad,
+            error_found = true;
+            check.error(format!(
                 "`{extra}` is registered in `src/bootstrap/src/utils/proc_macro_deps.rs`, but is not a proc-macro crate dependency",
-            );
+            ));
         }
-        if *bad != old_bad {
-            eprintln!("Run `./x.py test tidy --bless` to regenerate the list");
+        if error_found {
+            check.message("Run `./x.py test tidy --bless` to regenerate the list");
         }
     }
 }
@@ -686,7 +751,7 @@ pub fn has_missing_submodule(root: &Path, submodules: &[&str]) -> bool {
 ///
 /// Unlike for tools we don't allow exceptions to the `LICENSES` list for the runtime with the sole
 /// exception of `fortanix-sgx-abi` which is only used on x86_64-fortanix-unknown-sgx.
-fn check_runtime_license_exceptions(metadata: &Metadata, bad: &mut bool) {
+fn check_runtime_license_exceptions(metadata: &Metadata, check: &mut RunningCheck) {
     for pkg in &metadata.packages {
         if pkg.source.is_none() {
             // No need to check local packages.
@@ -695,7 +760,8 @@ fn check_runtime_license_exceptions(metadata: &Metadata, bad: &mut bool) {
         let license = match &pkg.license {
             Some(license) => license,
             None => {
-                tidy_error!(bad, "dependency `{}` does not define a license expression", pkg.id);
+                check
+                    .error(format!("dependency `{}` does not define a license expression", pkg.id));
                 continue;
             }
         };
@@ -708,7 +774,7 @@ fn check_runtime_license_exceptions(metadata: &Metadata, bad: &mut bool) {
                 continue;
             }
 
-            tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id);
+            check.error(format!("invalid license `{}` in `{}`", license, pkg.id));
         }
     }
 }
@@ -720,41 +786,41 @@ fn check_license_exceptions(
     metadata: &Metadata,
     workspace: &str,
     exceptions: &[(&str, &str)],
-    bad: &mut bool,
+    check: &mut RunningCheck,
 ) {
     // Validate the EXCEPTIONS list hasn't changed.
     for (name, license) in exceptions {
         // Check that the package actually exists.
         if !metadata.packages.iter().any(|p| *p.name == *name) {
-            tidy_error!(
-                bad,
-                "could not find exception package `{}` in workspace `{workspace}`\n\
+            check.error(format!(
+                "could not find exception package `{name}` in workspace `{workspace}`\n\
                 Remove from EXCEPTIONS list if it is no longer used.",
-                name
-            );
+            ));
         }
         // Check that the license hasn't changed.
         for pkg in metadata.packages.iter().filter(|p| *p.name == *name) {
             match &pkg.license {
                 None => {
-                    tidy_error!(
-                        bad,
+                    check.error(format!(
                         "dependency exception `{}` in workspace `{workspace}` does not declare a license expression",
                         pkg.id
-                    );
+                    ));
                 }
                 Some(pkg_license) => {
                     if pkg_license.as_str() != *license {
-                        println!(
-                            "dependency exception `{name}` license in workspace `{workspace}` has changed"
-                        );
-                        println!("    previously `{license}` now `{pkg_license}`");
-                        println!("    update EXCEPTIONS for the new license");
-                        *bad = true;
+                        check.error(format!(r#"dependency exception `{name}` license in workspace `{workspace}` has changed
+    previously `{license}` now `{pkg_license}`
+    update EXCEPTIONS for the new license
+"#));
                     }
                 }
             }
         }
+        if LICENSES.contains(license) || LICENSES_TOOLS.contains(license) {
+            check.error(format!(
+                "dependency exception `{name}` is not necessary. `{license}` is an allowed license"
+            ));
+        }
     }
 
     let exception_names: Vec<_> = exceptions.iter().map(|(name, _license)| *name).collect();
@@ -771,21 +837,50 @@ fn check_license_exceptions(
         let license = match &pkg.license {
             Some(license) => license,
             None => {
-                tidy_error!(
-                    bad,
+                check.error(format!(
                     "dependency `{}` in workspace `{workspace}` does not define a license expression",
                     pkg.id
-                );
+                ));
                 continue;
             }
         };
-        if !LICENSES.contains(&license.as_str()) {
-            tidy_error!(
-                bad,
+        if !LICENSES.contains(&license.as_str()) && !LICENSES_TOOLS.contains(&license.as_str()) {
+            check.error(format!(
                 "invalid license `{}` for package `{}` in workspace `{workspace}`",
-                license,
-                pkg.id
-            );
+                license, pkg.id
+            ));
+        }
+    }
+}
+
+fn check_runtime_no_duplicate_dependencies(metadata: &Metadata, check: &mut RunningCheck) {
+    let mut seen_pkgs = HashSet::new();
+    for pkg in &metadata.packages {
+        if pkg.source.is_none() {
+            continue;
+        }
+
+        // Skip the `wasi` crate here which the standard library explicitly
+        // depends on two version of (one for the `wasm32-wasip1` target and
+        // another for the `wasm32-wasip2` target).
+        if pkg.name.to_string() != "wasi" && !seen_pkgs.insert(&*pkg.name) {
+            check.error(format!(
+                "duplicate package `{}` is not allowed for the standard library",
+                pkg.name
+            ));
+        }
+    }
+}
+
+fn check_runtime_no_proc_macros(metadata: &Metadata, check: &mut RunningCheck) {
+    for pkg in &metadata.packages {
+        if pkg.targets.iter().any(|target| target.is_proc_macro()) {
+            check.error(format!(
+                "proc macro `{}` is not allowed as standard library dependency.\n\
+                Using proc macros in the standard library would break cross-compilation \
+                as proc-macros don't get shipped for the host tuple.",
+                pkg.name
+            ));
         }
     }
 }
@@ -799,7 +894,8 @@ fn check_permitted_dependencies(
     descr: &str,
     permitted_dependencies: &[&'static str],
     restricted_dependency_crates: &[&'static str],
-    bad: &mut bool,
+    permitted_location: ListLocation,
+    check: &mut RunningCheck,
 ) {
     let mut has_permitted_dep_error = false;
     let mut deps = HashSet::new();
@@ -821,11 +917,10 @@ fn check_permitted_dependencies(
             }
         }
         if !deps.iter().any(|dep_id| compare(pkg_from_id(metadata, dep_id), permitted)) {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "could not find allowed package `{permitted}`\n\
                 Remove from PERMITTED_DEPENDENCIES list if it is no longer used.",
-            );
+            ));
             has_permitted_dep_error = true;
         }
     }
@@ -852,14 +947,14 @@ fn check_permitted_dependencies(
                 false
             };
             if !is_eq {
-                tidy_error!(bad, "Dependency for {descr} not explicitly permitted: {}", dep.id);
+                check.error(format!("Dependency for {descr} not explicitly permitted: {}", dep.id));
                 has_permitted_dep_error = true;
             }
         }
     }
 
     if has_permitted_dep_error {
-        eprintln!("Go to `{PERMITTED_DEPS_LOCATION}` for the list.");
+        eprintln!("Go to `{}:{}` for the list.", permitted_location.path, permitted_location.line);
     }
 }
 
diff --git a/src/tools/tidy/src/diagnostics.rs b/src/tools/tidy/src/diagnostics.rs
new file mode 100644
index 000000000000..1e25f862522e
--- /dev/null
+++ b/src/tools/tidy/src/diagnostics.rs
@@ -0,0 +1,274 @@
+use std::collections::HashSet;
+use std::fmt::{Display, Formatter};
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Mutex};
+
+use termcolor::{Color, WriteColor};
+
+#[derive(Clone, Default)]
+///CLI flags used by tidy.
+pub struct TidyFlags {
+    ///Applies style and formatting changes during a tidy run.
+    bless: bool,
+}
+
+impl TidyFlags {
+    pub fn new(cfg_args: &[String]) -> Self {
+        let mut flags = Self::default();
+
+        for arg in cfg_args {
+            match arg.as_str() {
+                "--bless" => flags.bless = true,
+                _ => continue,
+            }
+        }
+        flags
+    }
+}
+
+/// Collects diagnostics from all tidy steps, and contains shared information
+/// that determines how should message and logs be presented.
+///
+/// Since checks are executed in parallel, the context is internally synchronized, to avoid
+/// all checks to lock it explicitly.
+#[derive(Clone)]
+pub struct TidyCtx {
+    tidy_flags: TidyFlags,
+    diag_ctx: Arc>,
+}
+
+impl TidyCtx {
+    pub fn new(root_path: &Path, verbose: bool, tidy_flags: TidyFlags) -> Self {
+        Self {
+            diag_ctx: Arc::new(Mutex::new(DiagCtxInner {
+                running_checks: Default::default(),
+                finished_checks: Default::default(),
+                root_path: root_path.to_path_buf(),
+                verbose,
+            })),
+            tidy_flags,
+        }
+    }
+
+    pub fn is_bless_enabled(&self) -> bool {
+        self.tidy_flags.bless
+    }
+
+    pub fn start_check>(&self, id: Id) -> RunningCheck {
+        let mut id = id.into();
+
+        let mut ctx = self.diag_ctx.lock().unwrap();
+
+        // Shorten path for shorter diagnostics
+        id.path = match id.path {
+            Some(path) => Some(path.strip_prefix(&ctx.root_path).unwrap_or(&path).to_path_buf()),
+            None => None,
+        };
+
+        ctx.start_check(id.clone());
+        RunningCheck {
+            id,
+            bad: false,
+            ctx: self.diag_ctx.clone(),
+            #[cfg(test)]
+            errors: vec![],
+        }
+    }
+
+    pub fn into_failed_checks(self) -> Vec {
+        let ctx = Arc::into_inner(self.diag_ctx).unwrap().into_inner().unwrap();
+        assert!(ctx.running_checks.is_empty(), "Some checks are still running");
+        ctx.finished_checks.into_iter().filter(|c| c.bad).collect()
+    }
+}
+
+struct DiagCtxInner {
+    running_checks: HashSet,
+    finished_checks: HashSet,
+    verbose: bool,
+    root_path: PathBuf,
+}
+
+impl DiagCtxInner {
+    fn start_check(&mut self, id: CheckId) {
+        if self.has_check_id(&id) {
+            panic!("Starting a check named `{id:?}` for the second time");
+        }
+
+        self.running_checks.insert(id);
+    }
+
+    fn finish_check(&mut self, check: FinishedCheck) {
+        assert!(
+            self.running_checks.remove(&check.id),
+            "Finishing check `{:?}` that was not started",
+            check.id
+        );
+
+        if check.bad {
+            output_message("FAIL", Some(&check.id), Some(COLOR_ERROR));
+        } else if self.verbose {
+            output_message("OK", Some(&check.id), Some(COLOR_SUCCESS));
+        }
+
+        self.finished_checks.insert(check);
+    }
+
+    fn has_check_id(&self, id: &CheckId) -> bool {
+        self.running_checks
+            .iter()
+            .chain(self.finished_checks.iter().map(|c| &c.id))
+            .any(|c| c == id)
+    }
+}
+
+/// Identifies a single step
+#[derive(PartialEq, Eq, Hash, Clone, Debug)]
+pub struct CheckId {
+    pub name: String,
+    pub path: Option,
+}
+
+impl CheckId {
+    pub fn new(name: &'static str) -> Self {
+        Self { name: name.to_string(), path: None }
+    }
+
+    pub fn path(self, path: &Path) -> Self {
+        Self { path: Some(path.to_path_buf()), ..self }
+    }
+}
+
+impl From<&'static str> for CheckId {
+    fn from(name: &'static str) -> Self {
+        Self::new(name)
+    }
+}
+
+impl Display for CheckId {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.name)?;
+        if let Some(path) = &self.path {
+            write!(f, " ({})", path.display())?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(PartialEq, Eq, Hash, Debug)]
+pub struct FinishedCheck {
+    id: CheckId,
+    bad: bool,
+}
+
+impl FinishedCheck {
+    pub fn id(&self) -> &CheckId {
+        &self.id
+    }
+}
+
+/// Represents a single tidy check, identified by its `name`, running.
+pub struct RunningCheck {
+    id: CheckId,
+    bad: bool,
+    ctx: Arc>,
+    #[cfg(test)]
+    errors: Vec,
+}
+
+impl RunningCheck {
+    /// Creates a new instance of a running check without going through the diag
+    /// context.
+    /// Useful if you want to run some functions from tidy without configuring
+    /// diagnostics.
+    pub fn new_noop() -> Self {
+        let ctx = TidyCtx::new(Path::new(""), false, TidyFlags::default());
+        ctx.start_check("noop")
+    }
+
+    /// Immediately output an error and mark the check as failed.
+    pub fn error(&mut self, msg: T) {
+        self.mark_as_bad();
+        let msg = msg.to_string();
+        output_message(&msg, Some(&self.id), Some(COLOR_ERROR));
+        #[cfg(test)]
+        self.errors.push(msg);
+    }
+
+    /// Immediately output a warning.
+    pub fn warning(&mut self, msg: T) {
+        output_message(&msg.to_string(), Some(&self.id), Some(COLOR_WARNING));
+    }
+
+    /// Output an informational message
+    pub fn message(&mut self, msg: T) {
+        output_message(&msg.to_string(), Some(&self.id), None);
+    }
+
+    /// Output a message only if verbose output is enabled.
+    pub fn verbose_msg(&mut self, msg: T) {
+        if self.is_verbose_enabled() {
+            self.message(msg);
+        }
+    }
+
+    /// Has an error already occured for this check?
+    pub fn is_bad(&self) -> bool {
+        self.bad
+    }
+
+    /// Is verbose output enabled?
+    pub fn is_verbose_enabled(&self) -> bool {
+        self.ctx.lock().unwrap().verbose
+    }
+
+    #[cfg(test)]
+    pub fn get_errors(&self) -> Vec {
+        self.errors.clone()
+    }
+
+    fn mark_as_bad(&mut self) {
+        self.bad = true;
+    }
+}
+
+impl Drop for RunningCheck {
+    fn drop(&mut self) {
+        self.ctx.lock().unwrap().finish_check(FinishedCheck { id: self.id.clone(), bad: self.bad })
+    }
+}
+
+pub const COLOR_SUCCESS: Color = Color::Green;
+pub const COLOR_ERROR: Color = Color::Red;
+pub const COLOR_WARNING: Color = Color::Yellow;
+
+/// Output a message to stderr.
+/// The message can be optionally scoped to a certain check, and it can also have a certain color.
+pub fn output_message(msg: &str, id: Option<&CheckId>, color: Option) {
+    use std::io::Write;
+
+    use termcolor::{ColorChoice, ColorSpec, StandardStream};
+
+    let mut stderr = StandardStream::stderr(ColorChoice::Auto);
+    if let Some(color) = &color {
+        stderr.set_color(ColorSpec::new().set_fg(Some(*color))).unwrap();
+    }
+
+    match id {
+        Some(id) => {
+            write!(&mut stderr, "tidy [{}", id.name).unwrap();
+            if let Some(path) = &id.path {
+                write!(&mut stderr, " ({})", path.display()).unwrap();
+            }
+            write!(&mut stderr, "]").unwrap();
+        }
+        None => {
+            write!(&mut stderr, "tidy").unwrap();
+        }
+    }
+    if color.is_some() {
+        stderr.set_color(&ColorSpec::new()).unwrap();
+    }
+
+    writeln!(&mut stderr, ": {msg}").unwrap();
+}
diff --git a/src/tools/tidy/src/edition.rs b/src/tools/tidy/src/edition.rs
index 08f6a3909f80..de21846ab188 100644
--- a/src/tools/tidy/src/edition.rs
+++ b/src/tools/tidy/src/edition.rs
@@ -2,9 +2,11 @@
 
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
-pub fn check(path: &Path, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("edition").path(path));
     walk(path, |path, _is_dir| filter_dirs(path), &mut |entry, contents| {
         let file = entry.path();
         let filename = file.file_name().unwrap();
@@ -23,11 +25,10 @@ pub fn check(path: &Path, bad: &mut bool) {
         // Check that all packages use the 2021 edition. Virtual workspaces don't allow setting an
         // edition, so these shouldn't be checked.
         if is_package && !is_current_edition {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "{} doesn't have `edition = \"2021\"` or `edition = \"2024\"` on a separate line",
                 file.display()
-            );
+            ));
         }
     });
 }
diff --git a/src/tools/tidy/src/error_codes.rs b/src/tools/tidy/src/error_codes.rs
index 65aa89fe8016..9150d36339c8 100644
--- a/src/tools/tidy/src/error_codes.rs
+++ b/src/tools/tidy/src/error_codes.rs
@@ -22,6 +22,7 @@ use std::path::Path;
 
 use regex::Regex;
 
+use crate::diagnostics::{RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, walk, walk_many};
 
 const ERROR_CODES_PATH: &str = "compiler/rustc_error_codes/src/lib.rs";
@@ -35,71 +36,50 @@ const IGNORE_DOCTEST_CHECK: &[&str] = &["E0464", "E0570", "E0601", "E0602", "E07
 const IGNORE_UI_TEST_CHECK: &[&str] =
     &["E0461", "E0465", "E0514", "E0554", "E0640", "E0717", "E0729"];
 
-macro_rules! verbose_print {
-    ($verbose:expr, $($fmt:tt)*) => {
-        if $verbose {
-            println!("{}", format_args!($($fmt)*));
-        }
-    };
-}
-
-pub fn check(
-    root_path: &Path,
-    search_paths: &[&Path],
-    verbose: bool,
-    ci_info: &crate::CiInfo,
-    bad: &mut bool,
-) {
-    let mut errors = Vec::new();
+pub fn check(root_path: &Path, search_paths: &[&Path], ci_info: &crate::CiInfo, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("error_codes");
 
     // Check that no error code explanation was removed.
-    check_removed_error_code_explanation(ci_info, bad);
+    check_removed_error_code_explanation(ci_info, &mut check);
 
     // Stage 1: create list
-    let error_codes = extract_error_codes(root_path, &mut errors);
-    if verbose {
-        println!("Found {} error codes", error_codes.len());
-        println!("Highest error code: `{}`", error_codes.iter().max().unwrap());
-    }
+    let error_codes = extract_error_codes(root_path, &mut check);
+    check.verbose_msg(format!("Found {} error codes", error_codes.len()));
+    check.verbose_msg(format!("Highest error code: `{}`", error_codes.iter().max().unwrap()));
 
     // Stage 2: check list has docs
-    let no_longer_emitted = check_error_codes_docs(root_path, &error_codes, &mut errors, verbose);
+    let no_longer_emitted = check_error_codes_docs(root_path, &error_codes, &mut check);
 
     // Stage 3: check list has UI tests
-    check_error_codes_tests(root_path, &error_codes, &mut errors, verbose, &no_longer_emitted);
+    check_error_codes_tests(root_path, &error_codes, &mut check, &no_longer_emitted);
 
     // Stage 4: check list is emitted by compiler
-    check_error_codes_used(search_paths, &error_codes, &mut errors, &no_longer_emitted, verbose);
-
-    // Print any errors.
-    for error in errors {
-        tidy_error!(bad, "{}", error);
-    }
+    check_error_codes_used(search_paths, &error_codes, &mut check, &no_longer_emitted);
 }
 
-fn check_removed_error_code_explanation(ci_info: &crate::CiInfo, bad: &mut bool) {
+fn check_removed_error_code_explanation(ci_info: &crate::CiInfo, check: &mut RunningCheck) {
     let Some(base_commit) = &ci_info.base_commit else {
-        eprintln!("Skipping error code explanation removal check");
+        check.verbose_msg("Skipping error code explanation removal check");
         return;
     };
     let Some(diff) = crate::git_diff(base_commit, "--name-status") else {
-        *bad = true;
-        eprintln!("removed error code explanation tidy check: Failed to run git diff");
+        check.error(format!("removed error code explanation: Failed to run git diff"));
         return;
     };
     if diff.lines().any(|line| {
         line.starts_with('D') && line.contains("compiler/rustc_error_codes/src/error_codes/")
     }) {
-        *bad = true;
-        eprintln!("tidy check error: Error code explanations should never be removed!");
-        eprintln!("Take a look at E0001 to see how to handle it.");
+        check.error(format!(
+            r#"Error code explanations should never be removed!
+Take a look at E0001 to see how to handle it."#
+        ));
         return;
     }
-    println!("No error code explanation was removed!");
+    check.verbose_msg("No error code explanation was removed!");
 }
 
 /// Stage 1: Parses a list of error codes from `error_codes.rs`.
-fn extract_error_codes(root_path: &Path, errors: &mut Vec) -> Vec {
+fn extract_error_codes(root_path: &Path, check: &mut RunningCheck) -> Vec {
     let path = root_path.join(Path::new(ERROR_CODES_PATH));
     let file =
         fs::read_to_string(&path).unwrap_or_else(|e| panic!("failed to read `{path:?}`: {e}"));
@@ -117,7 +97,7 @@ fn extract_error_codes(root_path: &Path, errors: &mut Vec) -> Vec) -> Vec) -> Vec) -> Vec) -> Vec,
-    verbose: bool,
+    check: &mut RunningCheck,
 ) -> Vec {
     let docs_path = root_path.join(Path::new(ERROR_DOCS_PATH));
 
@@ -184,7 +164,7 @@ fn check_error_codes_docs(
 
         // Error if the file isn't markdown.
         if path.extension() != Some(OsStr::new("md")) {
-            errors.push(format!(
+            check.error(format!(
                 "Found unexpected non-markdown file in error code docs directory: {}",
                 path.display()
             ));
@@ -196,7 +176,7 @@ fn check_error_codes_docs(
         let err_code = filename.unwrap().0; // `unwrap` is ok because we know the filename is in the correct format.
 
         if error_codes.iter().all(|e| e != err_code) {
-            errors.push(format!(
+            check.error(format!(
                 "Found valid file `{}` in error code docs directory without corresponding \
                 entry in `rustc_error_codes/src/lib.rs`",
                 path.display()
@@ -208,11 +188,10 @@ fn check_error_codes_docs(
             check_explanation_has_doctest(contents, err_code);
 
         if emit_ignore_warning {
-            verbose_print!(
-                verbose,
+            check.verbose_msg(format!(
                 "warning: Error code `{err_code}` uses the ignore header. This should not be used, add the error code to the \
                 `IGNORE_DOCTEST_CHECK` constant instead."
-            );
+            ));
         }
 
         if no_longer_emitted {
@@ -220,11 +199,10 @@ fn check_error_codes_docs(
         }
 
         if !found_code_example {
-            verbose_print!(
-                verbose,
+            check.verbose_msg(format!(
                 "warning: Error code `{err_code}` doesn't have a code example, all error codes are expected to have one \
                 (even if untested)."
-            );
+            ));
             return;
         }
 
@@ -232,12 +210,12 @@ fn check_error_codes_docs(
 
         // Check that the explanation has a doctest, and if it shouldn't, that it doesn't
         if !found_proper_doctest && !test_ignored {
-            errors.push(format!(
+            check.error(format!(
                 "`{}` doesn't use its own error code in compile_fail example",
                 path.display(),
             ));
         } else if found_proper_doctest && test_ignored {
-            errors.push(format!(
+            check.error(format!(
                 "`{}` has a compile_fail doctest with its own error code, it shouldn't \
                 be listed in `IGNORE_DOCTEST_CHECK`",
                 path.display(),
@@ -289,8 +267,7 @@ fn check_explanation_has_doctest(explanation: &str, err_code: &str) -> (bool, bo
 fn check_error_codes_tests(
     root_path: &Path,
     error_codes: &[String],
-    errors: &mut Vec,
-    verbose: bool,
+    check: &mut RunningCheck,
     no_longer_emitted: &[String],
 ) {
     let tests_path = root_path.join(Path::new(ERROR_TESTS_PATH));
@@ -299,15 +276,14 @@ fn check_error_codes_tests(
         let test_path = tests_path.join(format!("{code}.stderr"));
 
         if !test_path.exists() && !IGNORE_UI_TEST_CHECK.contains(&code.as_str()) {
-            verbose_print!(
-                verbose,
+            check.verbose_msg(format!(
                 "warning: Error code `{code}` needs to have at least one UI test in the `tests/error-codes/` directory`!"
-            );
+            ));
             continue;
         }
         if IGNORE_UI_TEST_CHECK.contains(&code.as_str()) {
             if test_path.exists() {
-                errors.push(format!(
+                check.error(format!(
                     "Error code `{code}` has a UI test in `tests/ui/error-codes/{code}.rs`, it shouldn't be listed in `EXEMPTED_FROM_TEST`!"
                 ));
             }
@@ -317,11 +293,10 @@ fn check_error_codes_tests(
         let file = match fs::read_to_string(&test_path) {
             Ok(file) => file,
             Err(err) => {
-                verbose_print!(
-                    verbose,
+                check.verbose_msg(format!(
                     "warning: Failed to read UI test file (`{}`) for `{code}` but the file exists. The test is assumed to work:\n{err}",
                     test_path.display()
-                );
+                ));
                 continue;
             }
         };
@@ -343,10 +318,9 @@ fn check_error_codes_tests(
         }
 
         if !found_code {
-            verbose_print!(
-                verbose,
+            check.verbose_msg(format!(
                 "warning: Error code `{code}` has a UI test file, but doesn't contain its own error code!"
-            );
+            ));
         }
     }
 }
@@ -355,9 +329,8 @@ fn check_error_codes_tests(
 fn check_error_codes_used(
     search_paths: &[&Path],
     error_codes: &[String],
-    errors: &mut Vec,
+    check: &mut RunningCheck,
     no_longer_emitted: &[String],
-    verbose: bool,
 ) {
     // Search for error codes in the form `E0123`.
     let regex = Regex::new(r#"\bE\d{4}\b"#).unwrap();
@@ -384,7 +357,7 @@ fn check_error_codes_used(
 
                     if !error_codes.contains(&error_code) {
                         // This error code isn't properly defined, we must error.
-                        errors.push(format!("Error code `{error_code}` is used in the compiler but not defined and documented in `compiler/rustc_error_codes/src/lib.rs`."));
+                        check.error(format!("Error code `{error_code}` is used in the compiler but not defined and documented in `compiler/rustc_error_codes/src/lib.rs`."));
                         continue;
                     }
 
@@ -397,7 +370,7 @@ fn check_error_codes_used(
 
     for code in error_codes {
         if !found_codes.contains(code) && !no_longer_emitted.contains(code) {
-            errors.push(format!(
+            check.error(format!(
                 "Error code `{code}` exists, but is not emitted by the compiler!\n\
                 Please mark the code as no longer emitted by adding the following note to the top of the `EXXXX.md` file:\n\
                 `#### Note: this error code is no longer emitted by the compiler`\n\
@@ -406,10 +379,9 @@ fn check_error_codes_used(
         }
 
         if found_codes.contains(code) && no_longer_emitted.contains(code) {
-            verbose_print!(
-                verbose,
+            check.verbose_msg(format!(
                 "warning: Error code `{code}` is used when it's marked as \"no longer emitted\""
-            );
+            ));
         }
     }
 }
diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs
index bc217a55cc19..19c773d12f7f 100644
--- a/src/tools/tidy/src/extdeps.rs
+++ b/src/tools/tidy/src/extdeps.rs
@@ -3,6 +3,9 @@
 use std::fs;
 use std::path::Path;
 
+use crate::deps::WorkspaceInfo;
+use crate::diagnostics::TidyCtx;
+
 /// List of allowed sources for packages.
 const ALLOWED_SOURCES: &[&str] = &[
     r#""registry+https://github.com/rust-lang/crates.io-index""#,
@@ -12,23 +15,25 @@ const ALLOWED_SOURCES: &[&str] = &[
 
 /// Checks for external package sources. `root` is the path to the directory that contains the
 /// workspace `Cargo.toml`.
-pub fn check(root: &Path, bad: &mut bool) {
-    for &(workspace, _, _, submodules) in crate::deps::WORKSPACES {
+pub fn check(root: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("extdeps");
+
+    for &WorkspaceInfo { path, submodules, .. } in crate::deps::WORKSPACES {
         if crate::deps::has_missing_submodule(root, submodules) {
             continue;
         }
 
         // FIXME check other workspaces too
         // `Cargo.lock` of rust.
-        let path = root.join(workspace).join("Cargo.lock");
+        let lockfile = root.join(path).join("Cargo.lock");
 
-        if !path.exists() {
-            tidy_error!(bad, "the `{workspace}` workspace doesn't have a Cargo.lock");
+        if !lockfile.exists() {
+            check.error(format!("the `{path}` workspace doesn't have a Cargo.lock"));
             continue;
         }
 
         // Open and read the whole file.
-        let cargo_lock = t!(fs::read_to_string(&path));
+        let cargo_lock = t!(fs::read_to_string(&lockfile));
 
         // Process each line.
         for line in cargo_lock.lines() {
@@ -42,7 +47,7 @@ pub fn check(root: &Path, bad: &mut bool) {
 
             // Ensure source is allowed.
             if !ALLOWED_SOURCES.contains(&source) {
-                tidy_error!(bad, "invalid source: {}", source);
+                check.error(format!("invalid source: {}", source));
             }
         }
     }
diff --git a/src/tools/tidy/src/extra_checks/mod.rs b/src/tools/tidy/src/extra_checks/mod.rs
index f90f716cd950..a45af7fcf158 100644
--- a/src/tools/tidy/src/extra_checks/mod.rs
+++ b/src/tools/tidy/src/extra_checks/mod.rs
@@ -24,6 +24,7 @@ use std::str::FromStr;
 use std::{fmt, fs, io};
 
 use crate::CiInfo;
+use crate::diagnostics::TidyCtx;
 
 mod rustdoc_js;
 
@@ -41,7 +42,6 @@ const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml
 const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"];
 const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];
 
-// this must be kept in sync with with .github/workflows/spellcheck.yml
 const SPELLCHECK_DIRS: &[&str] = &["compiler", "library", "src/bootstrap", "src/librustdoc"];
 
 pub fn check(
@@ -51,11 +51,13 @@ pub fn check(
     librustdoc_path: &Path,
     tools_path: &Path,
     npm: &Path,
-    bless: bool,
+    cargo: &Path,
     extra_checks: Option<&str>,
     pos_args: &[String],
-    bad: &mut bool,
+    tidy_ctx: TidyCtx,
 ) {
+    let mut check = tidy_ctx.start_check("extra_checks");
+
     if let Err(e) = check_impl(
         root_path,
         outdir,
@@ -63,11 +65,12 @@ pub fn check(
         librustdoc_path,
         tools_path,
         npm,
-        bless,
+        cargo,
         extra_checks,
         pos_args,
+        &tidy_ctx,
     ) {
-        tidy_error!(bad, "{e}");
+        check.error(e);
     }
 }
 
@@ -78,12 +81,14 @@ fn check_impl(
     librustdoc_path: &Path,
     tools_path: &Path,
     npm: &Path,
-    bless: bool,
+    cargo: &Path,
     extra_checks: Option<&str>,
     pos_args: &[String],
+    tidy_ctx: &TidyCtx,
 ) -> Result<(), Error> {
     let show_diff =
         std::env::var("TIDY_PRINT_DIFF").is_ok_and(|v| v.eq_ignore_ascii_case("true") || v == "1");
+    let bless = tidy_ctx.is_bless_enabled();
 
     // Split comma-separated args up
     let mut lint_args = match extra_checks {
@@ -122,6 +127,12 @@ fn check_impl(
         };
     }
 
+    let rerun_with_bless = |mode: &str, action: &str| {
+        if !bless {
+            eprintln!("rerun tidy with `--extra-checks={mode} --bless` to {action}");
+        }
+    };
+
     let python_lint = extra_check!(Py, Lint);
     let python_fmt = extra_check!(Py, Fmt);
     let shell_lint = extra_check!(Shell, Lint);
@@ -145,14 +156,21 @@ fn check_impl(
     }
 
     if python_lint {
-        eprintln!("linting python files");
         let py_path = py_path.as_ref().unwrap();
-        let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &["check".as_ref()]);
+        let args: &[&OsStr] = if bless {
+            eprintln!("linting python files and applying suggestions");
+            &["check".as_ref(), "--fix".as_ref()]
+        } else {
+            eprintln!("linting python files");
+            &["check".as_ref()]
+        };
 
-        if res.is_err() && show_diff {
+        let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, args);
+
+        if res.is_err() && show_diff && !bless {
             eprintln!("\npython linting failed! Printing diff suggestions:");
 
-            let _ = run_ruff(
+            let diff_res = run_ruff(
                 root_path,
                 outdir,
                 py_path,
@@ -160,6 +178,10 @@ fn check_impl(
                 &file_args,
                 &["check".as_ref(), "--diff".as_ref()],
             );
+            // `ruff check --diff` will return status 0 if there are no suggestions.
+            if diff_res.is_err() {
+                rerun_with_bless("py:lint", "apply ruff suggestions");
+            }
         }
         // Rethrow error
         res?;
@@ -190,7 +212,7 @@ fn check_impl(
                     &["format".as_ref(), "--diff".as_ref()],
                 );
             }
-            eprintln!("rerun tidy with `--extra-checks=py:fmt --bless` to reformat Python code");
+            rerun_with_bless("py:fmt", "reformat Python code");
         }
 
         // Rethrow error
@@ -223,7 +245,7 @@ fn check_impl(
         let args = merge_args(&cfg_args_clang_format, &file_args_clang_format);
         let res = py_runner(py_path.as_ref().unwrap(), false, None, "clang-format", &args);
 
-        if res.is_err() && show_diff {
+        if res.is_err() && show_diff && !bless {
             eprintln!("\nclang-format linting failed! Printing diff suggestions:");
 
             let mut cfg_args_clang_format_diff = cfg_args.clone();
@@ -263,6 +285,7 @@ fn check_impl(
                     );
                 }
             }
+            rerun_with_bless("cpp:fmt", "reformat C++ code");
         }
         // Rethrow error
         res?;
@@ -288,12 +311,16 @@ fn check_impl(
         args.extend_from_slice(SPELLCHECK_DIRS);
 
         if bless {
-            eprintln!("spellcheck files and fix");
+            eprintln!("spellchecking files and fixing typos");
             args.push("--write-changes");
         } else {
-            eprintln!("spellcheck files");
+            eprintln!("spellchecking files");
         }
-        spellcheck_runner(&args)?;
+        let res = spellcheck_runner(root_path, &outdir, &cargo, &args);
+        if res.is_err() {
+            rerun_with_bless("spellcheck", "fix typos");
+        }
+        res?;
     }
 
     if js_lint || js_typecheck {
@@ -301,11 +328,21 @@ fn check_impl(
     }
 
     if js_lint {
-        rustdoc_js::lint(outdir, librustdoc_path, tools_path)?;
+        if bless {
+            eprintln!("linting javascript files");
+        } else {
+            eprintln!("linting javascript files and applying suggestions");
+        }
+        let res = rustdoc_js::lint(outdir, librustdoc_path, tools_path, bless);
+        if res.is_err() {
+            rerun_with_bless("js:lint", "apply eslint suggestions");
+        }
+        res?;
         rustdoc_js::es_check(outdir, librustdoc_path)?;
     }
 
     if js_typecheck {
+        eprintln!("typechecking javascript files");
         rustdoc_js::typecheck(outdir, librustdoc_path)?;
     }
 
@@ -576,34 +613,25 @@ fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {
     if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) }
 }
 
-/// Check that spellchecker is installed then run it at the given path
-fn spellcheck_runner(args: &[&str]) -> Result<(), Error> {
-    // sync version with .github/workflows/spellcheck.yml
-    let expected_version = "typos-cli 1.34.0";
-    match Command::new("typos").arg("--version").output() {
-        Ok(o) => {
-            let stdout = String::from_utf8_lossy(&o.stdout);
-            if stdout.trim() != expected_version {
-                return Err(Error::Version {
-                    program: "typos",
-                    required: expected_version,
-                    installed: stdout.trim().to_string(),
-                });
+/// Ensure that spellchecker is installed then run it at the given path
+fn spellcheck_runner(
+    src_root: &Path,
+    outdir: &Path,
+    cargo: &Path,
+    args: &[&str],
+) -> Result<(), Error> {
+    let bin_path =
+        crate::ensure_version_or_cargo_install(outdir, cargo, "typos-cli", "typos", "1.38.1")?;
+    match Command::new(bin_path).current_dir(src_root).args(args).status() {
+        Ok(status) => {
+            if status.success() {
+                Ok(())
+            } else {
+                Err(Error::FailedCheck("typos"))
             }
         }
-        Err(e) if e.kind() == io::ErrorKind::NotFound => {
-            return Err(Error::MissingReq(
-                "typos",
-                "spellcheck file checks",
-                // sync version with .github/workflows/spellcheck.yml
-                Some("install tool via `cargo install typos-cli@1.34.0`".to_owned()),
-            ));
-        }
-        Err(e) => return Err(e.into()),
+        Err(err) => Err(Error::Generic(format!("failed to run typos tool: {err:?}"))),
     }
-
-    let status = Command::new("typos").args(args).status()?;
-    if status.success() { Ok(()) } else { Err(Error::FailedCheck("typos")) }
 }
 
 /// Check git for tracked files matching an extension
@@ -727,21 +755,19 @@ impl ExtraCheckArg {
         if !self.auto {
             return true;
         }
-        let ext = match self.lang {
-            ExtraCheckLang::Py => ".py",
-            ExtraCheckLang::Cpp => ".cpp",
-            ExtraCheckLang::Shell => ".sh",
-            ExtraCheckLang::Js => ".js",
+        let exts: &[&str] = match self.lang {
+            ExtraCheckLang::Py => &[".py"],
+            ExtraCheckLang::Cpp => &[".cpp"],
+            ExtraCheckLang::Shell => &[".sh"],
+            ExtraCheckLang::Js => &[".js", ".ts"],
             ExtraCheckLang::Spellcheck => {
-                for dir in SPELLCHECK_DIRS {
-                    if Path::new(filepath).starts_with(dir) {
-                        return true;
-                    }
+                if SPELLCHECK_DIRS.iter().any(|dir| Path::new(filepath).starts_with(dir)) {
+                    return true;
                 }
-                return false;
+                &[]
             }
         };
-        filepath.ends_with(ext)
+        exts.iter().any(|ext| filepath.ends_with(ext))
     }
 
     fn has_supported_kind(&self) -> bool {
diff --git a/src/tools/tidy/src/extra_checks/rustdoc_js.rs b/src/tools/tidy/src/extra_checks/rustdoc_js.rs
index 7708b128e23d..5137e6183672 100644
--- a/src/tools/tidy/src/extra_checks/rustdoc_js.rs
+++ b/src/tools/tidy/src/extra_checks/rustdoc_js.rs
@@ -40,19 +40,24 @@ fn rustdoc_js_files(librustdoc_path: &Path) -> Vec {
     return files;
 }
 
-fn run_eslint(outdir: &Path, args: &[PathBuf], config_folder: PathBuf) -> Result<(), super::Error> {
-    let mut child = spawn_cmd(
-        Command::new(node_module_bin(outdir, "eslint"))
-            .arg("-c")
-            .arg(config_folder.join(".eslintrc.js"))
-            .args(args),
-    )?;
+fn run_eslint(
+    outdir: &Path,
+    args: &[PathBuf],
+    config_folder: PathBuf,
+    bless: bool,
+) -> Result<(), super::Error> {
+    let mut cmd = Command::new(node_module_bin(outdir, "eslint"));
+    if bless {
+        cmd.arg("--fix");
+    }
+    cmd.arg("-c").arg(config_folder.join(".eslintrc.js")).args(args);
+    let mut child = spawn_cmd(&mut cmd)?;
     match child.wait() {
         Ok(exit_status) => {
             if exit_status.success() {
                 return Ok(());
             }
-            Err(super::Error::FailedCheck("eslint command failed"))
+            Err(super::Error::FailedCheck("eslint"))
         }
         Err(error) => Err(super::Error::Generic(format!("eslint command failed: {error:?}"))),
     }
@@ -62,16 +67,17 @@ pub(super) fn lint(
     outdir: &Path,
     librustdoc_path: &Path,
     tools_path: &Path,
+    bless: bool,
 ) -> Result<(), super::Error> {
     let files_to_check = rustdoc_js_files(librustdoc_path);
     println!("Running eslint on rustdoc JS files");
-    run_eslint(outdir, &files_to_check, librustdoc_path.join("html/static"))?;
+    run_eslint(outdir, &files_to_check, librustdoc_path.join("html/static"), bless)?;
 
-    run_eslint(outdir, &[tools_path.join("rustdoc-js/tester.js")], tools_path.join("rustdoc-js"))?;
     run_eslint(
         outdir,
-        &[tools_path.join("rustdoc-gui/tester.js")],
-        tools_path.join("rustdoc-gui"),
+        &[tools_path.join("rustdoc-js/tester.js")],
+        tools_path.join("rustdoc-js"),
+        bless,
     )?;
     Ok(())
 }
@@ -88,7 +94,7 @@ pub(super) fn typecheck(outdir: &Path, librustdoc_path: &Path) -> Result<(), sup
             if exit_status.success() {
                 return Ok(());
             }
-            Err(super::Error::FailedCheck("tsc command failed"))
+            Err(super::Error::FailedCheck("tsc"))
         }
         Err(error) => Err(super::Error::Generic(format!("tsc command failed: {error:?}"))),
     }
@@ -106,7 +112,7 @@ pub(super) fn es_check(outdir: &Path, librustdoc_path: &Path) -> Result<(), supe
             if exit_status.success() {
                 return Ok(());
             }
-            Err(super::Error::FailedCheck("es-check command failed"))
+            Err(super::Error::FailedCheck("es-check"))
         }
         Err(error) => Err(super::Error::Generic(format!("es-check command failed: {error:?}"))),
     }
diff --git a/src/tools/tidy/src/features.rs b/src/tools/tidy/src/features.rs
index fb00b3a943f8..08061bd834e3 100644
--- a/src/tools/tidy/src/features.rs
+++ b/src/tools/tidy/src/features.rs
@@ -9,18 +9,21 @@
 //! * All unstable lang features have tests to ensure they are actually unstable.
 //! * Language features in a group are sorted by feature name.
 
+use std::collections::BTreeSet;
 use std::collections::hash_map::{Entry, HashMap};
 use std::ffi::OsStr;
 use std::num::NonZeroU32;
 use std::path::{Path, PathBuf};
 use std::{fmt, fs};
 
+use crate::diagnostics::{RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, filter_not_rust, walk, walk_many};
 
 #[cfg(test)]
 mod tests;
 
 mod version;
+use regex::Regex;
 use version::Version;
 
 const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start";
@@ -89,13 +92,14 @@ pub fn check(
     tests_path: &Path,
     compiler_path: &Path,
     lib_path: &Path,
-    bad: &mut bool,
-    verbose: bool,
+    tidy_ctx: TidyCtx,
 ) -> CollectedFeatures {
-    let mut features = collect_lang_features(compiler_path, bad);
+    let mut check = tidy_ctx.start_check("features");
+
+    let mut features = collect_lang_features(compiler_path, &mut check);
     assert!(!features.is_empty());
 
-    let lib_features = get_and_check_lib_features(lib_path, bad, &features);
+    let lib_features = get_and_check_lib_features(lib_path, &mut check, &features);
     assert!(!lib_features.is_empty());
 
     walk_many(
@@ -119,7 +123,7 @@ pub fn check(
 
             for (i, line) in contents.lines().enumerate() {
                 let mut err = |msg: &str| {
-                    tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
+                    check.error(format!("{}:{}: {}", file.display(), i + 1, msg));
                 };
 
                 let gate_test_str = "gate-test-";
@@ -173,7 +177,7 @@ pub fn check(
     }
 
     if !gate_untested.is_empty() {
-        tidy_error!(bad, "Found {} features without a gate test.", gate_untested.len());
+        check.error(format!("Found {} features without a gate test.", gate_untested.len()));
     }
 
     let (version, channel) = get_version_and_channel(src_path);
@@ -187,39 +191,32 @@ pub fn check(
         let file = feature.file.display();
         let line = feature.line;
         if since > version && since != Version::CurrentPlaceholder {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "{file}:{line}: The stabilization version {since} of {kind} feature `{feature_name}` is newer than the current {version}"
-            );
+            ));
         }
         if channel == "nightly" && since == version {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "{file}:{line}: The stabilization version {since} of {kind} feature `{feature_name}` is written out but should be {}",
                 version::VERSION_PLACEHOLDER
-            );
+            ));
         }
         if channel != "nightly" && since == Version::CurrentPlaceholder {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "{file}:{line}: The placeholder use of {kind} feature `{feature_name}` is not allowed on the {channel} channel",
-            );
+            ));
         }
     }
 
-    if *bad {
-        return CollectedFeatures { lib: lib_features, lang: features };
-    }
-
-    if verbose {
+    if !check.is_bad() && check.is_verbose_enabled() {
         let mut lines = Vec::new();
         lines.extend(format_features(&features, "lang"));
         lines.extend(format_features(&lib_features, "lib"));
-
         lines.sort();
-        for line in lines {
-            println!("* {line}");
-        }
+
+        check.verbose_msg(
+            lines.into_iter().map(|l| format!("* {l}")).collect::>().join("\n"),
+        );
     }
 
     CollectedFeatures { lib: lib_features, lang: features }
@@ -273,15 +270,20 @@ fn test_filen_gate<'f>(filen_underscore: &'f str, features: &mut Features) -> Op
     None
 }
 
-pub fn collect_lang_features(base_compiler_path: &Path, bad: &mut bool) -> Features {
+pub fn collect_lang_features(base_compiler_path: &Path, check: &mut RunningCheck) -> Features {
     let mut features = Features::new();
-    collect_lang_features_in(&mut features, base_compiler_path, "accepted.rs", bad);
-    collect_lang_features_in(&mut features, base_compiler_path, "removed.rs", bad);
-    collect_lang_features_in(&mut features, base_compiler_path, "unstable.rs", bad);
+    collect_lang_features_in(&mut features, base_compiler_path, "accepted.rs", check);
+    collect_lang_features_in(&mut features, base_compiler_path, "removed.rs", check);
+    collect_lang_features_in(&mut features, base_compiler_path, "unstable.rs", check);
     features
 }
 
-fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, bad: &mut bool) {
+fn collect_lang_features_in(
+    features: &mut Features,
+    base: &Path,
+    file: &str,
+    check: &mut RunningCheck,
+) {
     let path = base.join("rustc_feature").join("src").join(file);
     let contents = t!(fs::read_to_string(&path));
 
@@ -313,13 +315,11 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
 
         if line.starts_with(FEATURE_GROUP_START_PREFIX) {
             if in_feature_group {
-                tidy_error!(
-                    bad,
-                    "{}:{}: \
+                check.error(format!(
+                    "{}:{line_number}: \
                         new feature group is started without ending the previous one",
-                    path.display(),
-                    line_number,
-                );
+                    path.display()
+                ));
             }
 
             in_feature_group = true;
@@ -351,14 +351,10 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
         let since = match since_str.parse() {
             Ok(since) => Some(since),
             Err(err) => {
-                tidy_error!(
-                    bad,
-                    "{}:{}: failed to parse since: {} ({:?})",
-                    path.display(),
-                    line_number,
-                    since_str,
-                    err,
-                );
+                check.error(format!(
+                    "{}:{line_number}: failed to parse since: {since_str} ({err:?})",
+                    path.display()
+                ));
                 None
             }
         };
@@ -369,13 +365,10 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
                 let correct_index = match prev_names.binary_search(&name) {
                     Ok(_) => {
                         // This only occurs when the feature name has already been declared.
-                        tidy_error!(
-                            bad,
-                            "{}:{}: duplicate feature {}",
-                            path.display(),
-                            line_number,
-                            name,
-                        );
+                        check.error(format!(
+                            "{}:{line_number}: duplicate feature {name}",
+                            path.display()
+                        ));
                         // skip any additional checks for this line
                         continue;
                     }
@@ -396,14 +389,10 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
                     )
                 };
 
-                tidy_error!(
-                    bad,
-                    "{}:{}: feature {} is not sorted by feature name (should be {})",
+                check.error(format!(
+                    "{}:{line_number}: feature {name} is not sorted by feature name (should be {correct_placement})",
                     path.display(),
-                    line_number,
-                    name,
-                    correct_placement,
-                );
+                ));
             }
             prev_names.push(name);
         }
@@ -411,13 +400,10 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
         let issue_str = parts.next().unwrap().trim();
         let tracking_issue = if issue_str.starts_with("None") {
             if level == Status::Unstable && !next_feature_omits_tracking_issue {
-                tidy_error!(
-                    bad,
-                    "{}:{}: no tracking issue for feature {}",
+                check.error(format!(
+                    "{}:{line_number}: no tracking issue for feature {name}",
                     path.display(),
-                    line_number,
-                    name,
-                );
+                ));
             }
             None
         } else {
@@ -426,13 +412,11 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
         };
         match features.entry(name.to_owned()) {
             Entry::Occupied(e) => {
-                tidy_error!(
-                    bad,
-                    "{}:{} feature {name} already specified with status '{}'",
+                check.error(format!(
+                    "{}:{line_number} feature {name} already specified with status '{}'",
                     path.display(),
-                    line_number,
                     e.get().level,
-                );
+                ));
             }
             Entry::Vacant(e) => {
                 e.insert(Feature {
@@ -456,7 +440,7 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
 
 fn get_and_check_lib_features(
     base_src_path: &Path,
-    bad: &mut bool,
+    check: &mut RunningCheck,
     lang_features: &Features,
 ) -> Features {
     let mut lib_features = Features::new();
@@ -467,16 +451,12 @@ fn get_and_check_lib_features(
                     && f.tracking_issue != s.tracking_issue
                     && f.level != Status::Accepted
                 {
-                    tidy_error!(
-                        bad,
-                        "{}:{}: feature gate {} has inconsistent `issue`: \"{}\" mismatches the {} `issue` of \"{}\"",
+                    check.error(format!(
+                        "{}:{line}: feature gate {name} has inconsistent `issue`: \"{}\" mismatches the {display} `issue` of \"{}\"",
                         file.display(),
-                        line,
-                        name,
                         f.tracking_issue_display(),
-                        display,
                         s.tracking_issue_display(),
-                    );
+                    ));
                 }
             };
             check_features(&f, lang_features, "corresponding lang feature");
@@ -484,7 +464,7 @@ fn get_and_check_lib_features(
             lib_features.insert(name.to_owned(), f);
         }
         Err(msg) => {
-            tidy_error!(bad, "{}:{}: {}", file.display(), line, msg);
+            check.error(format!("{}:{line}: {msg}", file.display()));
         }
     });
     lib_features
@@ -623,3 +603,36 @@ fn map_lib_features(
         },
     );
 }
+
+fn should_document(var: &str) -> bool {
+    if var.starts_with("RUSTC_") || var.starts_with("RUST_") || var.starts_with("UNSTABLE_RUSTDOC_")
+    {
+        return true;
+    }
+    ["SDKROOT", "QNX_TARGET", "COLORTERM", "TERM"].contains(&var)
+}
+
+pub fn collect_env_vars(compiler: &Path) -> BTreeSet {
+    let env_var_regex: Regex = Regex::new(r#"env::var(_os)?\("([^"]+)"#).unwrap();
+
+    let mut vars = BTreeSet::new();
+    walk(
+        compiler,
+        // skip build scripts, tests, and non-rust files
+        |path, _is_dir| {
+            filter_dirs(path)
+                || filter_not_rust(path)
+                || path.ends_with("build.rs")
+                || path.ends_with("tests.rs")
+        },
+        &mut |_entry, contents| {
+            for env_var in env_var_regex.captures_iter(contents).map(|c| c.get(2).unwrap().as_str())
+            {
+                if should_document(env_var) {
+                    vars.insert(env_var.to_owned());
+                }
+            }
+        },
+    );
+    vars
+}
diff --git a/src/tools/tidy/src/filenames.rs b/src/tools/tidy/src/filenames.rs
index 53115f4eaa41..a355588fd99c 100644
--- a/src/tools/tidy/src/filenames.rs
+++ b/src/tools/tidy/src/filenames.rs
@@ -10,7 +10,10 @@
 use std::path::Path;
 use std::process::Command;
 
-pub fn check(root_path: &Path, bad: &mut bool) {
+use crate::diagnostics::TidyCtx;
+
+pub fn check(root_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("filenames");
     let stat_output = Command::new("git")
         .arg("-C")
         .arg(root_path)
@@ -20,20 +23,17 @@ pub fn check(root_path: &Path, bad: &mut bool) {
         .stdout;
     for filename in stat_output.split(|&b| b == 0) {
         match str::from_utf8(filename) {
-            Err(_) => tidy_error!(
-                bad,
+            Err(_) => check.error(format!(
                 r#"non-UTF8 file names are not supported: "{}""#,
                 String::from_utf8_lossy(filename),
-            ),
-            Ok(name) if name.chars().any(|c| c.is_control()) => tidy_error!(
-                bad,
+            )),
+            Ok(name) if name.chars().any(|c| c.is_control()) => check.error(format!(
                 r#"control characters are not supported in file names: "{}""#,
                 String::from_utf8_lossy(filename),
-            ),
-            Ok(name) if name.contains(':') => tidy_error!(
-                bad,
+            )),
+            Ok(name) if name.contains(':') => check.error(format!(
                 r#"":" is not supported in file names because of Windows compatibility: "{name}""#,
-            ),
+            )),
             _ => (),
         }
     }
diff --git a/src/tools/tidy/src/fluent_alphabetical.rs b/src/tools/tidy/src/fluent_alphabetical.rs
index 48d14a37514b..7583241ea638 100644
--- a/src/tools/tidy/src/fluent_alphabetical.rs
+++ b/src/tools/tidy/src/fluent_alphabetical.rs
@@ -5,8 +5,11 @@ use std::fs::OpenOptions;
 use std::io::Write;
 use std::path::Path;
 
+use fluent_syntax::ast::Entry;
+use fluent_syntax::parser;
 use regex::Regex;
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 fn message() -> &'static Regex {
@@ -14,42 +17,40 @@ fn message() -> &'static Regex {
 }
 
 fn is_fluent(path: &Path) -> bool {
-    path.extension().is_some_and(|ext| ext == "flt")
+    path.extension().is_some_and(|ext| ext == "ftl")
 }
 
 fn check_alphabetic(
     filename: &str,
     fluent: &str,
-    bad: &mut bool,
+    check: &mut RunningCheck,
     all_defined_msgs: &mut HashMap,
 ) {
-    let mut matches = message().captures_iter(fluent).peekable();
-    while let Some(m) = matches.next() {
-        let name = m.get(1).unwrap();
-        if let Some(defined_filename) = all_defined_msgs.get(name.as_str()) {
-            tidy_error!(
-                bad,
-                "{filename}: message `{}` is already defined in {}",
-                name.as_str(),
-                defined_filename,
-            );
-        }
+    let Ok(resource) = parser::parse(fluent) else {
+        panic!("Errors encountered while parsing fluent file `{filename}`");
+    };
 
-        all_defined_msgs.insert(name.as_str().to_owned(), filename.to_owned());
+    let mut prev: Option<&str> = None;
 
-        if let Some(next) = matches.peek() {
-            let next = next.get(1).unwrap();
-            if name.as_str() > next.as_str() {
-                tidy_error!(
-                    bad,
-                    "{filename}: message `{}` appears before `{}`, but is alphabetically later than it
-run `./x.py test tidy --bless` to sort the file correctly",
-                    name.as_str(),
-                    next.as_str()
-                );
+    for entry in &resource.body {
+        if let Entry::Message(msg) = entry {
+            let name: &str = msg.id.name;
+            if let Some(defined_filename) = all_defined_msgs.get(name) {
+                check.error(format!(
+                    "{filename}: message `{name}` is already defined in {defined_filename}",
+                ));
+            } else {
+                all_defined_msgs.insert(name.to_string(), filename.to_owned());
             }
-        } else {
-            break;
+            if let Some(prev) = prev
+                && prev > name
+            {
+                check.error(format!(
+                    "{filename}: message `{prev}` appears before `{name}`, but is alphabetically \
+later than it. Run `./x.py test tidy --bless` to sort the file correctly",
+                ));
+            }
+            prev = Some(name);
         }
     }
 }
@@ -57,7 +58,7 @@ run `./x.py test tidy --bless` to sort the file correctly",
 fn sort_messages(
     filename: &str,
     fluent: &str,
-    bad: &mut bool,
+    check: &mut RunningCheck,
     all_defined_msgs: &mut HashMap,
 ) -> String {
     let mut chunks = vec![];
@@ -65,12 +66,10 @@ fn sort_messages(
     for line in fluent.lines() {
         if let Some(name) = message().find(line) {
             if let Some(defined_filename) = all_defined_msgs.get(name.as_str()) {
-                tidy_error!(
-                    bad,
-                    "{filename}: message `{}` is already defined in {}",
+                check.error(format!(
+                    "{filename}: message `{}` is already defined in {defined_filename}",
                     name.as_str(),
-                    defined_filename,
-                );
+                ));
             }
 
             all_defined_msgs.insert(name.as_str().to_owned(), filename.to_owned());
@@ -88,7 +87,10 @@ fn sort_messages(
     out
 }
 
-pub fn check(path: &Path, bless: bool, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("fluent_alphabetical").path(path));
+    let bless = tidy_ctx.is_bless_enabled();
+
     let mut all_defined_msgs = HashMap::new();
     walk(
         path,
@@ -98,7 +100,7 @@ pub fn check(path: &Path, bless: bool, bad: &mut bool) {
                 let sorted = sort_messages(
                     ent.path().to_str().unwrap(),
                     contents,
-                    bad,
+                    &mut check,
                     &mut all_defined_msgs,
                 );
                 if sorted != contents {
@@ -110,12 +112,14 @@ pub fn check(path: &Path, bless: bool, bad: &mut bool) {
                 check_alphabetic(
                     ent.path().to_str().unwrap(),
                     contents,
-                    bad,
+                    &mut check,
                     &mut all_defined_msgs,
                 );
             }
         },
     );
 
-    crate::fluent_used::check(path, all_defined_msgs, bad);
+    assert!(!all_defined_msgs.is_empty());
+
+    crate::fluent_used::check(path, all_defined_msgs, tidy_ctx);
 }
diff --git a/src/tools/tidy/src/fluent_lowercase.rs b/src/tools/tidy/src/fluent_lowercase.rs
new file mode 100644
index 000000000000..690b733a5b64
--- /dev/null
+++ b/src/tools/tidy/src/fluent_lowercase.rs
@@ -0,0 +1,65 @@
+//! Checks that the error messages start with a lowercased letter (except when allowed to).
+
+use std::path::Path;
+
+use fluent_syntax::ast::{Entry, Message, PatternElement};
+
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
+use crate::walk::{filter_dirs, walk};
+
+#[rustfmt::skip]
+const ALLOWED_CAPITALIZED_WORDS: &[&str] = &[
+    // tidy-alphabetical-start
+    "ABI",
+    "ABIs",
+    "ADT",
+    "C",
+    "CGU",
+    "Ferris",
+    "MIR",
+    "OK",
+    "Rust",
+    "VS", // VS Code
+    // tidy-alphabetical-end
+];
+
+fn filter_fluent(path: &Path) -> bool {
+    if let Some(ext) = path.extension() { ext.to_str() != Some("ftl") } else { true }
+}
+
+fn is_allowed_capitalized_word(msg: &str) -> bool {
+    ALLOWED_CAPITALIZED_WORDS.iter().any(|word| {
+        msg.strip_prefix(word)
+            .map(|tail| tail.chars().next().map(|c| c == '-' || c.is_whitespace()).unwrap_or(true))
+            .unwrap_or_default()
+    })
+}
+
+fn check_lowercase(filename: &str, contents: &str, check: &mut RunningCheck) {
+    let (Ok(parse) | Err((parse, _))) = fluent_syntax::parser::parse(contents);
+
+    for entry in &parse.body {
+        if let Entry::Message(msg) = entry
+            && let Message { value: Some(pattern), .. } = msg
+            && let [first_pattern, ..] = &pattern.elements[..]
+            && let PatternElement::TextElement { value } = first_pattern
+            && value.chars().next().is_some_and(char::is_uppercase)
+            && !is_allowed_capitalized_word(value)
+        {
+            check.error(format!(
+                "{filename}: message `{value}` starts with an uppercase letter. Fix it or add it to `ALLOWED_CAPITALIZED_WORDS`"
+            ));
+        }
+    }
+}
+
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("fluent_lowercase").path(path));
+    walk(
+        path,
+        |path, is_dir| filter_dirs(path) || (!is_dir && filter_fluent(path)),
+        &mut |ent, contents| {
+            check_lowercase(ent.path().to_str().unwrap(), contents, &mut check);
+        },
+    );
+}
diff --git a/src/tools/tidy/src/fluent_period.rs b/src/tools/tidy/src/fluent_period.rs
index 836b5699289f..e7d59e2ce2a9 100644
--- a/src/tools/tidy/src/fluent_period.rs
+++ b/src/tools/tidy/src/fluent_period.rs
@@ -4,6 +4,7 @@ use std::path::Path;
 
 use fluent_syntax::ast::{Entry, PatternElement};
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 fn filter_fluent(path: &Path) -> bool {
@@ -20,7 +21,7 @@ const ALLOWLIST: &[&str] = &[
     "incremental_corrupt_file",
 ];
 
-fn check_period(filename: &str, contents: &str, bad: &mut bool) {
+fn check_period(filename: &str, contents: &str, check: &mut RunningCheck) {
     if filename.contains("codegen") {
         // FIXME: Too many codegen messages have periods right now...
         return;
@@ -40,7 +41,7 @@ fn check_period(filename: &str, contents: &str, bad: &mut bool) {
                 if value.ends_with(".") && !value.ends_with("...") {
                     let ll = find_line(contents, value);
                     let name = m.id.name;
-                    tidy_error!(bad, "{filename}:{ll}: message `{name}` ends in a period");
+                    check.error(format!("{filename}:{ll}: message `{name}` ends in a period"));
                 }
             }
 
@@ -56,7 +57,7 @@ fn check_period(filename: &str, contents: &str, bad: &mut bool) {
                 {
                     let ll = find_line(contents, value);
                     let name = attr.id.name;
-                    tidy_error!(bad, "{filename}:{ll}: attr `{name}` ends in a period");
+                    check.error(format!("{filename}:{ll}: attr `{name}` ends in a period"));
                 }
             }
         }
@@ -74,12 +75,14 @@ fn find_line(haystack: &str, needle: &str) -> usize {
     1
 }
 
-pub fn check(path: &Path, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("fluent_period").path(path));
+
     walk(
         path,
         |path, is_dir| filter_dirs(path) || (!is_dir && filter_fluent(path)),
         &mut |ent, contents| {
-            check_period(ent.path().to_str().unwrap(), contents, bad);
+            check_period(ent.path().to_str().unwrap(), contents, &mut check);
         },
     );
 }
diff --git a/src/tools/tidy/src/fluent_used.rs b/src/tools/tidy/src/fluent_used.rs
index 909bf482ddfc..75da1d7ea49d 100644
--- a/src/tools/tidy/src/fluent_used.rs
+++ b/src/tools/tidy/src/fluent_used.rs
@@ -3,6 +3,7 @@
 use std::collections::HashMap;
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 fn filter_used_messages(
@@ -27,13 +28,15 @@ fn filter_used_messages(
     }
 }
 
-pub fn check(path: &Path, mut all_defined_msgs: HashMap, bad: &mut bool) {
+pub fn check(path: &Path, mut all_defined_msgs: HashMap, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("fluent_used").path(path));
+
     let mut msgs_appear_only_once = HashMap::new();
     walk(path, |path, _| filter_dirs(path), &mut |_, contents| {
         filter_used_messages(contents, &mut all_defined_msgs, &mut msgs_appear_only_once);
     });
 
     for (name, filename) in msgs_appear_only_once {
-        tidy_error!(bad, "{filename}: message `{}` is not used", name,);
+        check.error(format!("{filename}: message `{name}` is not used"));
     }
 }
diff --git a/src/tools/tidy/src/gcc_submodule.rs b/src/tools/tidy/src/gcc_submodule.rs
index 5d726c3ea48f..46b816f5c9d0 100644
--- a/src/tools/tidy/src/gcc_submodule.rs
+++ b/src/tools/tidy/src/gcc_submodule.rs
@@ -4,7 +4,11 @@
 use std::path::Path;
 use std::process::Command;
 
-pub fn check(root_path: &Path, compiler_path: &Path, bad: &mut bool) {
+use crate::diagnostics::TidyCtx;
+
+pub fn check(root_path: &Path, compiler_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("gcc_submodule");
+
     let cg_gcc_version_path = compiler_path.join("rustc_codegen_gcc/libgccjit.version");
     let cg_gcc_version = std::fs::read_to_string(&cg_gcc_version_path)
         .unwrap_or_else(|_| {
@@ -24,6 +28,12 @@ pub fn check(root_path: &Path, compiler_path: &Path, bad: &mut bool) {
         .output()
         .expect("Cannot determine git SHA of the src/gcc checkout");
 
+    // Git is not available or we are in a tarball
+    if !git_output.status.success() {
+        check.message("Cannot figure out the SHA of the GCC submodule");
+        return;
+    }
+
     // This can return e.g.
     // -e607be166673a8de9fc07f6f02c60426e556c5f2 src/gcc
     //  e607be166673a8de9fc07f6f02c60426e556c5f2 src/gcc (master-e607be166673a8de9fc07f6f02c60426e556c5f2.e607be)
@@ -37,12 +47,11 @@ pub fn check(root_path: &Path, compiler_path: &Path, bad: &mut bool) {
     // The SHA can start with + if the submodule is modified or - if it is not checked out.
     let gcc_submodule_sha = git_output.trim_start_matches(['+', '-']);
     if gcc_submodule_sha != cg_gcc_version {
-        *bad = true;
-        eprintln!(
+        check.error(format!(
             r#"Commit SHA of the src/gcc submodule (`{gcc_submodule_sha}`) does not match the required GCC version of the GCC codegen backend (`{cg_gcc_version}`).
 Make sure to set the src/gcc submodule to commit {cg_gcc_version}.
 The GCC codegen backend commit is configured at {}."#,
             cg_gcc_version_path.display(),
-        );
+        ));
     }
 }
diff --git a/src/tools/tidy/src/issues.txt b/src/tools/tidy/src/issues.txt
index ee06707415f5..849dcb9e88fb 100644
--- a/src/tools/tidy/src/issues.txt
+++ b/src/tools/tidy/src/issues.txt
@@ -2021,7 +2021,6 @@ ui/parser/issues/issue-5806.rs
 ui/parser/issues/issue-58094-missing-right-square-bracket.rs
 ui/parser/issues/issue-58856-1.rs
 ui/parser/issues/issue-58856-2.rs
-ui/parser/issues/issue-59418.rs
 ui/parser/issues/issue-60075.rs
 ui/parser/issues/issue-61858.rs
 ui/parser/issues/issue-62524.rs
diff --git a/src/tools/tidy/src/known_bug.rs b/src/tools/tidy/src/known_bug.rs
index e1921715ab92..e6038341ba96 100644
--- a/src/tools/tidy/src/known_bug.rs
+++ b/src/tools/tidy/src/known_bug.rs
@@ -2,9 +2,11 @@
 
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::*;
 
-pub fn check(filepath: &Path, bad: &mut bool) {
+pub fn check(filepath: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("known_bug").path(filepath));
     walk(filepath, |path, _is_dir| filter_not_rust(path), &mut |entry, contents| {
         let file: &Path = entry.path();
 
@@ -19,11 +21,10 @@ pub fn check(filepath: &Path, bad: &mut bool) {
             [.., "tests", "crashes", "auxiliary", _aux_file_rs]
         ) && !contents.lines().any(|line| line.starts_with("//@ known-bug: "))
         {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "{} crash/ice test does not have a \"//@ known-bug: \" directive",
                 file.display()
-            );
+            ));
         }
     });
 }
diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs
index 4ea9d051ddb5..756f9790e04a 100644
--- a/src/tools/tidy/src/lib.rs
+++ b/src/tools/tidy/src/lib.rs
@@ -4,12 +4,15 @@
 //! to be used by tools.
 
 use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
 use std::process::Command;
+use std::{env, io};
 
 use build_helper::ci::CiEnv;
 use build_helper::git::{GitConfig, get_closest_upstream_commit};
 use build_helper::stage0_parser::{Stage0Config, parse_stage0_file};
-use termcolor::WriteColor;
+
+use crate::diagnostics::{RunningCheck, TidyCtx};
 
 macro_rules! static_regex {
     ($re:literal) => {{
@@ -41,35 +44,6 @@ macro_rules! t {
     };
 }
 
-macro_rules! tidy_error {
-    ($bad:expr, $($fmt:tt)*) => ({
-        $crate::tidy_error(&format_args!($($fmt)*).to_string()).expect("failed to output error");
-        *$bad = true;
-    });
-}
-
-macro_rules! tidy_error_ext {
-    ($tidy_error:path, $bad:expr, $($fmt:tt)*) => ({
-        $tidy_error(&format_args!($($fmt)*).to_string()).expect("failed to output error");
-        *$bad = true;
-    });
-}
-
-fn tidy_error(args: &str) -> std::io::Result<()> {
-    use std::io::Write;
-
-    use termcolor::{Color, ColorChoice, ColorSpec, StandardStream};
-
-    let mut stderr = StandardStream::stdout(ColorChoice::Auto);
-    stderr.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?;
-
-    write!(&mut stderr, "tidy error")?;
-    stderr.set_color(&ColorSpec::new())?;
-
-    writeln!(&mut stderr, ": {args}")?;
-    Ok(())
-}
-
 pub struct CiInfo {
     pub git_merge_commit_email: String,
     pub nightly_branch: String,
@@ -78,7 +52,9 @@ pub struct CiInfo {
 }
 
 impl CiInfo {
-    pub fn new(bad: &mut bool) -> Self {
+    pub fn new(tidy_ctx: TidyCtx) -> Self {
+        let mut check = tidy_ctx.start_check("CI history");
+
         let stage0 = parse_stage0_file();
         let Stage0Config { nightly_branch, git_merge_commit_email, .. } = stage0.config;
 
@@ -91,11 +67,14 @@ impl CiInfo {
         let base_commit = match get_closest_upstream_commit(None, &info.git_config(), info.ci_env) {
             Ok(Some(commit)) => Some(commit),
             Ok(None) => {
-                info.error_if_in_ci("no base commit found", bad);
+                info.error_if_in_ci("no base commit found", &mut check);
                 None
             }
             Err(error) => {
-                info.error_if_in_ci(&format!("failed to retrieve base commit: {error}"), bad);
+                info.error_if_in_ci(
+                    &format!("failed to retrieve base commit: {error}"),
+                    &mut check,
+                );
                 None
             }
         };
@@ -110,12 +89,11 @@ impl CiInfo {
         }
     }
 
-    pub fn error_if_in_ci(&self, msg: &str, bad: &mut bool) {
+    pub fn error_if_in_ci(&self, msg: &str, check: &mut RunningCheck) {
         if self.ci_env.is_running_in_ci() {
-            *bad = true;
-            eprintln!("tidy check error: {msg}");
+            check.error(msg);
         } else {
-            eprintln!("tidy check warning: {msg}. Some checks will be skipped.");
+            check.warning(format!("{msg}. Some checks will be skipped."));
         }
     }
 }
@@ -180,10 +158,82 @@ pub fn files_modified(ci_info: &CiInfo, pred: impl Fn(&str) -> bool) -> bool {
     !v.is_empty()
 }
 
+/// If the given executable is installed with the given version, use that,
+/// otherwise install via cargo.
+pub fn ensure_version_or_cargo_install(
+    build_dir: &Path,
+    cargo: &Path,
+    pkg_name: &str,
+    bin_name: &str,
+    version: &str,
+) -> io::Result {
+    let tool_root_dir = build_dir.join("misc-tools");
+    let tool_bin_dir = tool_root_dir.join("bin");
+    let bin_path = tool_bin_dir.join(bin_name).with_extension(env::consts::EXE_EXTENSION);
+
+    // ignore the process exit code here and instead just let the version number check fail.
+    // we also importantly don't return if the program wasn't installed,
+    // instead we want to continue to the fallback.
+    'ck: {
+        // FIXME: rewrite as if-let chain once this crate is 2024 edition.
+        let Ok(output) = Command::new(&bin_path).arg("--version").output() else {
+            break 'ck;
+        };
+        let Ok(s) = str::from_utf8(&output.stdout) else {
+            break 'ck;
+        };
+        let Some(v) = s.trim().split_whitespace().last() else {
+            break 'ck;
+        };
+        if v == version {
+            return Ok(bin_path);
+        }
+    }
+
+    eprintln!("building external tool {bin_name} from package {pkg_name}@{version}");
+    // use --force to ensure that if the required version is bumped, we update it.
+    // use --target-dir to ensure we have a build cache so repeated invocations aren't slow.
+    // modify PATH so that cargo doesn't print a warning telling the user to modify the path.
+    let mut cmd = Command::new(cargo);
+    cmd.args(["install", "--locked", "--force", "--quiet"])
+        .arg("--root")
+        .arg(&tool_root_dir)
+        .arg("--target-dir")
+        .arg(tool_root_dir.join("target"))
+        .arg(format!("{pkg_name}@{version}"))
+        .env(
+            "PATH",
+            env::join_paths(
+                env::split_paths(&env::var("PATH").unwrap())
+                    .chain(std::iter::once(tool_bin_dir.clone())),
+            )
+            .expect("build dir contains invalid char"),
+        );
+
+    // On CI, we set opt-level flag for quicker installation.
+    // Since lower opt-level decreases the tool's performance,
+    // we don't set this option on local.
+    if CiEnv::is_ci() {
+        cmd.env("RUSTFLAGS", "-Copt-level=0");
+    }
+
+    let cargo_exit_code = cmd.spawn()?.wait()?;
+    if !cargo_exit_code.success() {
+        return Err(io::Error::other("cargo install failed"));
+    }
+    assert!(
+        matches!(bin_path.try_exists(), Ok(true)),
+        "cargo install did not produce the expected binary"
+    );
+    eprintln!("finished building tool {bin_name}");
+    Ok(bin_path)
+}
+
 pub mod alphabetical;
 pub mod bins;
 pub mod debug_artifacts;
 pub mod deps;
+pub mod diagnostics;
 pub mod edition;
 pub mod error_codes;
 pub mod extdeps;
@@ -191,6 +241,7 @@ pub mod extra_checks;
 pub mod features;
 pub mod filenames;
 pub mod fluent_alphabetical;
+pub mod fluent_lowercase;
 pub mod fluent_period;
 mod fluent_used;
 pub mod gcc_submodule;
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index cd2567ddb64b..94c24f11ed12 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -8,10 +8,10 @@ use std::collections::VecDeque;
 use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::str::FromStr;
-use std::sync::atomic::{AtomicBool, Ordering};
 use std::thread::{self, ScopedJoinHandle, scope};
 use std::{env, process};
 
+use tidy::diagnostics::{COLOR_ERROR, COLOR_SUCCESS, TidyCtx, TidyFlags, output_message};
 use tidy::*;
 
 fn main() {
@@ -46,13 +46,12 @@ fn main() {
         None => (&args[..], [].as_slice()),
     };
     let verbose = cfg_args.iter().any(|s| *s == "--verbose");
-    let bless = cfg_args.iter().any(|s| *s == "--bless");
     let extra_checks =
         cfg_args.iter().find(|s| s.starts_with("--extra-checks=")).map(String::as_str);
 
-    let mut bad = false;
-    let ci_info = CiInfo::new(&mut bad);
-    let bad = std::sync::Arc::new(AtomicBool::new(bad));
+    let tidy_flags = TidyFlags::new(cfg_args);
+    let tidy_ctx = TidyCtx::new(&root_path, verbose, tidy_flags);
+    let ci_info = CiInfo::new(tidy_ctx.clone());
 
     let drain_handles = |handles: &mut VecDeque>| {
         // poll all threads for completion before awaiting the oldest one
@@ -87,12 +86,9 @@ fn main() {
             (@ $p:ident, name=$name:expr $(, $args:expr)* ) => {
                 drain_handles(&mut handles);
 
+                let tidy_ctx = tidy_ctx.clone();
                 let handle = thread::Builder::new().name($name).spawn_scoped(s, || {
-                    let mut flag = false;
-                    $p::check($($args, )* &mut flag);
-                    if (flag) {
-                        bad.store(true, Ordering::Relaxed);
-                    }
+                    $p::check($($args, )* tidy_ctx);
                 }).unwrap();
                 handles.push_back(handle);
             }
@@ -101,15 +97,15 @@ fn main() {
         check!(target_specific_tests, &tests_path);
 
         // Checks that are done on the cargo workspace.
-        check!(deps, &root_path, &cargo, bless);
+        check!(deps, &root_path, &cargo);
         check!(extdeps, &root_path);
 
         // Checks over tests.
         check!(tests_placement, &root_path);
         check!(tests_revision_unpaired_stdout_stderr, &tests_path);
         check!(debug_artifacts, &tests_path);
-        check!(ui_tests, &root_path, bless);
-        check!(mir_opt_tests, &tests_path, bless);
+        check!(ui_tests, &root_path);
+        check!(mir_opt_tests, &tests_path);
         check!(rustdoc_gui_tests, &tests_path);
         check!(rustdoc_css_themes, &librustdoc_path);
         check!(rustdoc_templates, &librustdoc_path);
@@ -118,9 +114,10 @@ fn main() {
         check!(unknown_revision, &tests_path);
 
         // Checks that only make sense for the compiler.
-        check!(error_codes, &root_path, &[&compiler_path, &librustdoc_path], verbose, &ci_info);
-        check!(fluent_alphabetical, &compiler_path, bless);
+        check!(error_codes, &root_path, &[&compiler_path, &librustdoc_path], &ci_info);
+        check!(fluent_alphabetical, &compiler_path);
         check!(fluent_period, &compiler_path);
+        check!(fluent_lowercase, &compiler_path);
         check!(target_policy, &root_path);
         check!(gcc_submodule, &root_path, &compiler_path);
 
@@ -154,25 +151,12 @@ fn main() {
         check!(x_version, &root_path, &cargo);
 
         check!(triagebot, &root_path);
-
         check!(filenames, &root_path);
 
         let collected = {
             drain_handles(&mut handles);
 
-            let mut flag = false;
-            let r = features::check(
-                &src_path,
-                &tests_path,
-                &compiler_path,
-                &library_path,
-                &mut flag,
-                verbose,
-            );
-            if flag {
-                bad.store(true, Ordering::Relaxed);
-            }
-            r
+            features::check(&src_path, &tests_path, &compiler_path, &library_path, tidy_ctx.clone())
         };
         check!(unstable_book, &src_path, collected);
 
@@ -184,14 +168,28 @@ fn main() {
             &librustdoc_path,
             &tools_path,
             &npm,
-            bless,
+            &cargo,
             extra_checks,
             pos_args
         );
     });
 
-    if bad.load(Ordering::Relaxed) {
-        eprintln!("some tidy checks failed");
+    let failed_checks = tidy_ctx.into_failed_checks();
+    if !failed_checks.is_empty() {
+        let mut failed: Vec =
+            failed_checks.into_iter().map(|c| c.id().to_string()).collect();
+        failed.sort();
+        output_message(
+            &format!(
+                "The following check{} failed: {}",
+                if failed.len() > 1 { "s" } else { "" },
+                failed.join(", ")
+            ),
+            None,
+            Some(COLOR_ERROR),
+        );
         process::exit(1);
+    } else {
+        output_message("All tidy checks succeeded", None, Some(COLOR_SUCCESS));
     }
 }
diff --git a/src/tools/tidy/src/mir_opt_tests.rs b/src/tools/tidy/src/mir_opt_tests.rs
index 6119eb58383e..afc1dc2a1860 100644
--- a/src/tools/tidy/src/mir_opt_tests.rs
+++ b/src/tools/tidy/src/mir_opt_tests.rs
@@ -5,9 +5,10 @@ use std::path::{Path, PathBuf};
 
 use miropt_test_tools::PanicStrategy;
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::walk::walk_no_read;
 
-fn check_unused_files(path: &Path, bless: bool, bad: &mut bool) {
+fn check_unused_files(path: &Path, bless: bool, check: &mut RunningCheck) {
     let mut rs_files = Vec::::new();
     let mut output_files = HashSet::::new();
 
@@ -37,18 +38,17 @@ fn check_unused_files(path: &Path, bless: bool, bad: &mut bool) {
 
     for extra in output_files {
         if !bless {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "the following output file is not associated with any mir-opt test, you can remove it: {}",
                 extra.display()
-            );
+            ));
         } else {
             let _ = std::fs::remove_file(extra);
         }
     }
 }
 
-fn check_dash_files(path: &Path, bless: bool, bad: &mut bool) {
+fn check_dash_files(path: &Path, bless: bool, check: &mut RunningCheck) {
     for file in walkdir::WalkDir::new(path.join("mir-opt"))
         .into_iter()
         .filter_map(Result::ok)
@@ -60,11 +60,10 @@ fn check_dash_files(path: &Path, bless: bool, bad: &mut bool) {
             && name.contains('-')
         {
             if !bless {
-                tidy_error!(
-                    bad,
+                check.error(format!(
                     "mir-opt test files should not have dashes in them: {}",
                     path.display()
-                );
+                ));
             } else {
                 let new_name = name.replace('-', "_");
                 let mut new_path = path.to_owned();
@@ -75,7 +74,10 @@ fn check_dash_files(path: &Path, bless: bool, bad: &mut bool) {
     }
 }
 
-pub fn check(path: &Path, bless: bool, bad: &mut bool) {
-    check_unused_files(path, bless, bad);
-    check_dash_files(path, bless, bad);
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("mir_opt_tests").path(path));
+    let bless = tidy_ctx.is_bless_enabled();
+
+    check_unused_files(path, bless, &mut check);
+    check_dash_files(path, bless, &mut check);
 }
diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs
index 5b8b44429bbc..dfca2cda9a0f 100644
--- a/src/tools/tidy/src/pal.rs
+++ b/src/tools/tidy/src/pal.rs
@@ -32,6 +32,7 @@
 
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 // Paths that may contain platform-specific code.
@@ -53,6 +54,7 @@ const EXCEPTION_PATHS: &[&str] = &[
     // core::ffi contains platform-specific type and linkage configuration
     "library/core/src/ffi/mod.rs",
     "library/core/src/ffi/primitives.rs",
+    "library/core/src/os", // Platform-specific public interfaces
     "library/std/src/sys", // Platform-specific code for std lives here.
     "library/std/src/os",  // Platform-specific public interfaces
     // Temporary `std` exceptions
@@ -66,7 +68,9 @@ const EXCEPTION_PATHS: &[&str] = &[
     "library/std/src/io/error.rs", // Repr unpacked needed for UEFI
 ];
 
-pub fn check(path: &Path, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("pal").path(path));
+
     // Sanity check that the complex parsing here works.
     let mut saw_target_arch = false;
     let mut saw_cfg_bang = false;
@@ -87,7 +91,7 @@ pub fn check(path: &Path, bad: &mut bool) {
             return;
         }
 
-        check_cfgs(contents, file, bad, &mut saw_target_arch, &mut saw_cfg_bang);
+        check_cfgs(contents, file, &mut check, &mut saw_target_arch, &mut saw_cfg_bang);
     });
 
     assert!(saw_target_arch);
@@ -97,7 +101,7 @@ pub fn check(path: &Path, bad: &mut bool) {
 fn check_cfgs(
     contents: &str,
     file: &Path,
-    bad: &mut bool,
+    check: &mut RunningCheck,
     saw_target_arch: &mut bool,
     saw_cfg_bang: &mut bool,
 ) {
@@ -114,7 +118,7 @@ fn check_cfgs(
             Ok(_) => unreachable!(),
             Err(i) => i + 1,
         };
-        tidy_error!(bad, "{}:{}: platform-specific cfg: {}", file.display(), line, cfg);
+        check.error(format!("{}:{line}: platform-specific cfg: {cfg}", file.display()));
     };
 
     for (idx, cfg) in cfgs {
diff --git a/src/tools/tidy/src/rustdoc_css_themes.rs b/src/tools/tidy/src/rustdoc_css_themes.rs
index af36f9ba58e0..f3aecc15cd9a 100644
--- a/src/tools/tidy/src/rustdoc_css_themes.rs
+++ b/src/tools/tidy/src/rustdoc_css_themes.rs
@@ -3,7 +3,11 @@
 
 use std::path::Path;
 
-pub fn check(librustdoc_path: &Path, bad: &mut bool) {
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
+
+pub fn check(librustdoc_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("rustdoc_css_themes").path(librustdoc_path));
+
     let rustdoc_css = "html/static/css/rustdoc.css";
     let noscript_css = "html/static/css/noscript.css";
     let rustdoc_css_contents = std::fs::read_to_string(librustdoc_path.join(rustdoc_css))
@@ -14,13 +18,13 @@ pub fn check(librustdoc_path: &Path, bad: &mut bool) {
         "light",
         rustdoc_css_contents.lines().enumerate().map(|(i, l)| (i + 1, l.trim())),
         noscript_css_contents.lines().enumerate().map(|(i, l)| (i + 1, l.trim())),
-        bad,
+        &mut check,
     );
     compare_themes_from_files(
         "dark",
         rustdoc_css_contents.lines().enumerate(),
         noscript_css_contents.lines().enumerate(),
-        bad,
+        &mut check,
     );
 }
 
@@ -28,7 +32,7 @@ fn compare_themes_from_files<'a>(
     name: &str,
     mut rustdoc_css_lines: impl Iterator,
     mut noscript_css_lines: impl Iterator,
-    bad: &mut bool,
+    check: &mut RunningCheck,
 ) {
     let begin_theme_pat = format!("/* Begin theme: {name}");
     let mut found_theme = None;
@@ -38,10 +42,9 @@ fn compare_themes_from_files<'a>(
             continue;
         }
         if let Some(found_theme) = found_theme {
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "rustdoc.css contains two {name} themes on lines {rustdoc_css_line_number} and {found_theme}",
-            );
+            ));
             return;
         }
         found_theme = Some(rustdoc_css_line_number);
@@ -50,14 +53,13 @@ fn compare_themes_from_files<'a>(
                 continue;
             }
             if let Some(found_theme_noscript) = found_theme_noscript {
-                tidy_error!(
-                    bad,
+                check.error(format!(
                     "noscript.css contains two {name} themes on lines {noscript_css_line_number} and {found_theme_noscript}",
-                );
+                ));
                 return;
             }
             found_theme_noscript = Some(noscript_css_line_number);
-            compare_themes(name, &mut rustdoc_css_lines, &mut noscript_css_lines, bad);
+            compare_themes(name, &mut rustdoc_css_lines, &mut noscript_css_lines, check);
         }
     }
 }
@@ -66,7 +68,7 @@ fn compare_themes<'a>(
     name: &str,
     rustdoc_css_lines: impl Iterator,
     noscript_css_lines: impl Iterator,
-    bad: &mut bool,
+    check: &mut RunningCheck,
 ) {
     let end_theme_pat = format!("/* End theme: {name}");
     for (
@@ -90,12 +92,11 @@ fn compare_themes<'a>(
             break;
         }
         if rustdoc_css_line != noscript_css_line {
-            tidy_error!(
-                bad,
-                "noscript.css:{noscript_css_line_number} and rustdoc.css:{rustdoc_css_line_number} contain copies of {name} theme that are not the same",
-            );
-            eprintln!("- {noscript_css_line}");
-            eprintln!("+ {rustdoc_css_line}");
+            check.error(format!(
+                r#"noscript.css:{noscript_css_line_number} and rustdoc.css:{rustdoc_css_line_number} contain copies of {name} theme that are not the same
+- {noscript_css_line}
++ {rustdoc_css_line}"#,
+            ));
             return;
         }
     }
diff --git a/src/tools/tidy/src/rustdoc_gui_tests.rs b/src/tools/tidy/src/rustdoc_gui_tests.rs
index 3b995f219d26..848250e18e74 100644
--- a/src/tools/tidy/src/rustdoc_gui_tests.rs
+++ b/src/tools/tidy/src/rustdoc_gui_tests.rs
@@ -2,18 +2,21 @@
 
 use std::path::Path;
 
-pub fn check(path: &Path, bad: &mut bool) {
+use crate::diagnostics::{CheckId, TidyCtx};
+
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("rustdoc_gui_tests").path(path));
+
     crate::walk::walk(
         &path.join("rustdoc-gui"),
         |p, is_dir| !is_dir && p.extension().is_none_or(|e| e != "goml"),
         &mut |entry, content| {
             for line in content.lines() {
                 if !line.starts_with("// ") {
-                    tidy_error!(
-                        bad,
+                    check.error(format!(
                         "{}: rustdoc-gui tests must start with a small description",
                         entry.path().display(),
-                    );
+                    ));
                     return;
                 } else if line.starts_with("// ") {
                     let parts = line[2..].trim();
diff --git a/src/tools/tidy/src/rustdoc_json.rs b/src/tools/tidy/src/rustdoc_json.rs
index 722e1ebd0cad..b8fb04f2d4e1 100644
--- a/src/tools/tidy/src/rustdoc_json.rs
+++ b/src/tools/tidy/src/rustdoc_json.rs
@@ -4,21 +4,26 @@
 use std::path::Path;
 use std::str::FromStr;
 
+use crate::diagnostics::{CheckId, TidyCtx};
+
 const RUSTDOC_JSON_TYPES: &str = "src/rustdoc-json-types";
 
-pub fn check(src_path: &Path, ci_info: &crate::CiInfo, bad: &mut bool) {
-    println!("Checking tidy rustdoc_json...");
+pub fn check(src_path: &Path, ci_info: &crate::CiInfo, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("rustdoc_json").path(src_path));
+
     let Some(base_commit) = &ci_info.base_commit else {
-        eprintln!("No base commit, skipping rustdoc_json check");
+        check.verbose_msg("No base commit, skipping rustdoc_json check");
         return;
     };
 
     // First we check that `src/rustdoc-json-types` was modified.
-    if !crate::files_modified(ci_info, |p| p == RUSTDOC_JSON_TYPES) {
+    if !crate::files_modified(ci_info, |p| p.starts_with(RUSTDOC_JSON_TYPES)) {
         // `rustdoc-json-types` was not modified so nothing more to check here.
-        println!("`rustdoc-json-types` was not modified.");
         return;
     }
+
+    check.message("`rustdoc-json-types` modified, checking format version");
+
     // Then we check that if `FORMAT_VERSION` was updated, the `Latest feature:` was also updated.
     match crate::git_diff(base_commit, src_path.join("rustdoc-json-types")) {
         Some(output) => {
@@ -45,34 +50,29 @@ pub fn check(src_path: &Path, ci_info: &crate::CiInfo, bad: &mut bool) {
                 }
             }
             if format_version_updated != latest_feature_comment_updated {
-                *bad = true;
-                if latest_feature_comment_updated {
-                    eprintln!(
-                        "error in `rustdoc_json` tidy check: `Latest feature` comment was updated \
-                         whereas `FORMAT_VERSION` wasn't in `{RUSTDOC_JSON_TYPES}/lib.rs`"
-                    );
+                let msg = if latest_feature_comment_updated {
+                    format!(
+                        "`Latest feature` comment was updated whereas `FORMAT_VERSION` wasn't in `{RUSTDOC_JSON_TYPES}/lib.rs`"
+                    )
                 } else {
-                    eprintln!(
-                        "error in `rustdoc_json` tidy check: `Latest feature` comment was not \
-                         updated whereas `FORMAT_VERSION` was in `{RUSTDOC_JSON_TYPES}/lib.rs`"
-                    );
-                }
+                    format!(
+                        "`Latest feature` comment was not updated whereas `FORMAT_VERSION` was in `{RUSTDOC_JSON_TYPES}/lib.rs`"
+                    )
+                };
+                check.error(msg);
             }
             match (new_version, old_version) {
                 (Some(new_version), Some(old_version)) if new_version != old_version + 1 => {
-                    *bad = true;
-                    eprintln!(
-                        "error in `rustdoc_json` tidy check: invalid `FORMAT_VERSION` increase in \
-                         `{RUSTDOC_JSON_TYPES}/lib.rs`, should be `{}`, found `{new_version}`",
+                    check.error(format!(
+                        "invalid `FORMAT_VERSION` increase in `{RUSTDOC_JSON_TYPES}/lib.rs`, should be `{}`, found `{new_version}`",
                         old_version + 1,
-                    );
+                    ));
                 }
                 _ => {}
             }
         }
         None => {
-            *bad = true;
-            eprintln!("error: failed to run `git diff` in rustdoc_json check");
+            check.error("failed to run `git diff` in rustdoc_json check");
         }
     }
 }
diff --git a/src/tools/tidy/src/rustdoc_templates.rs b/src/tools/tidy/src/rustdoc_templates.rs
index 597290a6a9a8..faeb95a92b76 100644
--- a/src/tools/tidy/src/rustdoc_templates.rs
+++ b/src/tools/tidy/src/rustdoc_templates.rs
@@ -6,12 +6,15 @@ use std::path::Path;
 
 use ignore::DirEntry;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::walk;
 
 // Array containing `("beginning of tag", "end of tag")`.
 const TAGS: &[(&str, &str)] = &[("{#", "#}"), ("{%", "%}"), ("{{", "}}")];
 
-pub fn check(librustdoc_path: &Path, bad: &mut bool) {
+pub fn check(librustdoc_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("rustdoc_templates").path(librustdoc_path));
+
     walk(
         &librustdoc_path.join("html/templates"),
         |path, is_dir| is_dir || path.extension().is_none_or(|ext| ext != OsStr::new("html")),
@@ -46,12 +49,11 @@ pub fn check(librustdoc_path: &Path, bad: &mut bool) {
                         })
                     {
                         // It seems like ending this line with a jinja tag is not needed after all.
-                        tidy_error!(
-                            bad,
+                        check.error(format!(
                             "`{}` at line {}: unneeded `{{# #}}` tag at the end of the line",
                             path.path().display(),
                             pos + 1,
-                        );
+                        ));
                     }
                     continue;
                 }
@@ -67,12 +69,11 @@ pub fn check(librustdoc_path: &Path, bad: &mut bool) {
                 }) {
                     None => {
                         // No it's not, let's error.
-                        tidy_error!(
-                            bad,
+                        check.error(format!(
                             "`{}` at line {}: missing `{{# #}}` at the end of the line",
                             path.path().display(),
                             pos + 1,
-                        );
+                        ));
                     }
                     Some(end_tag) => {
                         // We skip the tag.
diff --git a/src/tools/tidy/src/style.rs b/src/tools/tidy/src/style.rs
index fca097c091b7..7bb424d926c2 100644
--- a/src/tools/tidy/src/style.rs
+++ b/src/tools/tidy/src/style.rs
@@ -24,6 +24,7 @@ use std::sync::LazyLock;
 use regex::RegexSetBuilder;
 use rustc_hash::FxHashMap;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
 #[cfg(test)]
@@ -338,7 +339,9 @@ fn is_unexplained_ignore(extension: &str, line: &str) -> bool {
     true
 }
 
-pub fn check(path: &Path, bad: &mut bool) {
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("style").path(path));
+
     fn skip(path: &Path, is_dir: bool) -> bool {
         if path.file_name().is_some_and(|name| name.to_string_lossy().starts_with(".#")) {
             // vim or emacs temporary file
@@ -391,7 +394,7 @@ pub fn check(path: &Path, bad: &mut bool) {
             });
 
         if contents.is_empty() {
-            tidy_error!(bad, "{}: empty file", file.display());
+            check.error(format!("{}: empty file", file.display()));
         }
 
         let extension = file.extension().unwrap().to_string_lossy();
@@ -467,7 +470,7 @@ pub fn check(path: &Path, bad: &mut bool) {
             }
 
             let mut err = |msg: &str| {
-                tidy_error!(bad, "{}:{}: {}", file.display(), i + 1, msg);
+                check.error(format!("{}:{}: {msg}", file.display(), i + 1));
             };
 
             if trimmed.contains("dbg!")
@@ -611,7 +614,7 @@ pub fn check(path: &Path, bad: &mut bool) {
                     && backtick_count % 2 == 1
                 {
                     let mut err = |msg: &str| {
-                        tidy_error!(bad, "{}:{start_line}: {msg}", file.display());
+                        check.error(format!("{}:{start_line}: {msg}", file.display()));
                     };
                     let block_len = (i + 1) - start_line;
                     if block_len == 1 {
@@ -632,12 +635,12 @@ pub fn check(path: &Path, bad: &mut bool) {
         }
         if leading_new_lines {
             let mut err = |_| {
-                tidy_error!(bad, "{}: leading newline", file.display());
+                check.error(format!("{}: leading newline", file.display()));
             };
             suppressible_tidy_err!(err, skip_leading_newlines, "missing leading newline");
         }
         let mut err = |msg: &str| {
-            tidy_error!(bad, "{}: {}", file.display(), msg);
+            check.error(format!("{}: {}", file.display(), msg));
         };
         match trailing_new_lines {
             0 => suppressible_tidy_err!(err, skip_trailing_newlines, "missing trailing newline"),
@@ -650,38 +653,36 @@ pub fn check(path: &Path, bad: &mut bool) {
         };
         if lines > LINES {
             let mut err = |_| {
-                tidy_error!(
-                    bad,
-                    "{}: too many lines ({}) (add `// \
+                check.error(format!(
+                    "{}: too many lines ({lines}) (add `// \
                      ignore-tidy-filelength` to the file to suppress this error)",
                     file.display(),
-                    lines
-                );
+                ));
             };
             suppressible_tidy_err!(err, skip_file_length, "");
         }
 
         if let Directive::Ignore(false) = skip_cr {
-            tidy_error!(bad, "{}: ignoring CR characters unnecessarily", file.display());
+            check.error(format!("{}: ignoring CR characters unnecessarily", file.display()));
         }
         if let Directive::Ignore(false) = skip_tab {
-            tidy_error!(bad, "{}: ignoring tab characters unnecessarily", file.display());
+            check.error(format!("{}: ignoring tab characters unnecessarily", file.display()));
         }
         if let Directive::Ignore(false) = skip_end_whitespace {
-            tidy_error!(bad, "{}: ignoring trailing whitespace unnecessarily", file.display());
+            check.error(format!("{}: ignoring trailing whitespace unnecessarily", file.display()));
         }
         if let Directive::Ignore(false) = skip_trailing_newlines {
-            tidy_error!(bad, "{}: ignoring trailing newlines unnecessarily", file.display());
+            check.error(format!("{}: ignoring trailing newlines unnecessarily", file.display()));
         }
         if let Directive::Ignore(false) = skip_leading_newlines {
-            tidy_error!(bad, "{}: ignoring leading newlines unnecessarily", file.display());
+            check.error(format!("{}: ignoring leading newlines unnecessarily", file.display()));
         }
         if let Directive::Ignore(false) = skip_copyright {
-            tidy_error!(bad, "{}: ignoring copyright unnecessarily", file.display());
+            check.error(format!("{}: ignoring copyright unnecessarily", file.display()));
         }
         // We deliberately do not warn about these being unnecessary,
         // that would just lead to annoying churn.
         let _unused = skip_line_length;
         let _unused = skip_file_length;
-    })
+    });
 }
diff --git a/src/tools/tidy/src/target_policy.rs b/src/tools/tidy/src/target_policy.rs
index 550932dbfdc3..83fbe6a0d8da 100644
--- a/src/tools/tidy/src/target_policy.rs
+++ b/src/tools/tidy/src/target_policy.rs
@@ -5,6 +5,7 @@
 use std::collections::HashSet;
 use std::path::Path;
 
+use crate::diagnostics::TidyCtx;
 use crate::walk::{filter_not_rust, walk};
 
 const TARGET_DEFINITIONS_PATH: &str = "compiler/rustc_target/src/spec/targets/";
@@ -23,7 +24,9 @@ const EXCEPTIONS: &[&str] = &[
     "xtensa_esp32s3_espidf",
 ];
 
-pub fn check(root_path: &Path, bad: &mut bool) {
+pub fn check(root_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("target_policy");
+
     let mut targets_to_find = HashSet::new();
 
     let definitions_path = root_path.join(TARGET_DEFINITIONS_PATH);
@@ -55,7 +58,7 @@ pub fn check(root_path: &Path, bad: &mut bool) {
 
     for target in targets_to_find {
         if !EXCEPTIONS.contains(&target.as_str()) {
-            tidy_error!(bad, "{ASSEMBLY_LLVM_TEST_PATH}: missing assembly test for {target}")
+            check.error(format!("{ASSEMBLY_LLVM_TEST_PATH}: missing assembly test for {target}"));
         }
     }
 }
diff --git a/src/tools/tidy/src/target_specific_tests.rs b/src/tools/tidy/src/target_specific_tests.rs
index b2d5f259eb2d..159d7278bb62 100644
--- a/src/tools/tidy/src/target_specific_tests.rs
+++ b/src/tools/tidy/src/target_specific_tests.rs
@@ -4,6 +4,7 @@
 use std::collections::BTreeMap;
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::iter_header::{HeaderLine, iter_header};
 use crate::walk::filter_not_rust;
 
@@ -16,7 +17,9 @@ struct RevisionInfo<'a> {
     llvm_components: Option>,
 }
 
-pub fn check(tests_path: &Path, bad: &mut bool) {
+pub fn check(tests_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("target-specific-tests").path(tests_path));
+
     crate::walk::walk(tests_path, |path, _is_dir| filter_not_rust(path), &mut |entry, content| {
         if content.contains("// ignore-tidy-target-specific-tests") {
             return;
@@ -44,8 +47,7 @@ pub fn check(tests_path: &Path, bad: &mut bool) {
                 } else if let Some((arch, _)) = v.split_once("-") {
                     info.target_arch.replace(Some(arch));
                 } else {
-                    eprintln!("{file}: seems to have a malformed --target value");
-                    *bad = true;
+                    check.error(format!("{file}: seems to have a malformed --target value"));
                 }
             }
         });
@@ -62,25 +64,22 @@ pub fn check(tests_path: &Path, bad: &mut bool) {
                 (Some(target_arch), None) => {
                     let llvm_component =
                         target_arch.map_or_else(|| "".to_string(), arch_to_llvm_component);
-                    eprintln!(
+                    check.error(format!(
                         "{file}: revision {rev} should specify `{LLVM_COMPONENTS_HEADER} {llvm_component}` as it has `--target` set"
-                    );
-                    *bad = true;
+                    ));
                 }
                 (None, Some(_)) => {
-                    eprintln!(
+                    check.error(format!(
                         "{file}: revision {rev} should not specify `{LLVM_COMPONENTS_HEADER}` as it doesn't need `--target`"
-                    );
-                    *bad = true;
+                    ));
                 }
                 (Some(target_arch), Some(llvm_components)) => {
                     if let Some(target_arch) = target_arch {
                         let llvm_component = arch_to_llvm_component(target_arch);
                         if !llvm_components.contains(&llvm_component.as_str()) {
-                            eprintln!(
+                            check.error(format!(
                                 "{file}: revision {rev} should specify `{LLVM_COMPONENTS_HEADER} {llvm_component}` as it has `--target` set"
-                            );
-                            *bad = true;
+                            ));
                         }
                     }
                 }
diff --git a/src/tools/tidy/src/tests_placement.rs b/src/tools/tidy/src/tests_placement.rs
index 9d0057df8bcd..18b00608fdce 100644
--- a/src/tools/tidy/src/tests_placement.rs
+++ b/src/tools/tidy/src/tests_placement.rs
@@ -1,15 +1,18 @@
 use std::path::Path;
 
+use crate::diagnostics::TidyCtx;
+
 const FORBIDDEN_PATH: &str = "src/test";
 const ALLOWED_PATH: &str = "tests";
 
-pub fn check(root_path: impl AsRef, bad: &mut bool) {
-    if root_path.as_ref().join(FORBIDDEN_PATH).exists() {
-        tidy_error!(
-            bad,
+pub fn check(root_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("tests_placement");
+
+    if root_path.join(FORBIDDEN_PATH).exists() {
+        check.error(format!(
             "Tests have been moved, please move them from {} to {}",
-            root_path.as_ref().join(FORBIDDEN_PATH).display(),
-            root_path.as_ref().join(ALLOWED_PATH).display()
-        )
+            root_path.join(FORBIDDEN_PATH).display(),
+            root_path.join(ALLOWED_PATH).display()
+        ));
     }
 }
diff --git a/src/tools/tidy/src/tests_revision_unpaired_stdout_stderr.rs b/src/tools/tidy/src/tests_revision_unpaired_stdout_stderr.rs
index 02412b6f190e..4d9cb55138af 100644
--- a/src/tools/tidy/src/tests_revision_unpaired_stdout_stderr.rs
+++ b/src/tools/tidy/src/tests_revision_unpaired_stdout_stderr.rs
@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::ffi::OsStr;
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::iter_header::*;
 use crate::walk::*;
 
@@ -21,7 +22,10 @@ const IGNORES: &[&str] = &[
 const EXTENSIONS: &[&str] = &["stdout", "stderr"];
 const SPECIAL_TEST: &str = "tests/ui/command/need-crate-arg-ignore-tidy.x.rs";
 
-pub fn check(tests_path: impl AsRef, bad: &mut bool) {
+pub fn check(tests_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx
+        .start_check(CheckId::new("tests_revision_unpaired_stdout_stderr").path(tests_path));
+
     // Recurse over subdirectories under `tests/`
     walk_dir(tests_path.as_ref(), filter, &mut |entry| {
         // We are inspecting a folder. Collect the paths to interesting files `.rs`, `.stderr`,
@@ -122,12 +126,11 @@ pub fn check(tests_path: impl AsRef, bad: &mut bool) {
                 [] | [_] => return,
                 [_, _] if !expected_revisions.is_empty() => {
                     // Found unrevisioned output files for a revisioned test.
-                    tidy_error!(
-                        bad,
+                    check.error(format!(
                         "found unrevisioned output file `{}` for a revisioned test `{}`",
                         sibling.display(),
                         test_path.display(),
-                    );
+                    ));
                 }
                 [_, _] => return,
                 [_, found_revision, .., extension] => {
@@ -138,13 +141,12 @@ pub fn check(tests_path: impl AsRef, bad: &mut bool) {
                     {
                         // Found some unexpected revision-esque component that is not a known
                         // compare-mode or expected revision.
-                        tidy_error!(
-                            bad,
+                        check.error(format!(
                             "found output file `{}` for unexpected revision `{}` of test `{}`",
                             sibling.display(),
                             found_revision,
                             test_path.display()
-                        );
+                        ));
                     }
                 }
             }
diff --git a/src/tools/tidy/src/triagebot.rs b/src/tools/tidy/src/triagebot.rs
index 305a0b4d2643..01401c94d730 100644
--- a/src/tools/tidy/src/triagebot.rs
+++ b/src/tools/tidy/src/triagebot.rs
@@ -4,7 +4,10 @@ use std::path::Path;
 
 use toml::Value;
 
-pub fn check(path: &Path, bad: &mut bool) {
+use crate::diagnostics::TidyCtx;
+
+pub fn check(path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("triagebot");
     let triagebot_path = path.join("triagebot.toml");
 
     // This check is mostly to catch broken path filters *within* `triagebot.toml`, and not enforce
@@ -19,23 +22,25 @@ pub fn check(path: &Path, bad: &mut bool) {
 
     // Check [mentions."*"] sections, i.e. [mentions."compiler/rustc_const_eval/src/"]
     if let Some(Value::Table(mentions)) = config.get("mentions") {
-        for path_str in mentions.keys() {
+        for (entry_key, entry_val) in mentions.iter() {
+            // If the type is set to something other than "filename", then this is not a path.
+            if entry_val.get("type").is_some_and(|t| t.as_str().unwrap_or_default() != "filename") {
+                continue;
+            }
+            let path_str = entry_key;
             // Remove quotes from the path
             let clean_path = path_str.trim_matches('"');
             let full_path = path.join(clean_path);
 
             if !full_path.exists() {
-                tidy_error!(
-                    bad,
-                    "triagebot.toml [mentions.*] contains path '{}' which doesn't exist",
-                    clean_path
-                );
+                check.error(format!(
+                    "triagebot.toml [mentions.*] contains path '{clean_path}' which doesn't exist"
+                ));
             }
         }
     } else {
-        tidy_error!(
-            bad,
-            "triagebot.toml missing [mentions.*] section, this wrong for rust-lang/rust repo."
+        check.error(
+            "triagebot.toml missing [mentions.*] section, this wrong for rust-lang/rust repo.",
         );
     }
 
@@ -50,16 +55,13 @@ pub fn check(path: &Path, bad: &mut bool) {
                 let full_path = path.join(clean_path);
 
                 if !full_path.exists() {
-                    tidy_error!(
-                        bad,
-                        "triagebot.toml [assign.owners] contains path '{}' which doesn't exist",
-                        clean_path
-                    );
+                    check.error(format!(
+                        "triagebot.toml [assign.owners] contains path '{clean_path}' which doesn't exist"
+                    ));
                 }
             }
         } else {
-            tidy_error!(
-                bad,
+            check.error(
                 "triagebot.toml missing [assign.owners] section, this wrong for rust-lang/rust repo."
             );
         }
@@ -81,12 +83,9 @@ pub fn check(path: &Path, bad: &mut bool) {
 
                         // Handle both file and directory paths
                         if !full_path.exists() {
-                            tidy_error!(
-                                bad,
-                                "triagebot.toml [autolabel.{}] contains trigger_files path '{}' which doesn't exist",
-                                label,
-                                file_str
-                            );
+                            check.error(format!(
+                                "triagebot.toml [autolabel.{label}] contains trigger_files path '{file_str}' which doesn't exist",
+                            ));
                         }
                     }
                 }
diff --git a/src/tools/tidy/src/ui_tests.rs b/src/tools/tidy/src/ui_tests.rs
index 4d195b3952e2..45c2e76d1c1c 100644
--- a/src/tools/tidy/src/ui_tests.rs
+++ b/src/tools/tidy/src/ui_tests.rs
@@ -7,55 +7,24 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
-const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
-    "rs",     // test source files
-    "stderr", // expected stderr file, corresponds to a rs file
-    "svg",    // expected svg file, corresponds to a rs file, equivalent to stderr
-    "stdout", // expected stdout file, corresponds to a rs file
-    "fixed",  // expected source file after applying fixes
-    "md",     // test directory descriptions
-    "ftl",    // translation tests
-];
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 
-const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
-    "tests/ui/asm/named-asm-labels.s", // loading an external asm file to test named labels lint
-    "tests/ui/codegen/mismatched-data-layout.json", // testing mismatched data layout w/ custom targets
-    "tests/ui/check-cfg/my-awesome-platform.json",  // testing custom targets with cfgs
-    "tests/ui/argfile/commandline-argfile-badutf8.args", // passing args via a file
-    "tests/ui/argfile/commandline-argfile.args",    // passing args via a file
-    "tests/ui/crate-loading/auxiliary/libfoo.rlib", // testing loading a manually created rlib
-    "tests/ui/include-macros/data.bin", // testing including data with the include macros
-    "tests/ui/include-macros/file.txt", // testing including data with the include macros
-    "tests/ui/macros/macro-expanded-include/file.txt", // testing including data with the include macros
-    "tests/ui/macros/not-utf8.bin", // testing including data with the include macros
-    "tests/ui/macros/syntax-extension-source-utils-files/includeme.fragment", // more include
-    "tests/ui/proc-macro/auxiliary/included-file.txt", // more include
-    "tests/ui/unpretty/auxiliary/data.txt", // more include
-    "tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer
-    "tests/ui/sanitizer/dataflow-abilist.txt", // dataflow sanitizer ABI list file
-    "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file
-    "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file
-    "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file
-    "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file
-    "tests/ui/std/windows-bat-args1.bat", // tests escaping arguments through batch files
-    "tests/ui/std/windows-bat-args2.bat", // tests escaping arguments through batch files
-    "tests/ui/std/windows-bat-args3.bat", // tests escaping arguments through batch files
-];
-
-pub fn check(root_path: &Path, bless: bool, bad: &mut bool) {
-    let issues_txt_header = r#"============================================================
+const ISSUES_TXT_HEADER: &str = r#"============================================================
     ⚠️⚠️⚠️NOTHING SHOULD EVER BE ADDED TO THIS LIST⚠️⚠️⚠️
 ============================================================
 "#;
 
+pub fn check(root_path: &Path, tidy_ctx: TidyCtx) {
     let path = &root_path.join("tests");
+    let mut check = tidy_ctx.start_check(CheckId::new("ui_tests").path(path));
+    let bless = tidy_ctx.is_bless_enabled();
 
     // the list of files in ui tests that are allowed to start with `issue-XXXX`
     // BTreeSet because we would like a stable ordering so --bless works
     let mut prev_line = "";
     let mut is_sorted = true;
     let allowed_issue_names: BTreeSet<_> = include_str!("issues.txt")
-        .strip_prefix(issues_txt_header)
+        .strip_prefix(ISSUES_TXT_HEADER)
         .unwrap()
         .lines()
         .inspect(|&line| {
@@ -68,80 +37,15 @@ pub fn check(root_path: &Path, bless: bool, bad: &mut bool) {
         .collect();
 
     if !is_sorted && !bless {
-        tidy_error!(
-            bad,
+        check.error(
             "`src/tools/tidy/src/issues.txt` is not in order, mostly because you modified it manually,
             please only update it with command `x test tidy --bless`"
         );
     }
 
-    let mut remaining_issue_names: BTreeSet<&str> = allowed_issue_names.clone();
+    deny_new_top_level_ui_tests(&mut check, &path.join("ui"));
 
-    let (ui, ui_fulldeps) = (path.join("ui"), path.join("ui-fulldeps"));
-    let paths = [ui.as_path(), ui_fulldeps.as_path()];
-    crate::walk::walk_no_read(&paths, |_, _| false, &mut |entry| {
-        let file_path = entry.path();
-        if let Some(ext) = file_path.extension().and_then(OsStr::to_str) {
-            // files that are neither an expected extension or an exception should not exist
-            // they're probably typos or not meant to exist
-            if !(EXPECTED_TEST_FILE_EXTENSIONS.contains(&ext)
-                || EXTENSION_EXCEPTION_PATHS.iter().any(|path| file_path.ends_with(path)))
-            {
-                tidy_error!(bad, "file {} has unexpected extension {}", file_path.display(), ext);
-            }
-
-            // NB: We do not use file_stem() as some file names have multiple `.`s and we
-            // must strip all of them.
-            let testname =
-                file_path.file_name().unwrap().to_str().unwrap().split_once('.').unwrap().0;
-            if ext == "stderr" || ext == "stdout" || ext == "fixed" {
-                // Test output filenames have one of the formats:
-                // ```
-                // $testname.stderr
-                // $testname.$mode.stderr
-                // $testname.$revision.stderr
-                // $testname.$revision.$mode.stderr
-                // ```
-                //
-                // For now, just make sure that there is a corresponding
-                // `$testname.rs` file.
-
-                if !file_path.with_file_name(testname).with_extension("rs").exists()
-                    && !testname.contains("ignore-tidy")
-                {
-                    tidy_error!(bad, "Stray file with UI testing output: {:?}", file_path);
-                }
-
-                if let Ok(metadata) = fs::metadata(file_path)
-                    && metadata.len() == 0
-                {
-                    tidy_error!(bad, "Empty file with UI testing output: {:?}", file_path);
-                }
-            }
-
-            if ext == "rs"
-                && let Some(test_name) = static_regex!(r"^issues?[-_]?(\d{3,})").captures(testname)
-            {
-                // these paths are always relative to the passed `path` and always UTF8
-                let stripped_path = file_path
-                    .strip_prefix(path)
-                    .unwrap()
-                    .to_str()
-                    .unwrap()
-                    .replace(std::path::MAIN_SEPARATOR_STR, "/");
-
-                if !remaining_issue_names.remove(stripped_path.as_str())
-                    && !stripped_path.starts_with("ui/issues/")
-                {
-                    tidy_error!(
-                        bad,
-                        "file `tests/{stripped_path}` must begin with a descriptive name, consider `{{reason}}-issue-{issue_n}.rs`",
-                        issue_n = &test_name[1],
-                    );
-                }
-            }
-        }
-    });
+    let remaining_issue_names = recursively_check_ui_tests(&mut check, path, &allowed_issue_names);
 
     // if there are any file names remaining, they were moved on the fs.
     // our data must remain up to date, so it must be removed from issues.txt
@@ -152,7 +56,7 @@ pub fn check(root_path: &Path, bless: bool, bad: &mut bool) {
         // so we don't bork things on panic or a contributor using Ctrl+C
         let blessed_issues_path = tidy_src.join("issues_blessed.txt");
         let mut blessed_issues_txt = fs::File::create(&blessed_issues_path).unwrap();
-        blessed_issues_txt.write_all(issues_txt_header.as_bytes()).unwrap();
+        blessed_issues_txt.write_all(ISSUES_TXT_HEADER.as_bytes()).unwrap();
         // If we changed paths to use the OS separator, reassert Unix chauvinism for blessing.
         for filename in allowed_issue_names.difference(&remaining_issue_names) {
             writeln!(blessed_issues_txt, "{filename}").unwrap();
@@ -163,11 +67,174 @@ pub fn check(root_path: &Path, bless: bool, bad: &mut bool) {
         for file_name in remaining_issue_names {
             let mut p = PathBuf::from(path);
             p.push(file_name);
-            tidy_error!(
-                bad,
+            check.error(format!(
                 "file `{}` no longer exists and should be removed from the exclusions in `src/tools/tidy/src/issues.txt`",
                 p.display()
-            );
+            ));
+        }
+    }
+}
+
+fn deny_new_top_level_ui_tests(check: &mut RunningCheck, tests_path: &Path) {
+    // See  where we propose banning adding
+    // new ui tests *directly* under `tests/ui/`. For more context, see:
+    //
+    // - 
+    // - 
+
+    let top_level_ui_tests = ignore::WalkBuilder::new(tests_path)
+        .max_depth(Some(1))
+        .follow_links(false)
+        .build()
+        .flatten()
+        .filter(|e| {
+            let file_name = e.file_name();
+            file_name != ".gitattributes" && file_name != "README.md"
+        })
+        .filter(|e| !e.file_type().is_some_and(|f| f.is_dir()));
+
+    for entry in top_level_ui_tests {
+        check.error(format!(
+            "ui tests should be added under meaningful subdirectories: `{}`, see https://github.com/rust-lang/compiler-team/issues/902",
+            entry.path().display()
+        ));
+    }
+}
+
+fn recursively_check_ui_tests<'issues>(
+    check: &mut RunningCheck,
+    path: &Path,
+    allowed_issue_names: &'issues BTreeSet<&'issues str>,
+) -> BTreeSet<&'issues str> {
+    let mut remaining_issue_names: BTreeSet<&str> = allowed_issue_names.clone();
+
+    let (ui, ui_fulldeps) = (path.join("ui"), path.join("ui-fulldeps"));
+    let paths = [ui.as_path(), ui_fulldeps.as_path()];
+    crate::walk::walk_no_read(&paths, |_, _| false, &mut |entry| {
+        let file_path = entry.path();
+        if let Some(ext) = file_path.extension().and_then(OsStr::to_str) {
+            check_unexpected_extension(check, file_path, ext);
+
+            // NB: We do not use file_stem() as some file names have multiple `.`s and we
+            // must strip all of them.
+            let testname =
+                file_path.file_name().unwrap().to_str().unwrap().split_once('.').unwrap().0;
+            if ext == "stderr" || ext == "stdout" || ext == "fixed" {
+                check_stray_output_snapshot(check, file_path, testname);
+                check_empty_output_snapshot(check, file_path);
+            }
+
+            deny_new_nondescriptive_test_names(
+                check,
+                path,
+                &mut remaining_issue_names,
+                file_path,
+                testname,
+                ext,
+            );
+        }
+    });
+    remaining_issue_names
+}
+
+fn check_unexpected_extension(check: &mut RunningCheck, file_path: &Path, ext: &str) {
+    const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
+        "rs",     // test source files
+        "stderr", // expected stderr file, corresponds to a rs file
+        "svg",    // expected svg file, corresponds to a rs file, equivalent to stderr
+        "stdout", // expected stdout file, corresponds to a rs file
+        "fixed",  // expected source file after applying fixes
+        "md",     // test directory descriptions
+        "ftl",    // translation tests
+    ];
+
+    const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
+        "tests/ui/asm/named-asm-labels.s", // loading an external asm file to test named labels lint
+        "tests/ui/codegen/mismatched-data-layout.json", // testing mismatched data layout w/ custom targets
+        "tests/ui/check-cfg/my-awesome-platform.json",  // testing custom targets with cfgs
+        "tests/ui/argfile/commandline-argfile-badutf8.args", // passing args via a file
+        "tests/ui/argfile/commandline-argfile.args",    // passing args via a file
+        "tests/ui/crate-loading/auxiliary/libfoo.rlib", // testing loading a manually created rlib
+        "tests/ui/include-macros/data.bin", // testing including data with the include macros
+        "tests/ui/include-macros/file.txt", // testing including data with the include macros
+        "tests/ui/macros/macro-expanded-include/file.txt", // testing including data with the include macros
+        "tests/ui/macros/not-utf8.bin", // testing including data with the include macros
+        "tests/ui/macros/syntax-extension-source-utils-files/includeme.fragment", // more include
+        "tests/ui/proc-macro/auxiliary/included-file.txt", // more include
+        "tests/ui/unpretty/auxiliary/data.txt", // more include
+        "tests/ui/invalid/foo.natvis.xml", // sample debugger visualizer
+        "tests/ui/sanitizer/dataflow-abilist.txt", // dataflow sanitizer ABI list file
+        "tests/ui/shell-argfiles/shell-argfiles.args", // passing args via a file
+        "tests/ui/shell-argfiles/shell-argfiles-badquotes.args", // passing args via a file
+        "tests/ui/shell-argfiles/shell-argfiles-via-argfile-shell.args", // passing args via a file
+        "tests/ui/shell-argfiles/shell-argfiles-via-argfile.args", // passing args via a file
+        "tests/ui/std/windows-bat-args1.bat", // tests escaping arguments through batch files
+        "tests/ui/std/windows-bat-args2.bat", // tests escaping arguments through batch files
+        "tests/ui/std/windows-bat-args3.bat", // tests escaping arguments through batch files
+    ];
+
+    // files that are neither an expected extension or an exception should not exist
+    // they're probably typos or not meant to exist
+    if !(EXPECTED_TEST_FILE_EXTENSIONS.contains(&ext)
+        || EXTENSION_EXCEPTION_PATHS.iter().any(|path| file_path.ends_with(path)))
+    {
+        check.error(format!("file {} has unexpected extension {}", file_path.display(), ext));
+    }
+}
+
+fn check_stray_output_snapshot(check: &mut RunningCheck, file_path: &Path, testname: &str) {
+    // Test output filenames have one of the formats:
+    // ```
+    // $testname.stderr
+    // $testname.$mode.stderr
+    // $testname.$revision.stderr
+    // $testname.$revision.$mode.stderr
+    // ```
+    //
+    // For now, just make sure that there is a corresponding
+    // `$testname.rs` file.
+
+    if !file_path.with_file_name(testname).with_extension("rs").exists()
+        && !testname.contains("ignore-tidy")
+    {
+        check.error(format!("Stray file with UI testing output: {:?}", file_path));
+    }
+}
+
+fn check_empty_output_snapshot(check: &mut RunningCheck, file_path: &Path) {
+    if let Ok(metadata) = fs::metadata(file_path)
+        && metadata.len() == 0
+    {
+        check.error(format!("Empty file with UI testing output: {:?}", file_path));
+    }
+}
+
+fn deny_new_nondescriptive_test_names(
+    check: &mut RunningCheck,
+    path: &Path,
+    remaining_issue_names: &mut BTreeSet<&str>,
+    file_path: &Path,
+    testname: &str,
+    ext: &str,
+) {
+    if ext == "rs"
+        && let Some(test_name) = static_regex!(r"^issues?[-_]?(\d{3,})").captures(testname)
+    {
+        // these paths are always relative to the passed `path` and always UTF8
+        let stripped_path = file_path
+            .strip_prefix(path)
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .replace(std::path::MAIN_SEPARATOR_STR, "/");
+
+        if !remaining_issue_names.remove(stripped_path.as_str())
+            && !stripped_path.starts_with("ui/issues/")
+        {
+            check.error(format!(
+                "file `tests/{stripped_path}` must begin with a descriptive name, consider `{{reason}}-issue-{issue_n}.rs`",
+                issue_n = &test_name[1],
+            ));
         }
     }
 }
diff --git a/src/tools/tidy/src/unit_tests.rs b/src/tools/tidy/src/unit_tests.rs
index 3d14a4673193..74b6c4a3845a 100644
--- a/src/tools/tidy/src/unit_tests.rs
+++ b/src/tools/tidy/src/unit_tests.rs
@@ -11,9 +11,12 @@
 
 use std::path::Path;
 
+use crate::diagnostics::{CheckId, TidyCtx};
 use crate::walk::{filter_dirs, walk};
 
-pub fn check(root_path: &Path, stdlib: bool, bad: &mut bool) {
+pub fn check(root_path: &Path, stdlib: bool, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("unit_tests").path(root_path));
+
     let skip = move |path: &Path, is_dir| {
         let file_name = path.file_name().unwrap_or_default();
 
@@ -61,6 +64,7 @@ pub fn check(root_path: &Path, stdlib: bool, bad: &mut bool) {
                 || path.ends_with("library/alloc/src/collections/linked_list/tests.rs")
                 || path.ends_with("library/alloc/src/collections/vec_deque/tests.rs")
                 || path.ends_with("library/alloc/src/raw_vec/tests.rs")
+                || path.ends_with("library/alloc/src/wtf8/tests.rs")
         }
     };
 
@@ -91,14 +95,11 @@ pub fn check(root_path: &Path, stdlib: bool, bad: &mut bool) {
                         .to_owned()
                 };
                 let name = if is_test() { "test" } else { "bench" };
-                tidy_error!(
-                    bad,
-                    "`{}:{}` contains `#[{}]`; {}",
+                check.error(format!(
+                    "`{}:{}` contains `#[{name}]`; {explanation}",
                     path.display(),
                     i + 1,
-                    name,
-                    explanation,
-                );
+                ));
                 return;
             }
         }
diff --git a/src/tools/tidy/src/unknown_revision.rs b/src/tools/tidy/src/unknown_revision.rs
index 0ba05c80a791..43de6ee18b4d 100644
--- a/src/tools/tidy/src/unknown_revision.rs
+++ b/src/tools/tidy/src/unknown_revision.rs
@@ -12,12 +12,14 @@ use std::sync::OnceLock;
 use ignore::DirEntry;
 use regex::Regex;
 
+use crate::diagnostics::{CheckId, RunningCheck, TidyCtx};
 use crate::iter_header::{HeaderLine, iter_header};
 use crate::walk::{filter_dirs, filter_not_rust, walk};
 
-pub fn check(tests_path: impl AsRef, bad: &mut bool) {
+pub fn check(tests_path: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("unknown_revision").path(tests_path));
     walk(
-        tests_path.as_ref(),
+        tests_path,
         |path, is_dir| {
             filter_dirs(path) || filter_not_rust(path) || {
                 // Auxiliary source files for incremental tests can refer to revisions
@@ -25,11 +27,11 @@ pub fn check(tests_path: impl AsRef, bad: &mut bool) {
                 is_dir && path.file_name().is_some_and(|name| name == "auxiliary")
             }
         },
-        &mut |entry, contents| visit_test_file(entry, contents, bad),
+        &mut |entry, contents| visit_test_file(entry, contents, &mut check),
     );
 }
 
-fn visit_test_file(entry: &DirEntry, contents: &str, bad: &mut bool) {
+fn visit_test_file(entry: &DirEntry, contents: &str, check: &mut RunningCheck) {
     let mut revisions = HashSet::new();
     let mut unused_revision_names = HashSet::new();
 
@@ -68,10 +70,9 @@ fn visit_test_file(entry: &DirEntry, contents: &str, bad: &mut bool) {
 
     // Fail if any revision names appear in both places, since that's probably a mistake.
     for rev in revisions.intersection(&unused_revision_names).copied().collect::>() {
-        tidy_error!(
-            bad,
+        check.error(format!(
             "revision name [{rev}] appears in both `revisions` and `unused-revision-names` in {path}"
-        );
+        ));
     }
 
     // Compute the set of revisions that were mentioned but not declared,
@@ -84,7 +85,7 @@ fn visit_test_file(entry: &DirEntry, contents: &str, bad: &mut bool) {
     bad_revisions.sort();
 
     for (line_number, rev) in bad_revisions {
-        tidy_error!(bad, "unknown revision [{rev}] at {path}:{line_number}");
+        check.error(format!("unknown revision [{rev}] at {path}:{line_number}"));
     }
 }
 
diff --git a/src/tools/tidy/src/unstable_book.rs b/src/tools/tidy/src/unstable_book.rs
index 9dc9d42d4660..cfaf56ae7b64 100644
--- a/src/tools/tidy/src/unstable_book.rs
+++ b/src/tools/tidy/src/unstable_book.rs
@@ -2,10 +2,13 @@ use std::collections::BTreeSet;
 use std::fs;
 use std::path::{Path, PathBuf};
 
+use crate::diagnostics::{RunningCheck, TidyCtx};
 use crate::features::{CollectedFeatures, Features, Status};
 
 pub const PATH_STR: &str = "doc/unstable-book";
 
+pub const ENV_VARS_DIR: &str = "src/compiler-environment-variables";
+
 pub const COMPILER_FLAGS_DIR: &str = "src/compiler-flags";
 
 pub const LANG_FEATURES_DIR: &str = "src/language-features";
@@ -73,19 +76,18 @@ fn collect_unstable_book_lib_features_section_file_names(base_src_path: &Path) -
 }
 
 /// Would switching underscores for dashes work?
-fn maybe_suggest_dashes(names: &BTreeSet, feature_name: &str, bad: &mut bool) {
+fn maybe_suggest_dashes(names: &BTreeSet, feature_name: &str, check: &mut RunningCheck) {
     let with_dashes = feature_name.replace('_', "-");
     if names.contains(&with_dashes) {
-        tidy_error!(
-            bad,
-            "the file `{}.md` contains underscores; use dashes instead: `{}.md`",
-            feature_name,
-            with_dashes,
-        );
+        check.error(format!(
+            "the file `{feature_name}.md` contains underscores; use dashes instead: `{with_dashes}.md`",
+        ));
     }
 }
 
-pub fn check(path: &Path, features: CollectedFeatures, bad: &mut bool) {
+pub fn check(path: &Path, features: CollectedFeatures, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check("unstable_book");
+
     let lang_features = features.lang;
     let lib_features = features
         .lib
@@ -106,26 +108,22 @@ pub fn check(path: &Path, features: CollectedFeatures, bad: &mut bool) {
     // Check for Unstable Book sections that don't have a corresponding unstable feature
     for feature_name in &unstable_book_lib_features_section_file_names - &unstable_lib_feature_names
     {
-        tidy_error!(
-            bad,
-            "The Unstable Book has a 'library feature' section '{}' which doesn't \
-                         correspond to an unstable library feature",
-            feature_name
-        );
-        maybe_suggest_dashes(&unstable_lib_feature_names, &feature_name, bad);
+        check.error(format!(
+            "The Unstable Book has a 'library feature' section '{feature_name}' which doesn't \
+                         correspond to an unstable library feature"
+        ));
+        maybe_suggest_dashes(&unstable_lib_feature_names, &feature_name, &mut check);
     }
 
     // Check for Unstable Book sections that don't have a corresponding unstable feature.
     for feature_name in
         &unstable_book_lang_features_section_file_names - &unstable_lang_feature_names
     {
-        tidy_error!(
-            bad,
-            "The Unstable Book has a 'language feature' section '{}' which doesn't \
-                     correspond to an unstable language feature",
-            feature_name
-        );
-        maybe_suggest_dashes(&unstable_lang_feature_names, &feature_name, bad);
+        check.error(format!(
+            "The Unstable Book has a 'language feature' section '{feature_name}' which doesn't \
+                     correspond to an unstable language feature"
+        ));
+        maybe_suggest_dashes(&unstable_lang_feature_names, &feature_name, &mut check);
     }
 
     // List unstable features that don't have Unstable Book sections.
diff --git a/src/tools/tidy/src/x_version.rs b/src/tools/tidy/src/x_version.rs
index 9f7f43c4000b..28fd725cd3e2 100644
--- a/src/tools/tidy/src/x_version.rs
+++ b/src/tools/tidy/src/x_version.rs
@@ -3,12 +3,18 @@ use std::process::{Command, Stdio};
 
 use semver::Version;
 
-pub fn check(root: &Path, cargo: &Path, bad: &mut bool) {
+use crate::diagnostics::{CheckId, TidyCtx};
+
+pub fn check(root: &Path, cargo: &Path, tidy_ctx: TidyCtx) {
+    let mut check = tidy_ctx.start_check(CheckId::new("x_version").path(root));
     let cargo_list = Command::new(cargo).args(["install", "--list"]).stdout(Stdio::piped()).spawn();
 
     let child = match cargo_list {
         Ok(child) => child,
-        Err(e) => return tidy_error!(bad, "failed to run `cargo`: {}", e),
+        Err(e) => {
+            check.error(format!("failed to run `cargo`: {e}"));
+            return;
+        }
     };
 
     let cargo_list = child.wait_with_output().unwrap();
@@ -47,13 +53,10 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) {
                 )
             }
         } else {
-            tidy_error!(
-                bad,
-                "Unable to parse the latest version of `x` at `src/tools/x/Cargo.toml`"
-            )
+            check.error("Unable to parse the latest version of `x` at `src/tools/x/Cargo.toml`")
         }
     } else {
-        tidy_error!(bad, "failed to check version of `x`: {}", cargo_list.status)
+        check.error(format!("failed to check version of `x`: {}", cargo_list.status))
     }
 }
 
diff --git a/src/tools/unicode-table-generator/src/cascading_map.rs b/src/tools/unicode-table-generator/src/cascading_map.rs
index 78a7bba32087..41124a8e85ee 100644
--- a/src/tools/unicode-table-generator/src/cascading_map.rs
+++ b/src/tools/unicode-table-generator/src/cascading_map.rs
@@ -1,9 +1,8 @@
 use std::collections::HashMap;
-use std::fmt::Write as _;
 use std::ops::Range;
 
-use crate::fmt_list;
 use crate::raw_emitter::RawEmitter;
+use crate::writeln;
 
 impl RawEmitter {
     pub fn emit_cascading_map(&mut self, ranges: &[Range]) -> bool {
@@ -24,8 +23,6 @@ impl RawEmitter {
             .flat_map(|r| (r.start..r.end).collect::>())
             .collect::>();
 
-        println!("there are {} points", points.len());
-
         // how many distinct ranges need to be counted?
         let mut codepoints_by_high_bytes = HashMap::>::new();
         for point in points {
@@ -37,7 +34,7 @@ impl RawEmitter {
         }
 
         let mut bit_for_high_byte = 1u8;
-        let mut arms = Vec::::new();
+        let mut arms = String::new();
 
         let mut high_bytes: Vec = codepoints_by_high_bytes.keys().copied().collect();
         high_bytes.sort();
@@ -45,32 +42,33 @@ impl RawEmitter {
             let codepoints = codepoints_by_high_bytes.get_mut(&high_byte).unwrap();
             if codepoints.len() == 1 {
                 let ch = codepoints.pop().unwrap();
-                arms.push(format!("{high_byte} => c as u32 == {ch:#04x}"));
+                writeln!(arms, "{high_byte:#04x} => c as u32 == {ch:#04x},");
                 continue;
             }
             // more than 1 codepoint in this arm
             for codepoint in codepoints {
                 map[(*codepoint & 0xff) as usize] |= bit_for_high_byte;
             }
-            arms.push(format!(
-                "{high_byte} => WHITESPACE_MAP[c as usize & 0xff] & {bit_for_high_byte} != 0"
-            ));
+            writeln!(
+                arms,
+                "{high_byte:#04x} => WHITESPACE_MAP[c as usize & 0xff] & {bit_for_high_byte} != 0,"
+            );
             bit_for_high_byte <<= 1;
         }
 
-        writeln!(&mut self.file, "static WHITESPACE_MAP: [u8; 256] = [{}];", fmt_list(map.iter()))
-            .unwrap();
         self.bytes_used += 256;
+        self.file = format!(
+            "static WHITESPACE_MAP: [u8; 256] = {map:?};
 
-        writeln!(&mut self.file, "#[inline]").unwrap();
-        writeln!(&mut self.file, "pub const fn lookup(c: char) -> bool {{").unwrap();
-        writeln!(&mut self.file, "    match c as u32 >> 8 {{").unwrap();
-        for arm in arms {
-            writeln!(&mut self.file, "        {arm},").unwrap();
-        }
-        writeln!(&mut self.file, "        _ => false,").unwrap();
-        writeln!(&mut self.file, "    }}").unwrap();
-        writeln!(&mut self.file, "}}").unwrap();
+            #[inline]
+            pub const fn lookup(c: char) -> bool {{
+                debug_assert!(!c.is_ascii());
+                match c as u32 >> 8 {{
+                    {arms}\
+                    _ => false,
+                }}
+            }}"
+        );
 
         true
     }
diff --git a/src/tools/unicode-table-generator/src/case_mapping.rs b/src/tools/unicode-table-generator/src/case_mapping.rs
index 9c6454492e7e..3280514692b0 100644
--- a/src/tools/unicode-table-generator/src/case_mapping.rs
+++ b/src/tools/unicode-table-generator/src/case_mapping.rs
@@ -1,29 +1,29 @@
 use std::char;
 use std::collections::BTreeMap;
-use std::fmt::{self, Write};
 
-use crate::{UnicodeData, fmt_list};
+use crate::fmt_helpers::Hex;
+use crate::{CharEscape, UnicodeData, fmt_list};
 
 const INDEX_MASK: u32 = 1 << 22;
 
-pub(crate) fn generate_case_mapping(data: &UnicodeData) -> String {
-    let mut file = String::new();
-
-    write!(file, "const INDEX_MASK: u32 = 0x{INDEX_MASK:x};").unwrap();
-    file.push_str("\n\n");
-    file.push_str(HEADER.trim_start());
-    file.push('\n');
-    file.push_str(&generate_tables("LOWER", &data.to_lower));
-    file.push_str("\n\n");
-    file.push_str(&generate_tables("UPPER", &data.to_upper));
-    file
+pub(crate) fn generate_case_mapping(data: &UnicodeData) -> (String, [usize; 2]) {
+    let (lower_tables, lower_size) = generate_tables("LOWER", &data.to_lower);
+    let (upper_tables, upper_size) = generate_tables("UPPER", &data.to_upper);
+    let file = format!(
+        "{lower_tables}
+        {upper_tables}"
+    );
+    (file, [lower_size, upper_size])
 }
 
-fn generate_tables(case: &str, data: &BTreeMap) -> String {
+fn generate_tables(case: &str, data: &BTreeMap) -> (String, usize) {
+    let case_lower = case.to_lowercase();
+    let case_upper = case.to_uppercase();
+
     let mut mappings = Vec::with_capacity(data.len());
     let mut multis = Vec::new();
 
-    for (&key, &(a, b, c)) in data.iter() {
+    for (&key, &[a, b, c]) in data.iter() {
         let key = char::from_u32(key).unwrap();
 
         if key.is_ascii() {
@@ -42,62 +42,49 @@ fn generate_tables(case: &str, data: &BTreeMap) -> String
             INDEX_MASK | (u32::try_from(multis.len()).unwrap() - 1)
         };
 
-        mappings.push((CharEscape(key), value));
+        mappings.push((CharEscape(key), Hex(value)));
     }
 
-    let mut tables = String::new();
+    let size = size_of_val(mappings.as_slice()) + size_of_val(multis.as_slice());
+    let file = format!(
+        "
+#[rustfmt::skip]
+static {case}CASE_TABLE: &[(char, u32); {mappings_len}] = &[{mappings}];
 
-    write!(tables, "static {}CASE_TABLE: &[(char, u32)] = &[{}];", case, fmt_list(mappings))
-        .unwrap();
+#[rustfmt::skip]
+static {case}CASE_TABLE_MULTI: &[[char; 3]; {multis_len}] = &[{multis}];
 
-    tables.push_str("\n\n");
+#[inline]
+pub fn to_{case_lower}(c: char) -> [char; 3] {{
+    const {{
+        let mut i = 0;
+        while i < {case_upper}CASE_TABLE.len() {{
+            let (_, val) = {case_upper}CASE_TABLE[i];
+            if val & (1 << 22) == 0 {{
+                assert!(char::from_u32(val).is_some());
+            }} else {{
+                let index = val & ((1 << 22) - 1);
+                assert!((index as usize) < {case_upper}CASE_TABLE_MULTI.len());
+            }}
+            i += 1;
+        }}
+    }}
 
-    write!(tables, "static {}CASE_TABLE_MULTI: &[[char; 3]] = &[{}];", case, fmt_list(multis))
-        .unwrap();
+    // SAFETY: Just checked that the tables are valid
+    unsafe {{
+        super::case_conversion(
+            c,
+            |c| c.to_ascii_{case_lower}case(),
+            {case_upper}CASE_TABLE,
+            {case_upper}CASE_TABLE_MULTI,
+        )
+    }}
+}}",
+        mappings = fmt_list(&mappings),
+        mappings_len = mappings.len(),
+        multis = fmt_list(&multis),
+        multis_len = multis.len(),
+    );
 
-    tables
+    (file, size)
 }
-
-struct CharEscape(char);
-
-impl fmt::Debug for CharEscape {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "'{}'", self.0.escape_default())
-    }
-}
-
-static HEADER: &str = r"
-pub fn to_lower(c: char) -> [char; 3] {
-    if c.is_ascii() {
-        [(c as u8).to_ascii_lowercase() as char, '\0', '\0']
-    } else {
-        LOWERCASE_TABLE
-            .binary_search_by(|&(key, _)| key.cmp(&c))
-            .map(|i| {
-                let u = LOWERCASE_TABLE[i].1;
-                char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
-                    // SAFETY: Index comes from statically generated table
-                    unsafe { *LOWERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
-                })
-            })
-            .unwrap_or([c, '\0', '\0'])
-    }
-}
-
-pub fn to_upper(c: char) -> [char; 3] {
-    if c.is_ascii() {
-        [(c as u8).to_ascii_uppercase() as char, '\0', '\0']
-    } else {
-        UPPERCASE_TABLE
-            .binary_search_by(|&(key, _)| key.cmp(&c))
-            .map(|i| {
-                let u = UPPERCASE_TABLE[i].1;
-                char::from_u32(u).map(|c| [c, '\0', '\0']).unwrap_or_else(|| {
-                    // SAFETY: Index comes from statically generated table
-                    unsafe { *UPPERCASE_TABLE_MULTI.get_unchecked((u & (INDEX_MASK - 1)) as usize) }
-                })
-            })
-            .unwrap_or([c, '\0', '\0'])
-    }
-}
-";
diff --git a/src/tools/unicode-table-generator/src/fmt_helpers.rs b/src/tools/unicode-table-generator/src/fmt_helpers.rs
new file mode 100644
index 000000000000..bd7417ca1100
--- /dev/null
+++ b/src/tools/unicode-table-generator/src/fmt_helpers.rs
@@ -0,0 +1,82 @@
+use std::fmt;
+
+// Convenience macros for writing and unwrapping.
+#[macro_export]
+macro_rules! writeln {
+    ($($args:tt)*) => {{
+        use std::fmt::Write as _;
+        std::writeln!($($args)*).unwrap();
+    }};
+}
+#[macro_export]
+macro_rules! write {
+    ($($args:tt)*) => {{
+        use std::fmt::Write as _;
+        std::write!($($args)*).unwrap();
+    }};
+}
+
+pub fn fmt_list(values: impl IntoIterator) -> String {
+    let pieces = values.into_iter().map(|b| format!("{b:?}, "));
+    let mut out = String::new();
+    let mut line = String::from("\n    ");
+    for piece in pieces {
+        if line.len() + piece.len() < 98 {
+            line.push_str(&piece);
+        } else {
+            writeln!(out, "{}", line.trim_end());
+            line = format!("    {piece}");
+        }
+    }
+    writeln!(out, "{}", line.trim_end());
+    out
+}
+
+/// Wrapper type for formatting a `T` using its `Binary` implementation.
+#[derive(Copy, Clone)]
+pub struct Bin(pub T);
+
+impl fmt::Debug for Bin {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let bits = size_of::() * 8;
+        std::write!(f, "0b{:0bits$b}", self.0)
+    }
+}
+
+impl fmt::Display for Bin {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(self, f)
+    }
+}
+
+/// Wrapper type for formatting a `T` using its `LowerHex` implementation.
+#[derive(Copy, Clone)]
+pub struct Hex(pub T);
+
+impl fmt::Debug for Hex {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        std::write!(f, "{:#x}", self.0)
+    }
+}
+
+impl fmt::Display for Hex {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(self, f)
+    }
+}
+
+/// Wrapper type for formatting a `char` using `escape_unicode`.
+#[derive(Copy, Clone)]
+pub struct CharEscape(pub char);
+
+impl fmt::Debug for CharEscape {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        std::write!(f, "'{}'", self.0.escape_unicode())
+    }
+}
+
+impl fmt::Display for CharEscape {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(self, f)
+    }
+}
diff --git a/src/tools/unicode-table-generator/src/main.rs b/src/tools/unicode-table-generator/src/main.rs
index 6cdb82a87bdf..78f63b5673a7 100644
--- a/src/tools/unicode-table-generator/src/main.rs
+++ b/src/tools/unicode-table-generator/src/main.rs
@@ -78,10 +78,12 @@ use ucd_parse::Codepoints;
 
 mod cascading_map;
 mod case_mapping;
+mod fmt_helpers;
 mod raw_emitter;
 mod skiplist;
 mod unicode_download;
 
+use fmt_helpers::*;
 use raw_emitter::{RawEmitter, emit_codepoints, emit_whitespace};
 
 static PROPERTIES: &[&str] = &[
@@ -92,17 +94,16 @@ static PROPERTIES: &[&str] = &[
     "Case_Ignorable",
     "Grapheme_Extend",
     "White_Space",
-    "Cc",
     "N",
 ];
 
 struct UnicodeData {
     ranges: Vec<(&'static str, Vec>)>,
-    to_upper: BTreeMap,
-    to_lower: BTreeMap,
+    to_upper: BTreeMap,
+    to_lower: BTreeMap,
 }
 
-fn to_mapping(origin: u32, codepoints: Vec) -> Option<(u32, u32, u32)> {
+fn to_mapping(origin: u32, codepoints: Vec) -> Option<[u32; 3]> {
     let mut a = None;
     let mut b = None;
     let mut c = None;
@@ -123,7 +124,7 @@ fn to_mapping(origin: u32, codepoints: Vec) -> Option<(u32
         }
     }
 
-    Some((a.unwrap(), b.unwrap_or(0), c.unwrap_or(0)))
+    Some([a.unwrap(), b.unwrap_or(0), c.unwrap_or(0)])
 }
 
 static UNICODE_DIRECTORY: &str = "unicode-downloads";
@@ -163,12 +164,12 @@ fn load_data() -> UnicodeData {
         if let Some(mapped) = row.simple_lowercase_mapping
             && mapped != row.codepoint
         {
-            to_lower.insert(row.codepoint.value(), (mapped.value(), 0, 0));
+            to_lower.insert(row.codepoint.value(), [mapped.value(), 0, 0]);
         }
         if let Some(mapped) = row.simple_uppercase_mapping
             && mapped != row.codepoint
         {
-            to_upper.insert(row.codepoint.value(), (mapped.value(), 0, 0));
+            to_upper.insert(row.codepoint.value(), [mapped.value(), 0, 0]);
         }
     }
 
@@ -187,57 +188,47 @@ fn load_data() -> UnicodeData {
         }
     }
 
-    let mut properties: HashMap<&'static str, Vec>> = properties
+    let mut properties: Vec<(&'static str, Vec>)> = properties
         .into_iter()
-        .map(|(k, v)| {
-            (
-                k,
-                v.into_iter()
-                    .flat_map(|codepoints| match codepoints {
-                        Codepoints::Single(c) => c
-                            .scalar()
-                            .map(|ch| ch as u32..ch as u32 + 1)
-                            .into_iter()
-                            .collect::>(),
-                        Codepoints::Range(c) => c
-                            .into_iter()
-                            .flat_map(|c| c.scalar().map(|ch| ch as u32..ch as u32 + 1))
-                            .collect::>(),
-                    })
-                    .collect::>>(),
-            )
+        .map(|(prop, codepoints)| {
+            let codepoints = codepoints
+                .into_iter()
+                .flatten()
+                .flat_map(|cp| cp.scalar())
+                .filter(|c| !c.is_ascii())
+                .map(u32::from)
+                .collect::>();
+            (prop, ranges_from_set(&codepoints))
         })
         .collect();
 
-    for ranges in properties.values_mut() {
-        merge_ranges(ranges);
-    }
-
-    let mut properties = properties.into_iter().collect::>();
     properties.sort_by_key(|p| p.0);
     UnicodeData { ranges: properties, to_lower, to_upper }
 }
 
 fn main() {
-    let write_location = std::env::args().nth(1).unwrap_or_else(|| {
-        eprintln!("Must provide path to write unicode tables to");
+    let args = std::env::args().collect::>();
+
+    if args.len() != 3 {
+        eprintln!("Must provide paths to write unicode tables and tests to");
         eprintln!(
-            "e.g. {} library/core/src/unicode/unicode_data.rs",
-            std::env::args().next().unwrap_or_default()
+            "e.g. {} library/core/src/unicode/unicode_data.rs library/coretests/tests/unicode/test_data.rs",
+            args[0]
         );
         std::process::exit(1);
-    });
+    }
 
-    // Optional test path, which is a Rust source file testing that the unicode
-    // property lookups are correct.
-    let test_path = std::env::args().nth(2);
+    let data_path = &args[1];
+    let test_path = &args[2];
 
     let unicode_data = load_data();
     let ranges_by_property = &unicode_data.ranges;
 
-    if let Some(path) = test_path {
-        std::fs::write(&path, generate_tests(&write_location, ranges_by_property)).unwrap();
-    }
+    let mut table_file = String::new();
+    writeln!(
+        table_file,
+        "//! This file is generated by `./x run src/tools/unicode-table-generator`; do not edit manually!",
+    );
 
     let mut total_bytes = 0;
     let mut modules = Vec::new();
@@ -252,8 +243,9 @@ fn main() {
         }
 
         modules.push((property.to_lowercase().to_string(), emitter.file));
-        println!(
-            "{:15}: {} bytes, {} codepoints in {} ranges ({} - {}) using {}",
+        writeln!(
+            table_file,
+            "// {:16}: {:5} bytes, {:6} codepoints in {:3} ranges (U+{:06X} - U+{:06X}) using {}",
             property,
             emitter.bytes_used,
             datapoints,
@@ -264,46 +256,39 @@ fn main() {
         );
         total_bytes += emitter.bytes_used;
     }
+    let (conversions, sizes) = case_mapping::generate_case_mapping(&unicode_data);
+    for (name, size) in ["to_lower", "to_upper"].iter().zip(sizes) {
+        writeln!(table_file, "// {:16}: {:5} bytes", name, size);
+        total_bytes += size;
+    }
+    writeln!(table_file, "// {:16}: {:5} bytes\n", "Total", total_bytes);
 
-    let mut table_file = String::new();
+    writeln!(table_file, "use super::rt::*;\n");
+    writeln!(table_file, "{}\n", version());
 
-    table_file.push_str(
-        "///! This file is generated by `./x run src/tools/unicode-table-generator`; do not edit manually!\n",
-    );
-
-    // Include the range search function
-    table_file.push('\n');
-    table_file.push_str(include_str!("range_search.rs"));
-    table_file.push('\n');
-
-    table_file.push_str(&version());
-
-    table_file.push('\n');
-
-    modules.push((String::from("conversions"), case_mapping::generate_case_mapping(&unicode_data)));
+    modules.push((String::from("conversions"), conversions));
 
     for (name, contents) in modules {
-        table_file.push_str("#[rustfmt::skip]\n");
-        table_file.push_str(&format!("pub mod {name} {{\n"));
-        for line in contents.lines() {
-            if !line.trim().is_empty() {
-                table_file.push_str("    ");
-                table_file.push_str(line);
-            }
-            table_file.push('\n');
+        writeln!(table_file, "pub mod {name} {{");
+        for line in contents.trim().lines() {
+            writeln!(table_file, "    {line}");
         }
-        table_file.push_str("}\n\n");
+        writeln!(table_file, "}}\n");
     }
 
-    std::fs::write(&write_location, format!("{}\n", table_file.trim_end())).unwrap();
+    let test_file = generate_tests(&unicode_data);
 
-    println!("Total table sizes: {total_bytes} bytes");
+    std::fs::write(&test_path, test_file).unwrap();
+    std::fs::write(&data_path, table_file).unwrap();
+    rustfmt(&data_path);
+    rustfmt(&test_path);
+}
+
+fn rustfmt(path: &str) {
+    std::process::Command::new("rustfmt").arg(path).status().expect("rustfmt failed");
 }
 
 fn version() -> String {
-    let mut out = String::new();
-    out.push_str("pub const UNICODE_VERSION: (u8, u8, u8) = ");
-
     let readme =
         std::fs::read_to_string(std::path::Path::new(UNICODE_DIRECTORY).join("ReadMe.txt"))
             .unwrap();
@@ -315,132 +300,81 @@ fn version() -> String {
         readme[start..end].split('.').map(|v| v.parse::().expect(v)).collect::>();
     let [major, minor, micro] = [version[0], version[1], version[2]];
 
-    out.push_str(&format!("({major}, {minor}, {micro});\n"));
-    out
+    format!("pub const UNICODE_VERSION: (u8, u8, u8) = ({major}, {minor}, {micro});")
 }
 
-fn fmt_list(values: impl IntoIterator) -> String {
-    let pieces = values.into_iter().map(|b| format!("{b:?}, ")).collect::>();
-    let mut out = String::new();
-    let mut line = String::from("\n    ");
-    for piece in pieces {
-        if line.len() + piece.len() < 98 {
-            line.push_str(&piece);
-        } else {
-            out.push_str(line.trim_end());
-            out.push('\n');
-            line = format!("    {piece}");
-        }
-    }
-    out.push_str(line.trim_end());
-    out.push('\n');
-    out
-}
-
-fn generate_tests(data_path: &str, ranges: &[(&str, Vec>)]) -> String {
+fn generate_tests(data: &UnicodeData) -> String {
     let mut s = String::new();
-    s.push_str("#![allow(incomplete_features, unused)]\n");
-    s.push_str("#![feature(const_generics)]\n\n");
-    s.push_str("\n#[allow(unused)]\nuse std::hint;\n");
-    s.push_str(&format!("#[path = \"{data_path}\"]\n"));
-    s.push_str("mod unicode_data;\n\n");
+    writeln!(
+        s,
+        "//! This file is generated by `./x run src/tools/unicode-table-generator`; do not edit manually!"
+    );
+    writeln!(s, "// ignore-tidy-filelength\n");
+    writeln!(s, "use std::ops::RangeInclusive;\n");
+    for (property, ranges) in &data.ranges {
+        let prop_upper = property.to_uppercase();
+        let is_true = (char::MIN..=char::MAX)
+            .filter(|c| !c.is_ascii())
+            .map(u32::from)
+            .filter(|c| ranges.iter().any(|r| r.contains(c)))
+            .collect::>();
+        let is_true = ranges_from_set(&is_true);
+        let is_true = is_true
+            .iter()
+            .map(|r| {
+                let start = char::from_u32(r.start).unwrap();
+                let end = char::from_u32(r.end - 1).unwrap();
+                CharEscape(start)..=CharEscape(end)
+            })
+            .collect::>();
 
-    s.push_str("\nfn main() {\n");
-
-    for (property, ranges) in ranges {
-        s.push_str(&format!(r#"    println!("Testing {property}");"#));
-        s.push('\n');
-        s.push_str(&format!("    {}_true();\n", property.to_lowercase()));
-        s.push_str(&format!("    {}_false();\n", property.to_lowercase()));
-        let mut is_true = Vec::new();
-        let mut is_false = Vec::new();
-        for ch_num in 0..(std::char::MAX as u32) {
-            if std::char::from_u32(ch_num).is_none() {
-                continue;
-            }
-            if ranges.iter().any(|r| r.contains(&ch_num)) {
-                is_true.push(ch_num);
-            } else {
-                is_false.push(ch_num);
-            }
-        }
-
-        s.push_str(&format!("    fn {}_true() {{\n", property.to_lowercase()));
-        generate_asserts(&mut s, property, &is_true, true);
-        s.push_str("    }\n\n");
-        s.push_str(&format!("    fn {}_false() {{\n", property.to_lowercase()));
-        generate_asserts(&mut s, property, &is_false, false);
-        s.push_str("    }\n\n");
+        writeln!(
+            s,
+            r#"
+#[rustfmt::skip]
+pub(super) static {prop_upper}: &[RangeInclusive; {is_true_len}] = &[{is_true}];
+"#,
+            is_true_len = is_true.len(),
+            is_true = fmt_list(is_true),
+        );
+    }
+
+    for (prop_lower, conversion) in
+        ["to_lower", "to_upper"].iter().zip([&data.to_lower, &data.to_upper])
+    {
+        let prop_upper = prop_lower.to_uppercase();
+
+        let mapped = conversion
+            .iter()
+            .map(|(c, chars)| {
+                (
+                    CharEscape(char::from_u32(*c).unwrap()),
+                    chars.map(|c| CharEscape(char::from_u32(c).unwrap())),
+                )
+            })
+            .collect::>();
+
+        writeln!(
+            s,
+            r#"
+#[rustfmt::skip]
+pub(super) static {prop_upper}: &[(char, [char; 3]); {mapped_len}] = &[{mapped}];
+"#,
+            mapped_len = mapped.len(),
+            mapped = fmt_list(mapped),
+        );
     }
 
-    s.push('}');
     s
 }
 
-fn generate_asserts(s: &mut String, property: &str, points: &[u32], truthy: bool) {
-    for range in ranges_from_set(points) {
-        if range.end == range.start + 1 {
-            s.push_str(&format!(
-                "        assert!({}unicode_data::{}::lookup({:?}), \"{}\");\n",
-                if truthy { "" } else { "!" },
-                property.to_lowercase(),
-                std::char::from_u32(range.start).unwrap(),
-                range.start,
-            ));
-        } else {
-            s.push_str(&format!("        for chn in {range:?}u32 {{\n"));
-            s.push_str(&format!(
-                "            assert!({}unicode_data::{}::lookup(std::char::from_u32(chn).unwrap()), \"{{:?}}\", chn);\n",
-                if truthy { "" } else { "!" },
-                property.to_lowercase(),
-            ));
-            s.push_str("        }\n");
-        }
-    }
-}
-
+/// Group the elements of `set` into contigous ranges
 fn ranges_from_set(set: &[u32]) -> Vec> {
-    let mut ranges = set.iter().map(|e| (*e)..(*e + 1)).collect::>>();
-    merge_ranges(&mut ranges);
-    ranges
-}
-
-fn merge_ranges(ranges: &mut Vec>) {
-    loop {
-        let mut new_ranges = Vec::new();
-        let mut idx_iter = 0..(ranges.len() - 1);
-        let mut should_insert_last = true;
-        while let Some(idx) = idx_iter.next() {
-            let cur = ranges[idx].clone();
-            let next = ranges[idx + 1].clone();
-            if cur.end == next.start {
-                if idx_iter.next().is_none() {
-                    // We're merging the last element
-                    should_insert_last = false;
-                }
-                new_ranges.push(cur.start..next.end);
-            } else {
-                // We're *not* merging the last element
-                should_insert_last = true;
-                new_ranges.push(cur);
-            }
-        }
-        if should_insert_last {
-            new_ranges.push(ranges.last().unwrap().clone());
-        }
-        if new_ranges.len() == ranges.len() {
-            *ranges = new_ranges;
-            break;
-        } else {
-            *ranges = new_ranges;
-        }
-    }
-
-    let mut last_end = None;
-    for range in ranges {
-        if let Some(last) = last_end {
-            assert!(range.start > last, "{range:?}");
-        }
-        last_end = Some(range.end);
-    }
+    set.chunk_by(|a, b| a + 1 == *b)
+        .map(|chunk| {
+            let start = *chunk.first().unwrap();
+            let end = *chunk.last().unwrap();
+            start..(end + 1)
+        })
+        .collect()
 }
diff --git a/src/tools/unicode-table-generator/src/range_search.rs b/src/tools/unicode-table-generator/src/range_search.rs
deleted file mode 100644
index 4d1dd9b423b5..000000000000
--- a/src/tools/unicode-table-generator/src/range_search.rs
+++ /dev/null
@@ -1,128 +0,0 @@
-#[inline(always)]
-const fn bitset_search<
-    const N: usize,
-    const CHUNK_SIZE: usize,
-    const N1: usize,
-    const CANONICAL: usize,
-    const CANONICALIZED: usize,
->(
-    needle: u32,
-    chunk_idx_map: &[u8; N],
-    bitset_chunk_idx: &[[u8; CHUNK_SIZE]; N1],
-    bitset_canonical: &[u64; CANONICAL],
-    bitset_canonicalized: &[(u8, u8); CANONICALIZED],
-) -> bool {
-    let bucket_idx = (needle / 64) as usize;
-    let chunk_map_idx = bucket_idx / CHUNK_SIZE;
-    let chunk_piece = bucket_idx % CHUNK_SIZE;
-    // FIXME(const-hack): Revert to `slice::get` when slice indexing becomes possible in const.
-    let chunk_idx = if chunk_map_idx < chunk_idx_map.len() {
-        chunk_idx_map[chunk_map_idx]
-    } else {
-        return false;
-    };
-    let idx = bitset_chunk_idx[chunk_idx as usize][chunk_piece] as usize;
-    // FIXME(const-hack): Revert to `slice::get` when slice indexing becomes possible in const.
-    let word = if idx < bitset_canonical.len() {
-        bitset_canonical[idx]
-    } else {
-        let (real_idx, mapping) = bitset_canonicalized[idx - bitset_canonical.len()];
-        let mut word = bitset_canonical[real_idx as usize];
-        let should_invert = mapping & (1 << 6) != 0;
-        if should_invert {
-            word = !word;
-        }
-        // Lower 6 bits
-        let quantity = mapping & ((1 << 6) - 1);
-        if mapping & (1 << 7) != 0 {
-            // shift
-            word >>= quantity as u64;
-        } else {
-            word = word.rotate_left(quantity as u32);
-        }
-        word
-    };
-    (word & (1 << (needle % 64) as u64)) != 0
-}
-
-#[repr(transparent)]
-struct ShortOffsetRunHeader(u32);
-
-impl ShortOffsetRunHeader {
-    const fn new(start_index: usize, prefix_sum: u32) -> Self {
-        assert!(start_index < (1 << 11));
-        assert!(prefix_sum < (1 << 21));
-
-        Self((start_index as u32) << 21 | prefix_sum)
-    }
-
-    #[inline]
-    const fn start_index(&self) -> usize {
-        (self.0 >> 21) as usize
-    }
-
-    #[inline]
-    const fn prefix_sum(&self) -> u32 {
-        self.0 & ((1 << 21) - 1)
-    }
-}
-
-/// # Safety
-///
-/// - The last element of `short_offset_runs` must be greater than `std::char::MAX`.
-/// - The start indices of all elements in `short_offset_runs` must be less than `OFFSETS`.
-#[inline(always)]
-unsafe fn skip_search(
-    needle: char,
-    short_offset_runs: &[ShortOffsetRunHeader; SOR],
-    offsets: &[u8; OFFSETS],
-) -> bool {
-    let needle = needle as u32;
-
-    let last_idx =
-        match short_offset_runs.binary_search_by_key(&(needle << 11), |header| header.0 << 11) {
-            Ok(idx) => idx + 1,
-            Err(idx) => idx,
-        };
-    // SAFETY: `last_idx` *cannot* be past the end of the array, as the last
-    // element is greater than `std::char::MAX` (the largest possible needle)
-    // as guaranteed by the caller.
-    //
-    // So, we cannot have found it (i.e. `Ok(idx) => idx + 1 != length`) and the
-    // correct location cannot be past it, so `Err(idx) => idx != length` either.
-    //
-    // This means that we can avoid bounds checking for the accesses below, too.
-    //
-    // We need to use `intrinsics::assume` since the `panic_nounwind` contained
-    // in `hint::assert_unchecked` may not be optimized out.
-    unsafe { crate::intrinsics::assume(last_idx < SOR) };
-
-    let mut offset_idx = short_offset_runs[last_idx].start_index();
-    let length = if let Some(next) = short_offset_runs.get(last_idx + 1) {
-        (*next).start_index() - offset_idx
-    } else {
-        offsets.len() - offset_idx
-    };
-
-    let prev =
-        last_idx.checked_sub(1).map(|prev| short_offset_runs[prev].prefix_sum()).unwrap_or(0);
-
-    let total = needle - prev;
-    let mut prefix_sum = 0;
-    for _ in 0..(length - 1) {
-        // SAFETY: It is guaranteed that `length <= OFFSETS - offset_idx`,
-        // so it follows that `length - 1 + offset_idx < OFFSETS`, therefore
-        // `offset_idx < OFFSETS` is always true in this loop.
-        //
-        // We need to use `intrinsics::assume` since the `panic_nounwind` contained
-        // in `hint::assert_unchecked` may not be optimized out.
-        unsafe { crate::intrinsics::assume(offset_idx < OFFSETS) };
-        let offset = offsets[offset_idx];
-        prefix_sum += offset as u32;
-        if prefix_sum > total {
-            break;
-        }
-        offset_idx += 1;
-    }
-    offset_idx % 2 == 1
-}
diff --git a/src/tools/unicode-table-generator/src/raw_emitter.rs b/src/tools/unicode-table-generator/src/raw_emitter.rs
index e9e0efc45944..048507a06d44 100644
--- a/src/tools/unicode-table-generator/src/raw_emitter.rs
+++ b/src/tools/unicode-table-generator/src/raw_emitter.rs
@@ -1,8 +1,7 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap};
-use std::fmt::{self, Write};
 use std::ops::Range;
 
-use crate::fmt_list;
+use crate::{Bin, fmt_list, writeln};
 
 #[derive(Clone)]
 pub struct RawEmitter {
@@ -16,13 +15,6 @@ impl RawEmitter {
         RawEmitter { file: String::new(), bytes_used: 0, desc: String::new() }
     }
 
-    fn blank_line(&mut self) {
-        if self.file.is_empty() || self.file.ends_with("\n\n") {
-            return;
-        }
-        writeln!(&mut self.file).unwrap();
-    }
-
     fn emit_bitset(&mut self, ranges: &[Range]) -> Result<(), String> {
         let first_code_point = ranges.first().unwrap().start;
         let last_code_point = ranges.last().unwrap().end;
@@ -68,47 +60,33 @@ impl RawEmitter {
         }
         self.emit_chunk_map(word_indices[&0], &compressed_words, best.unwrap().0);
 
-        struct Bits(u64);
-        impl fmt::Debug for Bits {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                write!(f, "0b{:064b}", self.0)
-            }
-        }
-
-        writeln!(
-            &mut self.file,
-            "static BITSET_CANONICAL: [u64; {}] = [{}];",
-            canonicalized.canonical_words.len(),
-            fmt_list(canonicalized.canonical_words.iter().map(|v| Bits(*v))),
-        )
-        .unwrap();
         self.bytes_used += 8 * canonicalized.canonical_words.len();
-        writeln!(
-            &mut self.file,
-            "static BITSET_MAPPING: [(u8, u8); {}] = [{}];",
-            canonicalized.canonicalized_words.len(),
-            fmt_list(&canonicalized.canonicalized_words),
-        )
-        .unwrap();
         // 8 bit index into shifted words, 7 bits for shift + optional flip
         // We only need it for the words that we removed by applying a shift and
         // flip to them.
         self.bytes_used += 2 * canonicalized.canonicalized_words.len();
 
-        self.blank_line();
+        writeln!(
+            self.file,
+            "static BITSET_CANONICAL: [u64; {canonical_words_len}] = {canonical_words:?};
+            static BITSET_MAPPING: [(u8, u8); {canonicalized_words_len}] = {canonicalized_words:?};
 
-        writeln!(&mut self.file, "pub const fn lookup(c: char) -> bool {{").unwrap();
-        if first_code_point > 0x7f {
-            writeln!(&mut self.file, "    (c as u32) >= {first_code_point:#04x} &&").unwrap();
-        }
-        writeln!(&mut self.file, "    super::bitset_search(").unwrap();
-        writeln!(&mut self.file, "        c as u32,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_CHUNKS_MAP,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_INDEX_CHUNKS,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_CANONICAL,").unwrap();
-        writeln!(&mut self.file, "        &BITSET_MAPPING,").unwrap();
-        writeln!(&mut self.file, "    )").unwrap();
-        writeln!(&mut self.file, "}}").unwrap();
+            pub const fn lookup(c: char) -> bool {{
+                debug_assert!(!c.is_ascii());
+                (c as u32) >= {first_code_point:#04x} &&
+                    super::bitset_search(
+                        c as u32,
+                        &BITSET_CHUNKS_MAP,
+                        &BITSET_INDEX_CHUNKS,
+                        &BITSET_CANONICAL,
+                        &BITSET_MAPPING,
+                    )
+            }}",
+            canonical_words = canonicalized.canonical_words,
+            canonical_words_len = canonicalized.canonical_words.len(),
+            canonicalized_words = canonicalized.canonicalized_words,
+            canonicalized_words_len = canonicalized.canonicalized_words.len(),
+        );
 
         Ok(())
     }
@@ -132,29 +110,21 @@ impl RawEmitter {
             chunk_indices.push(chunk_map[chunk]);
         }
 
-        writeln!(
-            &mut self.file,
-            "static BITSET_CHUNKS_MAP: [u8; {}] = [{}];",
-            chunk_indices.len(),
-            fmt_list(&chunk_indices),
-        )
-        .unwrap();
         self.bytes_used += chunk_indices.len();
         writeln!(
-            &mut self.file,
-            "static BITSET_INDEX_CHUNKS: [[u8; {}]; {}] = [{}];",
-            chunk_length,
-            chunks.len(),
-            fmt_list(chunks.iter()),
-        )
-        .unwrap();
+            self.file,
+            "static BITSET_CHUNKS_MAP: [u8; {chunk_indices_len}] = {chunk_indices:?};
+            static BITSET_INDEX_CHUNKS: [[u8; {chunk_len}]; {chunks_len}] = [{chunks}];",
+            chunk_indices_len = chunk_indices.len(),
+            chunk_len = chunk_length,
+            chunks_len = chunks.len(),
+            chunks = fmt_list(chunks.iter()),
+        );
         self.bytes_used += chunk_length * chunks.len();
     }
 }
 
 pub fn emit_codepoints(emitter: &mut RawEmitter, ranges: &[Range]) {
-    emitter.blank_line();
-
     let mut bitset = emitter.clone();
     let bitset_ok = bitset.emit_bitset(ranges).is_ok();
 
@@ -171,8 +141,6 @@ pub fn emit_codepoints(emitter: &mut RawEmitter, ranges: &[Range]) {
 }
 
 pub fn emit_whitespace(emitter: &mut RawEmitter, ranges: &[Range]) {
-    emitter.blank_line();
-
     let mut cascading = emitter.clone();
     cascading.emit_cascading_map(ranges);
     *emitter = cascading;
@@ -180,7 +148,7 @@ pub fn emit_whitespace(emitter: &mut RawEmitter, ranges: &[Range]) {
 }
 
 struct Canonicalized {
-    canonical_words: Vec,
+    canonical_words: Vec>,
     canonicalized_words: Vec<(u8, u8)>,
 
     /// Maps an input unique word to the associated index (u8) which is into
@@ -341,7 +309,7 @@ impl Canonicalized {
         for &w in unique_words {
             unique_mapping.entry(w).or_insert_with(|| {
                 canonical_words.push(w);
-                UniqueMapping::Canonical(canonical_words.len())
+                UniqueMapping::Canonical(canonical_words.len() - 1)
             });
         }
         assert_eq!(canonicalized_words.len() + canonical_words.len(), unique_words.len());
@@ -393,6 +361,7 @@ impl Canonicalized {
                 )
             })
             .collect::>();
+        let canonical_words = canonical_words.into_iter().map(Bin).collect::>();
         Canonicalized { unique_mapping, canonical_words, canonicalized_words }
     }
 }
diff --git a/src/tools/unicode-table-generator/src/skiplist.rs b/src/tools/unicode-table-generator/src/skiplist.rs
index 34c9802e122f..742d61153db3 100644
--- a/src/tools/unicode-table-generator/src/skiplist.rs
+++ b/src/tools/unicode-table-generator/src/skiplist.rs
@@ -1,8 +1,8 @@
-use std::fmt::{self, Write as _};
+use std::fmt::{self};
 use std::ops::Range;
 
-use crate::fmt_list;
 use crate::raw_emitter::RawEmitter;
+use crate::writeln;
 
 /// This will get packed into a single u32 before inserting into the data set.
 #[derive(PartialEq)]
@@ -68,77 +68,45 @@ impl RawEmitter {
             assert!(inserted);
         }
 
-        writeln!(&mut self.file, "use super::ShortOffsetRunHeader;\n").unwrap();
-        writeln!(
-            &mut self.file,
-            "static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; {}] = [{}];",
-            short_offset_runs.len(),
-            fmt_list(short_offset_runs.iter())
-        )
-        .unwrap();
         self.bytes_used += 4 * short_offset_runs.len();
-        writeln!(
-            &mut self.file,
-            "static OFFSETS: [u8; {}] = [{}];",
-            coded_offsets.len(),
-            fmt_list(&coded_offsets)
-        )
-        .unwrap();
         self.bytes_used += coded_offsets.len();
 
         // The inlining in this code works like the following:
         //
-        // The `skip_search` function is always inlined into the parent `lookup` fn,
+        // The `skip_search` function is always inlined into the parent `lookup_slow` fn,
         // thus the compiler can generate optimal code based on the referenced `static`s.
         //
-        // In the case of ASCII optimization, the lower-bounds check is inlined into
-        // the caller, and slower-path `skip_search` is outlined into a separate `lookup_slow` fn.
-        //
-        // Thus, in both cases, the `skip_search` function is specialized for the `static`s,
-        // and outlined into the prebuilt `std`.
-        if first_code_point > 0x7f {
-            writeln!(&mut self.file, "#[inline]").unwrap();
-            writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
-            writeln!(&mut self.file, "    (c as u32) >= {first_code_point:#04x} && lookup_slow(c)")
-                .unwrap();
-            writeln!(&mut self.file, "}}").unwrap();
-            writeln!(&mut self.file).unwrap();
-            writeln!(&mut self.file, "#[inline(never)]").unwrap();
-            writeln!(&mut self.file, "fn lookup_slow(c: char) -> bool {{").unwrap();
-        } else {
-            writeln!(&mut self.file, "pub fn lookup(c: char) -> bool {{").unwrap();
-        }
-        writeln!(&mut self.file, "    const {{").unwrap();
-        writeln!(
-            &mut self.file,
-            "        assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);",
-        )
-        .unwrap();
-        writeln!(&mut self.file, "        let mut i = 0;").unwrap();
-        writeln!(&mut self.file, "        while i < SHORT_OFFSET_RUNS.len() {{").unwrap();
-        writeln!(
-            &mut self.file,
-            "            assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());",
-        )
-        .unwrap();
-        writeln!(&mut self.file, "            i += 1;").unwrap();
-        writeln!(&mut self.file, "        }}").unwrap();
-        writeln!(&mut self.file, "    }}").unwrap();
-        writeln!(
-            &mut self.file,
-            "    // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`",
-        )
-        .unwrap();
-        writeln!(
-            &mut self.file,
-            "    // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.",
-        )
-        .unwrap();
-        writeln!(
-            &mut self.file,
-            "    unsafe {{ super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }}"
-        )
-        .unwrap();
-        writeln!(&mut self.file, "}}").unwrap();
+        // The lower-bounds check is inlined into the caller, and slower-path
+        // `skip_search` is outlined into a separate `lookup_slow` fn.
+        assert!(first_code_point > 0x7f);
+        writeln!(self.file,
+            "use super::ShortOffsetRunHeader;
+
+            static SHORT_OFFSET_RUNS: [ShortOffsetRunHeader; {short_offset_runs_len}] = {short_offset_runs:?};
+            static OFFSETS: [u8; {coded_offset_len}] = {coded_offsets:?};
+
+            #[inline]
+            pub fn lookup(c: char) -> bool {{
+                debug_assert!(!c.is_ascii());
+                (c as u32) >= {first_code_point:#04x} && lookup_slow(c)
+            }}
+
+            #[inline(never)]
+            fn lookup_slow(c: char) -> bool {{
+                const {{
+                    assert!(SHORT_OFFSET_RUNS.last().unwrap().0 > char::MAX as u32);
+                    let mut i = 0;
+                    while i < SHORT_OFFSET_RUNS.len() {{
+                        assert!(SHORT_OFFSET_RUNS[i].start_index() < OFFSETS.len());
+                        i += 1;
+                    }}
+                }}
+                // SAFETY: We just ensured the last element of `SHORT_OFFSET_RUNS` is greater than `std::char::MAX`
+                // and the start indices of all elements in `SHORT_OFFSET_RUNS` are smaller than `OFFSETS.len()`.
+                unsafe {{ super::skip_search(c, &SHORT_OFFSET_RUNS, &OFFSETS) }}
+            }}",
+            short_offset_runs_len = short_offset_runs.len(),
+            coded_offset_len = coded_offsets.len(),
+        );
     }
 }
diff --git a/src/tools/unstable-book-gen/src/main.rs b/src/tools/unstable-book-gen/src/main.rs
index 159a1d0fa174..16550f83003d 100644
--- a/src/tools/unstable-book-gen/src/main.rs
+++ b/src/tools/unstable-book-gen/src/main.rs
@@ -5,11 +5,12 @@ use std::env;
 use std::fs::{self, write};
 use std::path::Path;
 
-use tidy::features::{Features, collect_lang_features, collect_lib_features};
+use tidy::diagnostics::RunningCheck;
+use tidy::features::{Features, collect_env_vars, collect_lang_features, collect_lib_features};
 use tidy::t;
 use tidy::unstable_book::{
-    LANG_FEATURES_DIR, LIB_FEATURES_DIR, PATH_STR, collect_unstable_book_section_file_names,
-    collect_unstable_feature_names,
+    ENV_VARS_DIR, LANG_FEATURES_DIR, LIB_FEATURES_DIR, PATH_STR,
+    collect_unstable_book_section_file_names, collect_unstable_feature_names,
 };
 
 fn generate_stub_issue(path: &Path, name: &str, issue: u32, description: &str) {
@@ -27,6 +28,11 @@ fn generate_stub_no_issue(path: &Path, name: &str, description: &str) {
     t!(write(path, content), path);
 }
 
+fn generate_stub_env_var(path: &Path, name: &str) {
+    let content = format!(include_str!("stub-env-var.md"), name = name);
+    t!(write(path, content), path);
+}
+
 fn set_to_summary_str(set: &BTreeSet, dir: &str) -> String {
     set.iter()
         .map(|ref n| format!("    - [{}]({}/{}.md)", n.replace('-', "_"), dir, n))
@@ -59,7 +65,7 @@ fn generate_summary(path: &Path, lang_features: &Features, lib_features: &Featur
     t!(write(&summary_path, content), summary_path);
 }
 
-fn generate_unstable_book_files(src: &Path, out: &Path, features: &Features) {
+fn generate_feature_files(src: &Path, out: &Path, features: &Features) {
     let unstable_features = collect_unstable_feature_names(features);
     let unstable_section_file_names = collect_unstable_book_section_file_names(src);
     t!(fs::create_dir_all(&out));
@@ -83,6 +89,16 @@ fn generate_unstable_book_files(src: &Path, out: &Path, features: &Features) {
     }
 }
 
+fn generate_env_files(src: &Path, out: &Path, env_vars: &BTreeSet) {
+    let env_var_file_names = collect_unstable_book_section_file_names(src);
+    t!(fs::create_dir_all(&out));
+    for env_var in env_vars - &env_var_file_names {
+        let file_name = format!("{env_var}.md");
+        let out_file_path = out.join(&file_name);
+        generate_stub_env_var(&out_file_path, &env_var);
+    }
+}
+
 fn copy_recursive(from: &Path, to: &Path) {
     for entry in t!(fs::read_dir(from)) {
         let e = t!(entry);
@@ -107,26 +123,28 @@ fn main() {
     let src_path = Path::new(&src_path_str);
     let dest_path = Path::new(&dest_path_str);
 
-    let lang_features = collect_lang_features(compiler_path, &mut false);
+    let lang_features = collect_lang_features(compiler_path, &mut RunningCheck::new_noop());
     let lib_features = collect_lib_features(library_path)
         .into_iter()
         .filter(|&(ref name, _)| !lang_features.contains_key(name))
         .collect();
+    let env_vars = collect_env_vars(compiler_path);
 
     let doc_src_path = src_path.join(PATH_STR);
 
     t!(fs::create_dir_all(&dest_path));
 
-    generate_unstable_book_files(
+    generate_feature_files(
         &doc_src_path.join(LANG_FEATURES_DIR),
         &dest_path.join(LANG_FEATURES_DIR),
         &lang_features,
     );
-    generate_unstable_book_files(
+    generate_feature_files(
         &doc_src_path.join(LIB_FEATURES_DIR),
         &dest_path.join(LIB_FEATURES_DIR),
         &lib_features,
     );
+    generate_env_files(&doc_src_path.join(ENV_VARS_DIR), &dest_path.join(ENV_VARS_DIR), &env_vars);
 
     copy_recursive(&doc_src_path, &dest_path);
 
diff --git a/src/tools/unstable-book-gen/src/stub-env-var.md b/src/tools/unstable-book-gen/src/stub-env-var.md
new file mode 100644
index 000000000000..e8a7ddb855a6
--- /dev/null
+++ b/src/tools/unstable-book-gen/src/stub-env-var.md
@@ -0,0 +1,9 @@
+# `{name}`
+
+Environment variables have no tracking issue. This environment variable has no documentation, and therefore is likely internal to the compiler and not meant for general use.
+
+See [the code][github search] for more information.
+
+[github search]: https://github.com/search?q=repo%3Arust-lang%2Frust+%22{name}%22+path%3Acompiler&type=code
+
+------------------------
diff --git a/src/tools/wasm-component-ld/Cargo.toml b/src/tools/wasm-component-ld/Cargo.toml
index ce718902b29f..235e934b5541 100644
--- a/src/tools/wasm-component-ld/Cargo.toml
+++ b/src/tools/wasm-component-ld/Cargo.toml
@@ -10,4 +10,4 @@ name = "wasm-component-ld"
 path = "src/main.rs"
 
 [dependencies]
-wasm-component-ld = "0.5.14"
+wasm-component-ld = "0.5.18"
diff --git a/src/version b/src/version
index 82e24bf241e5..95784efddbc4 100644
--- a/src/version
+++ b/src/version
@@ -1 +1 @@
-1.90.0
+1.93.0
diff --git a/tests/assembly-llvm/aarch64-pointer-auth.rs b/tests/assembly-llvm/aarch64-pointer-auth.rs
index 56a26df469f3..e1ca6d775813 100644
--- a/tests/assembly-llvm/aarch64-pointer-auth.rs
+++ b/tests/assembly-llvm/aarch64-pointer-auth.rs
@@ -1,10 +1,13 @@
 // Test that PAC instructions are emitted when branch-protection is specified.
 
 //@ add-core-stubs
-//@ revisions: PACRET PAUTHLR_NOP PAUTHLR
+//@ revisions: GCS PACRET PAUTHLR_NOP PAUTHLR
 //@ assembly-output: emit-asm
 //@ needs-llvm-components: aarch64
 //@ compile-flags: --target aarch64-unknown-linux-gnu
+//@ [GCS] min-llvm-version: 21
+//@ [GCS] ignore-apple (XCode version needs updating)
+//@ [GCS] compile-flags: -Z branch-protection=gcs
 //@ [PACRET] compile-flags: -Z branch-protection=pac-ret,leaf
 //@ [PAUTHLR_NOP] compile-flags: -Z branch-protection=pac-ret,pc,leaf
 //@ [PAUTHLR] compile-flags: -C target-feature=+pauth-lr -Z branch-protection=pac-ret,pc,leaf
@@ -17,6 +20,7 @@
 extern crate minicore;
 use minicore::*;
 
+// GCS: .aeabi_attribute 2, 1 // Tag_Feature_GCS
 // PACRET: hint #25
 // PACRET: hint #29
 // PAUTHLR_NOP: hint #25
diff --git a/tests/assembly-llvm/asm/aarch64-outline-atomics.rs b/tests/assembly-llvm/asm/aarch64-outline-atomics.rs
index 22599c18dcf3..1177c1e68ed5 100644
--- a/tests/assembly-llvm/asm/aarch64-outline-atomics.rs
+++ b/tests/assembly-llvm/asm/aarch64-outline-atomics.rs
@@ -8,6 +8,10 @@
 use std::sync::atomic::AtomicI32;
 use std::sync::atomic::Ordering::*;
 
+// Verify config on outline-atomics works (it is always enabled on aarch64-linux).
+#[cfg(not(target_feature = "outline-atomics"))]
+compile_error!("outline-atomics is not enabled");
+
 pub fn compare_exchange(a: &AtomicI32) {
     // On AArch64 LLVM should outline atomic operations.
     // CHECK: __aarch64_cas4_relax
diff --git a/tests/assembly-llvm/asm/loongarch-type.rs b/tests/assembly-llvm/asm/loongarch-type.rs
index c782be19f1d2..95d811a6bd0a 100644
--- a/tests/assembly-llvm/asm/loongarch-type.rs
+++ b/tests/assembly-llvm/asm/loongarch-type.rs
@@ -1,8 +1,15 @@
 //@ add-core-stubs
+//@ revisions: loongarch32 loongarch64
+
 //@ assembly-output: emit-asm
-//@ compile-flags: --target loongarch64-unknown-linux-gnu
+
+//@[loongarch32] compile-flags: --target loongarch32-unknown-none
+//@[loongarch32] needs-llvm-components: loongarch
+
+//@[loongarch64] compile-flags: --target loongarch64-unknown-none
+//@[loongarch64] needs-llvm-components: loongarch
+
 //@ compile-flags: -Zmerge-functions=disabled
-//@ needs-llvm-components: loongarch
 
 #![feature(no_core, f16)]
 #![crate_type = "rlib"]
@@ -22,7 +29,7 @@ extern "C" {
 // CHECK-LABEL: sym_fn:
 // CHECK: #APP
 // CHECK: pcalau12i $t0, %got_pc_hi20(extern_func)
-// CHECK: ld.d $t0, $t0, %got_pc_lo12(extern_func)
+// CHECK: ld.{{[wd]}} $t0, $t0, %got_pc_lo12(extern_func)
 // CHECK: #NO_APP
 #[no_mangle]
 pub unsafe fn sym_fn() {
@@ -32,7 +39,7 @@ pub unsafe fn sym_fn() {
 // CHECK-LABEL: sym_static:
 // CHECK: #APP
 // CHECK: pcalau12i $t0, %got_pc_hi20(extern_static)
-// CHECK: ld.d $t0, $t0, %got_pc_lo12(extern_static)
+// CHECK: ld.{{[wd]}} $t0, $t0, %got_pc_lo12(extern_static)
 // CHECK: #NO_APP
 #[no_mangle]
 pub unsafe fn sym_static() {
@@ -87,16 +94,18 @@ check!(reg_i32, i32, reg, "move");
 // CHECK: #NO_APP
 check!(reg_f32, f32, reg, "move");
 
-// CHECK-LABEL: reg_i64:
-// CHECK: #APP
-// CHECK: move ${{[a-z0-9]+}}, ${{[a-z0-9]+}}
-// CHECK: #NO_APP
+// loongarch64-LABEL: reg_i64:
+// loongarch64: #APP
+// loongarch64: move ${{[a-z0-9]+}}, ${{[a-z0-9]+}}
+// loongarch64: #NO_APP
+#[cfg(loongarch64)]
 check!(reg_i64, i64, reg, "move");
 
-// CHECK-LABEL: reg_f64:
-// CHECK: #APP
-// CHECK: move ${{[a-z0-9]+}}, ${{[a-z0-9]+}}
-// CHECK: #NO_APP
+// loongarch64-LABEL: reg_f64:
+// loongarch64: #APP
+// loongarch64: move ${{[a-z0-9]+}}, ${{[a-z0-9]+}}
+// loongarch64: #NO_APP
+#[cfg(loongarch64)]
 check!(reg_f64, f64, reg, "move");
 
 // CHECK-LABEL: reg_ptr:
@@ -153,16 +162,18 @@ check_reg!(r4_i32, i32, "$r4", "move");
 // CHECK: #NO_APP
 check_reg!(r4_f32, f32, "$r4", "move");
 
-// CHECK-LABEL: r4_i64:
-// CHECK: #APP
-// CHECK: move $a0, $a0
-// CHECK: #NO_APP
+// loongarch64-LABEL: r4_i64:
+// loongarch64: #APP
+// loongarch64: move $a0, $a0
+// loongarch64: #NO_APP
+#[cfg(loongarch64)]
 check_reg!(r4_i64, i64, "$r4", "move");
 
-// CHECK-LABEL: r4_f64:
-// CHECK: #APP
-// CHECK: move $a0, $a0
-// CHECK: #NO_APP
+// loongarch64-LABEL: r4_f64:
+// loongarch64: #APP
+// loongarch64: move $a0, $a0
+// loongarch64: #NO_APP
+#[cfg(loongarch64)]
 check_reg!(r4_f64, f64, "$r4", "move");
 
 // CHECK-LABEL: r4_ptr:
diff --git a/tests/assembly-llvm/asm/mips-types.rs b/tests/assembly-llvm/asm/mips-types.rs
index 00e8ce0b874a..1dd345ff8fe0 100644
--- a/tests/assembly-llvm/asm/mips-types.rs
+++ b/tests/assembly-llvm/asm/mips-types.rs
@@ -94,7 +94,6 @@ check!(reg_f32, f32, freg, "mov.s");
 // CHECK: #APP
 // CHECK: mov.s $f0, $f0
 // CHECK: #NO_APP
-#[no_mangle]
 check_reg!(f0_f32, f32, "$f0", "mov.s");
 
 // CHECK-LABEL: reg_f32_64:
@@ -107,21 +106,18 @@ check!(reg_f32_64, f32, freg, "mov.d");
 // CHECK: #APP
 // CHECK: mov.d $f0, $f0
 // CHECK: #NO_APP
-#[no_mangle]
 check_reg!(f0_f32_64, f32, "$f0", "mov.d");
 
 // CHECK-LABEL: reg_f64:
 // CHECK: #APP
 // CHECK: mov.d $f{{[0-9]+}}, $f{{[0-9]+}}
 // CHECK: #NO_APP
-#[no_mangle]
 check!(reg_f64, f64, freg, "mov.d");
 
 // CHECK-LABEL: f0_f64:
 // CHECK: #APP
 // CHECK: mov.d $f0, $f0
 // CHECK: #NO_APP
-#[no_mangle]
 check_reg!(f0_f64, f64, "$f0", "mov.d");
 
 // CHECK-LABEL: reg_ptr:
diff --git a/tests/assembly-llvm/asm/powerpc-types.rs b/tests/assembly-llvm/asm/powerpc-types.rs
index 4291e4c02f3b..c3a0fc4cfa97 100644
--- a/tests/assembly-llvm/asm/powerpc-types.rs
+++ b/tests/assembly-llvm/asm/powerpc-types.rs
@@ -223,6 +223,94 @@ check!(vreg_f32, f32, vreg, "vmr");
 #[cfg(vsx)]
 check!(vreg_f64, f64, vreg, "vmr");
 
+// powerpc_vsx-LABEL: vsreg_i8x16:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i8x16:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_i8x16, i8x16, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i16x8:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i16x8:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_i16x8, i16x8, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i32x4:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i32x4:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_i32x4, i32x4, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i64x2:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i64x2:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_i64x2, i64x2, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32x4:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32x4:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_f32x4, f32x4, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64x2:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64x2:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_f64x2, f64x2, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_f32, f32, vsreg, "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp {{[0-9]+}}, {{[0-9]+}}
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check!(vsreg_f64, f64, vsreg, "xvsqrtdp");
+
 // CHECK-LABEL: reg_i8_r0:
 // CHECK: #APP
 // CHECK: mr 0, 0
@@ -472,3 +560,179 @@ check_reg!(vreg_f32_v18, f32, "18", "v18", "vmr");
 // powerpc64_vsx: #NO_APP
 #[cfg(vsx)]
 check_reg!(vreg_f64_v18, f64, "18", "v18", "vmr");
+
+// powerpc_vsx-LABEL: vsreg_i8x16_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i8x16_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i8x16_vs0, i8x16, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i16x8_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i16x8_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i16x8_vs0, i16x8, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i32x4_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i32x4_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i32x4_vs0, i32x4, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i64x2_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i64x2_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i64x2_vs0, i64x2, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32x4_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32x4_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f32x4_vs0, f32x4, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64x2_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64x2_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f64x2_vs0, f64x2, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f32_vs0, f32, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64_vs0:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 0, 0
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64_vs0:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 0, 0
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f64_vs0, f64, "0", "vs0", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i8x16_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i8x16_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i8x16_v40, i8x16, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i16x8_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i16x8_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i16x8_v40, i16x8, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i32x4_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i32x4_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i32x4_v40, i32x4, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_i64x2_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_i64x2_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_i64x2_v40, i64x2, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32x4_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32x4_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f32x4_v40, f32x4, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64x2_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64x2_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f64x2_v40, f64x2, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f32_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f32_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f32_v40, f32, "40", "vs40", "xvsqrtdp");
+
+// powerpc_vsx-LABEL: vsreg_f64_v40:
+// powerpc_vsx: #APP
+// powerpc_vsx: xvsqrtdp 40, 40
+// powerpc_vsx: #NO_APP
+// powerpc64_vsx-LABEL: vsreg_f64_v40:
+// powerpc64_vsx: #APP
+// powerpc64_vsx: xvsqrtdp 40, 40
+// powerpc64_vsx: #NO_APP
+#[cfg(vsx)]
+check_reg!(vsreg_f64_v40, f64, "40", "vs40", "xvsqrtdp");
diff --git a/tests/assembly-llvm/breakpoint.rs b/tests/assembly-llvm/breakpoint.rs
index e0cc2d1eebb7..d119b68e899b 100644
--- a/tests/assembly-llvm/breakpoint.rs
+++ b/tests/assembly-llvm/breakpoint.rs
@@ -9,6 +9,7 @@
 // CHECK-LABEL: use_bp
 // aarch64: brk #0xf000
 // x86_64: int3
+#[inline(never)]
 pub fn use_bp() {
     core::arch::breakpoint();
 }
diff --git a/tests/assembly-llvm/c-variadic-arm.rs b/tests/assembly-llvm/c-variadic-arm.rs
new file mode 100644
index 000000000000..2ef307405e13
--- /dev/null
+++ b/tests/assembly-llvm/c-variadic-arm.rs
@@ -0,0 +1,26 @@
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3
+//@ only-arm
+//@ ignore-thumb
+//@ ignore-android
+#![no_std]
+#![crate_type = "lib"]
+#![feature(c_variadic)]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn variadic(a: f64, mut args: ...) -> f64 {
+    // CHECK-LABEL: variadic
+    // CHECK: sub sp, sp
+
+    // CHECK: vldr
+    // CHECK: vadd.f64
+    // CHECK: vldr
+    // CHECK: vadd.f64
+    let b = args.arg::();
+    let c = args.arg::();
+    a + b + c
+
+    // CHECK: add sp, sp
+}
diff --git a/tests/assembly-llvm/force-target-feature.rs b/tests/assembly-llvm/force-target-feature.rs
new file mode 100644
index 000000000000..c11060d8d6dc
--- /dev/null
+++ b/tests/assembly-llvm/force-target-feature.rs
@@ -0,0 +1,33 @@
+//@ only-x86_64
+//@ assembly-output: emit-asm
+//@ compile-flags: -C opt-level=3 -C target-feature=-avx2
+//@ ignore-sgx Tests incompatible with LVI mitigations
+
+#![feature(effective_target_features)]
+
+use std::arch::x86_64::{__m256i, _mm256_add_epi32, _mm256_setzero_si256};
+use std::ops::Add;
+
+#[derive(Clone, Copy)]
+struct AvxU32(__m256i);
+
+impl Add for AvxU32 {
+    type Output = Self;
+
+    #[no_mangle]
+    #[inline(never)]
+    #[unsafe(force_target_feature(enable = "avx2"))]
+    fn add(self, oth: AvxU32) -> AvxU32 {
+        // CHECK-LABEL: add:
+        // CHECK-NOT: callq
+        // CHECK: vpaddd
+        // CHECK: retq
+        Self(_mm256_add_epi32(self.0, oth.0))
+    }
+}
+
+fn main() {
+    assert!(is_x86_feature_detected!("avx2"));
+    let v = AvxU32(unsafe { _mm256_setzero_si256() });
+    v + v;
+}
diff --git a/tests/assembly-llvm/loongarch-float-struct-abi.rs b/tests/assembly-llvm/loongarch-float-struct-abi.rs
new file mode 100644
index 000000000000..4991004fc05a
--- /dev/null
+++ b/tests/assembly-llvm/loongarch-float-struct-abi.rs
@@ -0,0 +1,134 @@
+//@ add-core-stubs
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3 --target loongarch64-unknown-linux-gnu
+//@ needs-llvm-components: loongarch
+
+#![feature(no_core, lang_items)]
+#![no_std]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[repr(C, align(64))]
+struct Aligned(f64);
+
+#[repr(C)]
+struct Padded(u8, Aligned);
+
+#[repr(C, packed)]
+struct Packed(u8, f32);
+
+impl Copy for Aligned {}
+impl Copy for Padded {}
+impl Copy for Packed {}
+
+extern "C" {
+    fn take_padded(x: Padded);
+    fn get_padded() -> Padded;
+    fn take_packed(x: Packed);
+    fn get_packed() -> Packed;
+}
+
+// CHECK-LABEL: pass_padded
+#[unsafe(no_mangle)]
+extern "C" fn pass_padded(out: &mut Padded, x: Padded) {
+    // CHECK: st.b $a1, $a0, 0
+    // CHECK-NEXT: fst.d $fa0, $a0, 64
+    // CHECK-NEXT: ret
+    *out = x;
+}
+
+// CHECK-LABEL: ret_padded
+#[unsafe(no_mangle)]
+extern "C" fn ret_padded(x: &Padded) -> Padded {
+    // CHECK: fld.d $fa0, $a0, 64
+    // CHECK-NEXT: ld.b $a0, $a0, 0
+    // CHECK-NEXT: ret
+    *x
+}
+
+#[unsafe(no_mangle)]
+extern "C" fn call_padded(x: &Padded) {
+    // CHECK: fld.d $fa0, $a0, 64
+    // CHECK-NEXT: ld.b $a0, $a0, 0
+    // CHECK-NEXT: pcaddu18i $t8, %call36(take_padded)
+    // CHECK-NEXT: jr $t8
+    unsafe {
+        take_padded(*x);
+    }
+}
+
+#[unsafe(no_mangle)]
+extern "C" fn receive_padded(out: &mut Padded) {
+    // CHECK: addi.d $sp, $sp, -16
+    // CHECK-NEXT: .cfi_def_cfa_offset 16
+    // CHECK-NEXT: st.d $ra, $sp, [[#%d,RA_SPILL:]]
+    // CHECK-NEXT: st.d [[TEMP:.*]], $sp, [[#%d,TEMP_SPILL:]]
+    // CHECK-NEXT: .cfi_offset 1, [[#%d,RA_SPILL - 16]]
+    // CHECK-NEXT: .cfi_offset [[#%d,TEMP_NUM:]], [[#%d,TEMP_SPILL - 16]]
+    // CHECK-NEXT: move [[TEMP]], $a0
+    // CHECK-NEXT: pcaddu18i $ra, %call36(get_padded)
+    // CHECK-NEXT: jirl $ra, $ra, 0
+    // CHECK-NEXT: st.b $a0, [[TEMP]], 0
+    // CHECK-NEXT: fst.d $fa0, [[TEMP]], 64
+    // CHECK-NEXT: ld.d [[TEMP]], $sp, [[#%d,TEMP_SPILL]]
+    // CHECK-NEXT: ld.d $ra, $sp, [[#%d,RA_SPILL]]
+    // CHECK: addi.d $sp, $sp, 16
+    // CHECK: ret
+    unsafe {
+        *out = get_padded();
+    }
+}
+
+// CHECK-LABEL: pass_packed
+#[unsafe(no_mangle)]
+extern "C" fn pass_packed(out: &mut Packed, x: Packed) {
+    // CHECK: st.b $a1, $a0, 0
+    // CHECK-NEXT: fst.s $fa0, $a0, 1
+    // CHECK-NEXT: ret
+    *out = x;
+}
+
+// CHECK-LABEL: ret_packed
+#[unsafe(no_mangle)]
+extern "C" fn ret_packed(x: &Packed) -> Packed {
+    // CHECK: fld.s $fa0, $a0, 1
+    // CHECK-NEXT: ld.b $a0, $a0, 0
+    // CHECK-NEXT: ret
+    *x
+}
+
+#[unsafe(no_mangle)]
+extern "C" fn call_packed(x: &Packed) {
+    // CHECK: fld.s $fa0, $a0, 1
+    // CHECK-NEXT: ld.b $a0, $a0, 0
+    // CHECK-NEXT: pcaddu18i $t8, %call36(take_packed)
+    // CHECK-NEXT: jr $t8
+    unsafe {
+        take_packed(*x);
+    }
+}
+
+#[unsafe(no_mangle)]
+extern "C" fn receive_packed(out: &mut Packed) {
+    // CHECK: addi.d $sp, $sp, -16
+    // CHECK-NEXT: .cfi_def_cfa_offset 16
+    // CHECK-NEXT: st.d $ra, $sp, [[#%d,RA_SPILL:]]
+    // CHECK-NEXT: st.d [[TEMP:.*]], $sp, [[#%d,TEMP_SPILL:]]
+    // CHECK-NEXT: .cfi_offset 1, [[#%d,RA_SPILL - 16]]
+    // CHECK-NEXT: .cfi_offset [[#%d,TEMP_NUM:]], [[#%d,TEMP_SPILL - 16]]
+    // CHECK-NEXT: move [[TEMP]], $a0
+    // CHECK-NEXT: pcaddu18i $ra, %call36(get_packed)
+    // CHECK-NEXT: jirl $ra, $ra, 0
+    // CHECK-NEXT: st.b $a0, [[TEMP]], 0
+    // CHECK-NEXT: fst.s $fa0, [[TEMP]], 1
+    // CHECK-NEXT: ld.d [[TEMP]], $sp, [[#%d,TEMP_SPILL]]
+    // CHECK-NEXT: ld.d $ra, $sp, [[#%d,RA_SPILL]]
+    // CHECK: addi.d $sp, $sp, 16
+    // CHECK: ret
+    unsafe {
+        *out = get_packed();
+    }
+}
diff --git a/tests/assembly-llvm/naked-functions/wasm32.rs b/tests/assembly-llvm/naked-functions/wasm32.rs
index 77547e82041f..4bf04dd39234 100644
--- a/tests/assembly-llvm/naked-functions/wasm32.rs
+++ b/tests/assembly-llvm/naked-functions/wasm32.rs
@@ -21,6 +21,7 @@ use minicore::*;
 // CHECK: .functype nop () -> ()
 // CHECK-NOT: .size
 // CHECK: end_function
+// CHECK-LABEL: .Lfunc_end_nop:
 #[no_mangle]
 #[unsafe(naked)]
 extern "C" fn nop() {
diff --git a/tests/assembly-llvm/reg-struct-return.rs b/tests/assembly-llvm/reg-struct-return.rs
new file mode 100644
index 000000000000..b251d791d513
--- /dev/null
+++ b/tests/assembly-llvm/reg-struct-return.rs
@@ -0,0 +1,143 @@
+//! Tests that -Zreg-struct-return changes ABI for small struct returns
+//! from hidden-pointer convention to register-return on x86_32.
+//! This test covers:
+//! * Callee side, verifying that the structs are correctly loaded into registers when
+//!   `-Zreg-struct-return` is activated
+//! * Caller side, verifying callers do receive returned structs in registers when
+//!   `-Zreg-struct-return` is activated
+//@ add-core-stubs
+//@ assembly-output: emit-asm
+//@ compile-flags: -O --target=i686-unknown-linux-gnu -Crelocation-model=static
+//@ revisions: WITH WITHOUT
+//@[WITH] compile-flags: -Zreg-struct-return
+//@ needs-llvm-components: x86
+
+#![feature(no_core)]
+#![no_std]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+// Verifies ABI changes for small structs, where both fields fit into one register.
+// WITH is expected to use register return, WITHOUT should use hidden pointer.
+mod Small {
+    struct SmallStruct {
+        a: i8,
+        b: i8,
+    }
+
+    unsafe extern "C" {
+        fn small() -> SmallStruct;
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn small_callee() -> SmallStruct {
+        // (42 << 8) | 42 = 10794
+
+        // WITH-LABEL: small_callee
+        // WITH: movw $10794, %ax
+        // WITH: retl
+
+        // WITHOUT-LABEL: small_callee
+        // WITHOUT: movl 4(%esp), %e{{.*}}
+        // WITHOUT: movw $10794, (%e{{.*}})
+        // WITHOUT: retl $4
+        SmallStruct { a: 42, b: 42 }
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn small_caller(dst: &mut SmallStruct) {
+        // WITH-LABEL: small_caller
+        // WITH: calll small
+        // WITH: movw %ax, (%e{{.*}})
+
+        // WITHOUT-LABEL: small_caller
+        // WITHOUT: calll small
+        // WITHOUT: movzwl {{.*}}(%esp), %e[[TMP:..]]
+        // WITHOUT: movw %[[TMP]], (%e{{..}})
+        *dst = small();
+    }
+}
+
+// Verifies ABI changes for a struct of size 8, which is the maximum size
+// for reg-struct-return.
+// WITH is expected to still use register return, WITHOUT should use hidden
+// pointer.
+mod Pivot {
+    struct PivotStruct {
+        a: i32,
+        b: i32,
+    }
+
+    unsafe extern "C" {
+        fn pivot() -> PivotStruct;
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn pivot_callee() -> PivotStruct {
+        // WITH-LABEL: pivot_callee
+        // WITH: movl $42, %e{{.*}}
+        // WITH: movl $42, %e{{.*}}
+        // WITH: retl
+
+        // WITHOUT-LABEL: pivot_callee
+        // WITHOUT: movl 4(%esp), %e{{.*}}
+        // WITHOUT-DAG: movl $42, (%e{{.*}})
+        // WITHOUT-DAG: movl $42, 4(%e{{.*}})
+        // WITHOUT: retl $4
+        PivotStruct { a: 42, b: 42 }
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn pivot_caller(dst: &mut PivotStruct) {
+        // WITH-LABEL: pivot_caller
+        // WITH: calll pivot
+        // WITH-DAG: movl %e{{.*}}, 4(%e{{.*}})
+        // WITH-DAG: movl %e{{.*}}, (%e{{.*}})
+
+        // WITHOUT-LABEL: pivot_caller
+        // WITHOUT: calll pivot
+        // WITHOUT: movsd {{.*}}(%esp), %[[TMP:xmm.]]
+        // WITHOUT: movsd %[[TMP]], (%e{{..}})
+        *dst = pivot();
+    }
+}
+
+// Verifies ABI changes for a struct of size 12, which is larger than the
+// maximum size for reg-struct-return (8 bytes).
+// Here, the hidden pointer convention should be used even when `-Zreg-struct-return` is set.
+mod Large {
+    struct LargeStruct {
+        a: i32,
+        b: i32,
+        c: i32,
+    }
+
+    unsafe extern "C" {
+        fn large() -> LargeStruct;
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn large_callee() -> LargeStruct {
+        // CHECK-LABEL: large_callee
+        // CHECK: movl 4(%esp), %e{{.*}}
+        // CHECK-DAG: movl $42, (%e{{.*}})
+        // CHECK-DAG: movl $42, 4(%e{{.*}})
+        // CHECK-DAG: movl $42, 8(%e{{.*}})
+        // CHECK: retl $4
+        LargeStruct { a: 42, b: 42, c: 42 }
+    }
+
+    #[unsafe(no_mangle)]
+    pub unsafe extern "C" fn large_caller(dst: &mut LargeStruct) {
+        // CHECK-LABEL: large_caller
+        // CHECK: calll large
+        // CHECK-DAG: movl   {{.*}}(%esp), %[[TMP1:e..]]
+        // CHECK-DAG: movl  %[[TMP1]], {{.*}}(%e{{..}})
+        // CHECK-DAG: movsd  {{.*}}(%esp), %[[TMP2:xmm.]]
+        // CHECK-DAG: movsd  %[[TMP2]], {{.*}}(%e{{..}})
+        *dst = large();
+    }
+}
diff --git a/tests/assembly-llvm/regparm-module-flag.rs b/tests/assembly-llvm/regparm-module-flag.rs
new file mode 100644
index 000000000000..67ef44285eac
--- /dev/null
+++ b/tests/assembly-llvm/regparm-module-flag.rs
@@ -0,0 +1,70 @@
+// Test the regparm ABI with builtin and non-builtin calls
+// Issue: https://github.com/rust-lang/rust/issues/145271
+//@ add-core-stubs
+//@ assembly-output: emit-asm
+//@ compile-flags: -O --target=i686-unknown-linux-gnu -Crelocation-model=static
+//@ revisions: REGPARM1 REGPARM2 REGPARM3
+//@[REGPARM1] compile-flags: -Zregparm=1
+//@[REGPARM2] compile-flags: -Zregparm=2
+//@[REGPARM3] compile-flags: -Zregparm=3
+//@ needs-llvm-components: x86
+#![feature(no_core)]
+#![no_std]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+unsafe extern "C" {
+    fn memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void;
+    fn non_builtin_memset(p: *mut c_void, val: i32, len: usize) -> *mut c_void;
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn entrypoint(len: usize, ptr: *mut c_void, val: i32) -> *mut c_void {
+    // REGPARM1-LABEL: entrypoint
+    // REGPARM1: movl %e{{.*}}, %ecx
+    // REGPARM1: pushl
+    // REGPARM1: pushl
+    // REGPARM1: calll memset
+
+    // REGPARM2-LABEL: entrypoint
+    // REGPARM2: movl 16(%esp), %edx
+    // REGPARM2: movl %e{{.*}}, (%esp)
+    // REGPARM2: movl %e{{.*}}, %eax
+    // REGPARM2: calll memset
+
+    // REGPARM3-LABEL: entrypoint
+    // REGPARM3: movl %e{{.*}}, %esi
+    // REGPARM3: movl %e{{.*}}, %eax
+    // REGPARM3: movl %e{{.*}}, %ecx
+    // REGPARM3: jmp memset
+    unsafe { memset(ptr, val, len) }
+}
+
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn non_builtin_entrypoint(
+    len: usize,
+    ptr: *mut c_void,
+    val: i32,
+) -> *mut c_void {
+    // REGPARM1-LABEL: non_builtin_entrypoint
+    // REGPARM1: movl %e{{.*}}, %ecx
+    // REGPARM1: pushl
+    // REGPARM1: pushl
+    // REGPARM1: calll non_builtin_memset
+
+    // REGPARM2-LABEL: non_builtin_entrypoint
+    // REGPARM2: movl 16(%esp), %edx
+    // REGPARM2: movl %e{{.*}}, (%esp)
+    // REGPARM2: movl %e{{.*}}, %eax
+    // REGPARM2: calll non_builtin_memset
+
+    // REGPARM3-LABEL: non_builtin_entrypoint
+    // REGPARM3: movl %e{{.*}}, %esi
+    // REGPARM3: movl %e{{.*}}, %eax
+    // REGPARM3: movl %e{{.*}}, %ecx
+    // REGPARM3: jmp non_builtin_memset
+    unsafe { non_builtin_memset(ptr, val, len) }
+}
diff --git a/tests/assembly-llvm/riscv-soft-abi-with-float-features.rs b/tests/assembly-llvm/riscv-soft-abi-with-float-features.rs
index 72cbd3841c12..085ea9facd06 100644
--- a/tests/assembly-llvm/riscv-soft-abi-with-float-features.rs
+++ b/tests/assembly-llvm/riscv-soft-abi-with-float-features.rs
@@ -2,9 +2,6 @@
 //@ assembly-output: emit-asm
 //@ compile-flags: --target riscv64imac-unknown-none-elf -Ctarget-feature=+f,+d
 //@ needs-llvm-components: riscv
-//@ revisions: LLVM-PRE-20 LLVM-POST-20
-//@ [LLVM-PRE-20] max-llvm-major-version: 19
-//@ [LLVM-POST-20] min-llvm-version: 20
 
 #![feature(no_core, lang_items, f16)]
 #![crate_type = "lib"]
@@ -28,11 +25,8 @@ pub extern "C" fn read_f16(x: &f16) -> f16 {
 // CHECK-LABEL: read_f32
 #[no_mangle]
 pub extern "C" fn read_f32(x: &f32) -> f32 {
-    // LLVM-PRE-20: flw fa5, 0(a0)
-    // LLVM-PRE-20-NEXT: fmv.x.w a0, fa5
-    // LLVM-PRE-20-NEXT: ret
-    // LLVM-POST-20: lw a0, 0(a0)
-    // LLVM-POST-20-NEXT: ret
+    // CHECK: lw a0, 0(a0)
+    // CHECK-NEXT: ret
     *x
 }
 
diff --git a/tests/assembly-llvm/s390x-vector-abi.rs b/tests/assembly-llvm/s390x-vector-abi.rs
index fcf42664034c..c9c3266a18fa 100644
--- a/tests/assembly-llvm/s390x-vector-abi.rs
+++ b/tests/assembly-llvm/s390x-vector-abi.rs
@@ -1,4 +1,5 @@
 //@ revisions: z10 z10_vector z13 z13_no_vector
+//@ add-core-stubs
 // ignore-tidy-linelength
 //@ assembly-output: emit-asm
 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
@@ -18,24 +19,8 @@
 // Cases where vector feature is disabled are rejected.
 // See tests/ui/simd-abi-checks-s390x.rs for test for them.
 
-#[lang = "pointee_sized"]
-pub trait PointeeSized {}
-
-#[lang = "meta_sized"]
-pub trait MetaSized: PointeeSized {}
-
-#[lang = "sized"]
-pub trait Sized: MetaSized {}
-#[lang = "copy"]
-pub trait Copy {}
-#[lang = "freeze"]
-pub trait Freeze {}
-
-impl Copy for [T; N] {}
-
-#[lang = "phantom_data"]
-pub struct PhantomData;
-impl Copy for PhantomData {}
+extern crate minicore;
+use minicore::*;
 
 #[repr(simd)]
 pub struct i8x8([i8; 8]);
@@ -52,8 +37,6 @@ pub struct WrapperWithZst(T, PhantomData<()>);
 #[repr(transparent)]
 pub struct TransparentWrapper(T);
 
-impl Copy for i8 {}
-impl Copy for i64 {}
 impl Copy for i8x8 {}
 impl Copy for i8x16 {}
 impl Copy for i8x32 {}
@@ -221,7 +204,7 @@ unsafe extern "C" fn vector_transparent_wrapper_ret_large(
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_arg_small(x: i8x8) -> i64 {
-    unsafe { *(&x as *const i8x8 as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_arg:
 // CHECK: vlgvg %r2, %v24, 0
@@ -229,7 +212,7 @@ unsafe extern "C" fn vector_arg_small(x: i8x8) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_arg(x: i8x16) -> i64 {
-    unsafe { *(&x as *const i8x16 as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_arg_large:
 // CHECK: lg %r2, 0(%r2)
@@ -237,7 +220,7 @@ unsafe extern "C" fn vector_arg(x: i8x16) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_arg_large(x: i8x32) -> i64 {
-    unsafe { *(&x as *const i8x32 as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 
 // CHECK-LABEL: vector_wrapper_arg_small:
@@ -246,7 +229,7 @@ unsafe extern "C" fn vector_arg_large(x: i8x32) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_arg_small(x: Wrapper) -> i64 {
-    unsafe { *(&x as *const Wrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_wrapper_arg:
 // CHECK: vlgvg %r2, %v24, 0
@@ -254,7 +237,7 @@ unsafe extern "C" fn vector_wrapper_arg_small(x: Wrapper) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_arg(x: Wrapper) -> i64 {
-    unsafe { *(&x as *const Wrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_wrapper_arg_large:
 // CHECK: lg %r2, 0(%r2)
@@ -262,7 +245,7 @@ unsafe extern "C" fn vector_wrapper_arg(x: Wrapper) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_arg_large(x: Wrapper) -> i64 {
-    unsafe { *(&x as *const Wrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 
 // https://github.com/rust-lang/rust/pull/131586#discussion_r1837071121
@@ -272,7 +255,7 @@ unsafe extern "C" fn vector_wrapper_arg_large(x: Wrapper) -> i64 {
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_padding_arg(x: WrapperAlign16) -> i64 {
-    unsafe { *(&x as *const WrapperAlign16 as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 
 // CHECK-LABEL: vector_wrapper_with_zst_arg_small:
@@ -282,7 +265,7 @@ unsafe extern "C" fn vector_wrapper_padding_arg(x: WrapperAlign16) -> i64
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_with_zst_arg_small(x: WrapperWithZst) -> i64 {
-    unsafe { *(&x as *const WrapperWithZst as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_wrapper_with_zst_arg:
 // CHECK: lg %r2, 0(%r2)
@@ -290,7 +273,7 @@ unsafe extern "C" fn vector_wrapper_with_zst_arg_small(x: WrapperWithZst)
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_with_zst_arg(x: WrapperWithZst) -> i64 {
-    unsafe { *(&x as *const WrapperWithZst as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_wrapper_with_zst_arg_large:
 // CHECK: lg %r2, 0(%r2)
@@ -298,7 +281,7 @@ unsafe extern "C" fn vector_wrapper_with_zst_arg(x: WrapperWithZst) -> i6
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_wrapper_with_zst_arg_large(x: WrapperWithZst) -> i64 {
-    unsafe { *(&x as *const WrapperWithZst as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 
 // CHECK-LABEL: vector_transparent_wrapper_arg_small:
@@ -307,7 +290,7 @@ unsafe extern "C" fn vector_wrapper_with_zst_arg_large(x: WrapperWithZst)
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_transparent_wrapper_arg_small(x: TransparentWrapper) -> i64 {
-    unsafe { *(&x as *const TransparentWrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_transparent_wrapper_arg:
 // CHECK: vlgvg %r2, %v24, 0
@@ -315,7 +298,7 @@ unsafe extern "C" fn vector_transparent_wrapper_arg_small(x: TransparentWrapper<
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_transparent_wrapper_arg(x: TransparentWrapper) -> i64 {
-    unsafe { *(&x as *const TransparentWrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
 // CHECK-LABEL: vector_transparent_wrapper_arg_large:
 // CHECK: lg %r2, 0(%r2)
@@ -323,5 +306,5 @@ unsafe extern "C" fn vector_transparent_wrapper_arg(x: TransparentWrapper
 #[cfg_attr(no_vector, target_feature(enable = "vector"))]
 #[no_mangle]
 unsafe extern "C" fn vector_transparent_wrapper_arg_large(x: TransparentWrapper) -> i64 {
-    unsafe { *(&x as *const TransparentWrapper as *const i64) }
+    unsafe { *(&raw const x as *const i64) }
 }
diff --git a/tests/assembly-llvm/simd/reduce-fadd-unordered.rs b/tests/assembly-llvm/simd/reduce-fadd-unordered.rs
index e872826f6ef7..fdd03639da0e 100644
--- a/tests/assembly-llvm/simd/reduce-fadd-unordered.rs
+++ b/tests/assembly-llvm/simd/reduce-fadd-unordered.rs
@@ -16,6 +16,7 @@ use std::simd::*;
 // It would emit about an extra fadd, depending on the architecture.
 
 // CHECK-LABEL: reduce_fadd_negative_zero
+#[inline(never)]
 pub unsafe fn reduce_fadd_negative_zero(v: f32x4) -> f32 {
     // x86_64: addps
     // x86_64-NEXT: movshdup
diff --git a/tests/assembly-llvm/targets/targets-elf.rs b/tests/assembly-llvm/targets/targets-elf.rs
index edf16548e7de..d2f22fd8d1a5 100644
--- a/tests/assembly-llvm/targets/targets-elf.rs
+++ b/tests/assembly-llvm/targets/targets-elf.rs
@@ -1,15 +1,24 @@
 //@ add-core-stubs
 //@ assembly-output: emit-asm
 // ignore-tidy-linelength
+//@ revisions: aarch64_be_unknown_hermit
+//@ [aarch64_be_unknown_hermit] compile-flags: --target aarch64_be-unknown-hermit
+//@ [aarch64_be_unknown_hermit] needs-llvm-components: aarch64
 //@ revisions: aarch64_be_unknown_linux_gnu
 //@ [aarch64_be_unknown_linux_gnu] compile-flags: --target aarch64_be-unknown-linux-gnu
 //@ [aarch64_be_unknown_linux_gnu] needs-llvm-components: aarch64
 //@ revisions: aarch64_be_unknown_linux_gnu_ilp32
 //@ [aarch64_be_unknown_linux_gnu_ilp32] compile-flags: --target aarch64_be-unknown-linux-gnu_ilp32
 //@ [aarch64_be_unknown_linux_gnu_ilp32] needs-llvm-components: aarch64
+//@ revisions: aarch64_be_unknown_linux_musl
+//@ [aarch64_be_unknown_linux_musl] compile-flags: --target aarch64_be-unknown-linux-musl
+//@ [aarch64_be_unknown_linux_musl] needs-llvm-components: aarch64
 //@ revisions: aarch64_be_unknown_netbsd
 //@ [aarch64_be_unknown_netbsd] compile-flags: --target aarch64_be-unknown-netbsd
 //@ [aarch64_be_unknown_netbsd] needs-llvm-components: aarch64
+//@ revisions: aarch64_be_unknown_none_softfloat
+//@ [aarch64_be_unknown_none_softfloat] compile-flags: --target aarch64_be-unknown-none-softfloat
+//@ [aarch64_be_unknown_none_softfloat] needs-llvm-components: aarch64
 //@ revisions: aarch64_kmc_solid_asp3
 //@ [aarch64_kmc_solid_asp3] compile-flags: --target aarch64-kmc-solid_asp3
 //@ [aarch64_kmc_solid_asp3] needs-llvm-components: aarch64
@@ -25,6 +34,9 @@
 //@ revisions: aarch64_unknown_fuchsia
 //@ [aarch64_unknown_fuchsia] compile-flags: --target aarch64-unknown-fuchsia
 //@ [aarch64_unknown_fuchsia] needs-llvm-components: aarch64
+//@ revisions: aarch64_unknown_helenos
+//@ [aarch64_unknown_helenos] compile-flags: --target aarch64-unknown-helenos
+//@ [aarch64_unknown_helenos] needs-llvm-components: aarch64
 //@ revisions: aarch64_unknown_hermit
 //@ [aarch64_unknown_hermit] compile-flags: --target aarch64-unknown-hermit
 //@ [aarch64_unknown_hermit] needs-llvm-components: aarch64
@@ -43,6 +55,9 @@
 //@ revisions: aarch64_unknown_linux_ohos
 //@ [aarch64_unknown_linux_ohos] compile-flags: --target aarch64-unknown-linux-ohos
 //@ [aarch64_unknown_linux_ohos] needs-llvm-components: aarch64
+//@ revisions: aarch64_unknown_managarm_mlibc
+//@ [aarch64_unknown_managarm_mlibc] compile-flags: --target aarch64-unknown-managarm-mlibc
+//@ [aarch64_unknown_managarm_mlibc] needs-llvm-components: aarch64
 //@ revisions: aarch64_unknown_netbsd
 //@ [aarch64_unknown_netbsd] compile-flags: --target aarch64-unknown-netbsd
 //@ [aarch64_unknown_netbsd] needs-llvm-components: aarch64
@@ -193,6 +208,9 @@
 //@ revisions: armv7a_nuttx_eabihf
 //@ [armv7a_nuttx_eabihf] compile-flags: --target armv7a-nuttx-eabihf
 //@ [armv7a_nuttx_eabihf] needs-llvm-components: arm
+//@ revisions: armv7a_vex_v5
+//@ [armv7a_vex_v5] compile-flags: --target armv7a-vex-v5
+//@ [armv7a_vex_v5] needs-llvm-components: arm
 //@ revisions: armv7r_none_eabi
 //@ [armv7r_none_eabi] compile-flags: --target armv7r-none-eabi
 //@ [armv7r_none_eabi] needs-llvm-components: arm
@@ -241,6 +259,9 @@
 //@ revisions: i686_unknown_haiku
 //@ [i686_unknown_haiku] compile-flags: --target i686-unknown-haiku
 //@ [i686_unknown_haiku] needs-llvm-components: x86
+//@ revisions: i686_unknown_helenos
+//@ [i686_unknown_helenos] compile-flags: --target i686-unknown-helenos
+//@ [i686_unknown_helenos] needs-llvm-components: x86
 //@ revisions: i686_unknown_hurd_gnu
 //@ [i686_unknown_hurd_gnu] compile-flags: --target i686-unknown-hurd-gnu
 //@ [i686_unknown_hurd_gnu] needs-llvm-components: x86
@@ -379,6 +400,9 @@
 //@ revisions: powerpc_unknown_freebsd
 //@ [powerpc_unknown_freebsd] compile-flags: --target powerpc-unknown-freebsd
 //@ [powerpc_unknown_freebsd] needs-llvm-components: powerpc
+//@ revisions: powerpc_unknown_helenos
+//@ [powerpc_unknown_helenos] compile-flags: --target powerpc-unknown-helenos
+//@ [powerpc_unknown_helenos] needs-llvm-components: powerpc
 //@ revisions: powerpc_unknown_linux_gnu
 //@ [powerpc_unknown_linux_gnu] compile-flags: --target powerpc-unknown-linux-gnu
 //@ [powerpc_unknown_linux_gnu] needs-llvm-components: powerpc
@@ -472,9 +496,15 @@
 //@ revisions: riscv64gc_unknown_linux_gnu
 //@ [riscv64gc_unknown_linux_gnu] compile-flags: --target riscv64gc-unknown-linux-gnu
 //@ [riscv64gc_unknown_linux_gnu] needs-llvm-components: riscv
+//@ revisions: riscv64a23_unknown_linux_gnu
+//@ [riscv64a23_unknown_linux_gnu] compile-flags: --target riscv64a23-unknown-linux-gnu
+//@ [riscv64a23_unknown_linux_gnu] needs-llvm-components: riscv
 //@ revisions: riscv64gc_unknown_linux_musl
 //@ [riscv64gc_unknown_linux_musl] compile-flags: --target riscv64gc-unknown-linux-musl
 //@ [riscv64gc_unknown_linux_musl] needs-llvm-components: riscv
+//@ revisions: riscv64gc_unknown_managarm_mlibc
+//@ [riscv64gc_unknown_managarm_mlibc] compile-flags: --target riscv64gc-unknown-managarm-mlibc
+//@ [riscv64gc_unknown_managarm_mlibc] needs-llvm-components: riscv
 //@ revisions: riscv64gc_unknown_netbsd
 //@ [riscv64gc_unknown_netbsd] compile-flags: --target riscv64gc-unknown-netbsd
 //@ [riscv64gc_unknown_netbsd] needs-llvm-components: riscv
@@ -484,6 +514,9 @@
 //@ revisions: riscv64gc_unknown_openbsd
 //@ [riscv64gc_unknown_openbsd] compile-flags: --target riscv64gc-unknown-openbsd
 //@ [riscv64gc_unknown_openbsd] needs-llvm-components: riscv
+//@ revisions: riscv64gc_unknown_redox
+//@ [riscv64gc_unknown_redox] compile-flags: --target riscv64gc-unknown-redox
+//@ [riscv64gc_unknown_redox] needs-llvm-components: riscv
 //@ revisions: riscv64imac_unknown_none_elf
 //@ [riscv64imac_unknown_none_elf] compile-flags: --target riscv64imac-unknown-none-elf
 //@ [riscv64imac_unknown_none_elf] needs-llvm-components: riscv
@@ -493,6 +526,9 @@
 //@ revisions: s390x_unknown_linux_musl
 //@ [s390x_unknown_linux_musl] compile-flags: --target s390x-unknown-linux-musl
 //@ [s390x_unknown_linux_musl] needs-llvm-components: systemz
+//@ revisions: sparc64_unknown_helenos
+//@ [sparc64_unknown_helenos] compile-flags: --target sparc64-unknown-helenos
+//@ [sparc64_unknown_helenos] needs-llvm-components: sparc
 //@ revisions: sparc64_unknown_linux_gnu
 //@ [sparc64_unknown_linux_gnu] compile-flags: --target sparc64-unknown-linux-gnu
 //@ [sparc64_unknown_linux_gnu] needs-llvm-components: sparc
@@ -565,6 +601,9 @@
 //@ revisions: wasm32_wasip2
 //@ [wasm32_wasip2] compile-flags: --target wasm32-wasip2
 //@ [wasm32_wasip2] needs-llvm-components: webassembly
+//@ revisions: wasm32_wasip3
+//@ [wasm32_wasip3] compile-flags: --target wasm32-wasip3
+//@ [wasm32_wasip3] needs-llvm-components: webassembly
 //@ revisions: wasm32_wali_linux_musl
 //@ [wasm32_wali_linux_musl] compile-flags: --target wasm32-wali-linux-musl
 //@ [wasm32_wali_linux_musl] needs-llvm-components: webassembly
@@ -607,6 +646,9 @@
 //@ revisions: x86_64_unknown_haiku
 //@ [x86_64_unknown_haiku] compile-flags: --target x86_64-unknown-haiku
 //@ [x86_64_unknown_haiku] needs-llvm-components: x86
+//@ revisions: x86_64_unknown_helenos
+//@ [x86_64_unknown_helenos] compile-flags: --target x86_64-unknown-helenos
+//@ [x86_64_unknown_helenos] needs-llvm-components: x86
 //@ revisions: x86_64_unknown_hurd_gnu
 //@ [x86_64_unknown_hurd_gnu] compile-flags: --target x86_64-unknown-hurd-gnu
 //@ [x86_64_unknown_hurd_gnu] needs-llvm-components: x86
@@ -634,6 +676,12 @@
 //@ revisions: x86_64_unknown_linux_none
 //@ [x86_64_unknown_linux_none] compile-flags: --target x86_64-unknown-linux-none
 //@ [x86_64_unknown_linux_none] needs-llvm-components: x86
+//@ revisions: x86_64_unknown_managarm_mlibc
+//@ [x86_64_unknown_managarm_mlibc] compile-flags: --target x86_64-unknown-managarm-mlibc
+//@ [x86_64_unknown_managarm_mlibc] needs-llvm-components: x86
+//@ revisions: x86_64_unknown_motor
+//@ [x86_64_unknown_motor] compile-flags: --target x86_64-unknown-motor
+//@ [x86_64_unknown_motor] needs-llvm-components: x86
 //@ revisions: x86_64_unknown_netbsd
 //@ [x86_64_unknown_netbsd] compile-flags: --target x86_64-unknown-netbsd
 //@ [x86_64_unknown_netbsd] needs-llvm-components: x86
diff --git a/tests/assembly-llvm/x86-return-float.rs b/tests/assembly-llvm/x86-return-float.rs
index 4db93f68485a..1b283c2759b3 100644
--- a/tests/assembly-llvm/x86-return-float.rs
+++ b/tests/assembly-llvm/x86-return-float.rs
@@ -1,9 +1,6 @@
 //@ assembly-output: emit-asm
 // FIXME(#114479): LLVM miscompiles loading and storing `f32` and `f64` when SSE is disabled.
-// There's no compiletest directive to ignore a test on i586 only, so just always explicitly enable
-// SSE2.
-// Use the same target CPU as `i686` so that LLVM orders the instructions in the same order.
-//@ compile-flags: -Ctarget-feature=+sse2 -Ctarget-cpu=pentium4
+// (As of #136758, this test cross-compiles to selected i686 targets only, which have SSE.)
 // Force frame pointers to make ASM more consistent between targets
 //@ compile-flags: -C force-frame-pointers
 // At opt-level=3, LLVM can merge two movss into one movsd, and we aren't testing for that.
diff --git a/tests/assembly-llvm/x86_64-bigint-helpers.rs b/tests/assembly-llvm/x86_64-bigint-helpers.rs
index 58785932bc2f..9d998a31cf30 100644
--- a/tests/assembly-llvm/x86_64-bigint-helpers.rs
+++ b/tests/assembly-llvm/x86_64-bigint-helpers.rs
@@ -2,9 +2,6 @@
 //@ assembly-output: emit-asm
 //@ compile-flags: --crate-type=lib -Copt-level=3 -C target-cpu=x86-64-v4
 //@ compile-flags: -C llvm-args=-x86-asm-syntax=intel
-//@ revisions: llvm-pre-20 llvm-20
-//@ [llvm-20] min-llvm-version: 20
-//@ [llvm-pre-20] max-llvm-major-version: 19
 
 #![no_std]
 #![feature(bigint_helper_methods)]
@@ -23,16 +20,15 @@ pub unsafe extern "sysv64" fn bigint_chain_carrying_add(
     n: usize,
     mut carry: bool,
 ) -> bool {
-    // llvm-pre-20: mov [[TEMP:r..]], qword ptr [rsi + 8*[[IND:r..]] + 8]
-    // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8]
-    // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP]]
-    // llvm-pre-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 16]
-    // llvm-pre-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 16]
-    // llvm-pre-20: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP]]
-    // llvm-20: adc [[TEMP:r..]], qword ptr [rdx + 8*[[IND:r..]]]
-    // llvm-20: mov qword ptr [rdi + 8*[[IND]]], [[TEMP]]
-    // llvm-20: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 8]
-    // llvm-20: adc [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8]
+    // Even if we emit A+B, LLVM will sometimes reorder that to B+A, so this
+    // test doesn't actually check which register is mov vs which is adc.
+
+    // CHECK: mov [[TEMP1:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND:.+]] + 8]
+    // CHECK: adc [[TEMP1]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 8]
+    // CHECK: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP1]]
+    // CHECK: mov [[TEMP2:.+]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16]
+    // CHECK: adc [[TEMP2]], qword ptr [{{rdx|rsi}} + 8*[[IND]] + 16]
+    // CHECK: mov qword ptr [rdi + 8*[[IND]] + 16], [[TEMP2]]
     for i in 0..n {
         (*dest.add(i), carry) = u64::carrying_add(*src1.add(i), *src2.add(i), carry);
     }
@@ -48,7 +44,7 @@ pub unsafe extern "sysv64" fn bigint_chain_borrowing_sub(
     n: usize,
     mut carry: bool,
 ) -> bool {
-    // CHECK: mov [[TEMP:r..]], qword ptr [rsi + 8*[[IND:r..]] + 8]
+    // CHECK: mov [[TEMP:r.+]], qword ptr [rsi + 8*[[IND:r.+]] + 8]
     // CHECK: sbb [[TEMP]], qword ptr [rdx + 8*[[IND]] + 8]
     // CHECK: mov qword ptr [rdi + 8*[[IND]] + 8], [[TEMP]]
     // CHECK: mov [[TEMP]], qword ptr [rsi + 8*[[IND]] + 16]
diff --git a/tests/assembly-llvm/x86_64-cmp.rs b/tests/assembly-llvm/x86_64-cmp.rs
index 26c9013d96fc..1f1fe7fd0052 100644
--- a/tests/assembly-llvm/x86_64-cmp.rs
+++ b/tests/assembly-llvm/x86_64-cmp.rs
@@ -1,12 +1,6 @@
-//@ revisions: LLVM-PRE-20-DEBUG LLVM-20-DEBUG LLVM-PRE-20-OPTIM LLVM-20-OPTIM
-//@ [LLVM-PRE-20-DEBUG] compile-flags: -C opt-level=0
-//@ [LLVM-PRE-20-DEBUG] max-llvm-major-version: 19
-//@ [LLVM-20-DEBUG] compile-flags: -C opt-level=0
-//@ [LLVM-20-DEBUG] min-llvm-version: 20
-//@ [LLVM-PRE-20-OPTIM] compile-flags: -C opt-level=3
-//@ [LLVM-PRE-20-OPTIM] max-llvm-major-version: 19
-//@ [LLVM-20-OPTIM] compile-flags: -C opt-level=3
-//@ [LLVM-20-OPTIM] min-llvm-version: 20
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
 //@ assembly-output: emit-asm
 //@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel
 //@ only-x86_64
@@ -19,61 +13,22 @@ use std::intrinsics::three_way_compare;
 #[no_mangle]
 // CHECK-LABEL: signed_cmp:
 pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
-    // LLVM-PRE-20-DEBUG: cmp
-    // LLVM-PRE-20-DEBUG: setg
-    // LLVM-PRE-20-DEBUG: and
-    // LLVM-PRE-20-DEBUG: cmp
-    // LLVM-PRE-20-DEBUG: setl
-    // LLVM-PRE-20-DEBUG: and
-    // LLVM-PRE-20-DEBUG: sub
-    //
-    // LLVM-20-DEBUG: sub
-    // LLVM-20-DEBUG: setl
-    // LLVM-20-DEBUG: setg
-    // LLVM-20-DEBUG: sub
-    // LLVM-20-DEBUG: ret
-
-    // LLVM-PRE-20-OPTIM: xor
-    // LLVM-PRE-20-OPTIM: cmp
-    // LLVM-PRE-20-OPTIM: setne
-    // LLVM-PRE-20-OPTIM: mov
-    // LLVM-PRE-20-OPTIM: cmovge
-    // LLVM-PRE-20-OPTIM: ret
-    //
-    // LLVM-20-OPTIM: cmp
-    // LLVM-20-OPTIM: setl
-    // LLVM-20-OPTIM: setg
-    // LLVM-20-OPTIM: sub
-    // LLVM-20-OPTIM: ret
+    // DEBUG: sub
+    // OPTIM: cmp
+    // CHECK: setl
+    // CHECK: setg
+    // CHECK: sub
+    // CHECK: ret
     three_way_compare(a, b)
 }
 
 #[no_mangle]
 // CHECK-LABEL: unsigned_cmp:
 pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
-    // LLVM-PRE-20-DEBUG: cmp
-    // LLVM-PRE-20-DEBUG: seta
-    // LLVM-PRE-20-DEBUG: and
-    // LLVM-PRE-20-DEBUG: cmp
-    // LLVM-PRE-20-DEBUG: setb
-    // LLVM-PRE-20-DEBUG: and
-    // LLVM-PRE-20-DEBUG: sub
-    //
-    // LLVM-20-DEBUG: sub
-    // LLVM-20-DEBUG: seta
-    // LLVM-20-DEBUG: sbb
-    // LLVM-20-DEBUG: ret
-
-    // LLVM-PRE-20-OPTIM: xor
-    // LLVM-PRE-20-OPTIM: cmp
-    // LLVM-PRE-20-OPTIM: setne
-    // LLVM-PRE-20-OPTIM: mov
-    // LLVM-PRE-20-OPTIM: cmovae
-    // LLVM-PRE-20-OPTIM: ret
-    //
-    // LLVM-20-OPTIM: cmp
-    // LLVM-20-OPTIM: seta
-    // LLVM-20-OPTIM: sbb
-    // LLVM-20-OPTIM: ret
+    // DEBUG: sub
+    // OPTIM: cmp
+    // CHECK: seta
+    // CHECK: sbb
+    // CHECK: ret
     three_way_compare(a, b)
 }
diff --git a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-load.rs b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-load.rs
index f5e2f18e68e4..2892ff2882a7 100644
--- a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-load.rs
+++ b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-load.rs
@@ -1,17 +1,24 @@
-// Test LVI load hardening on SGX enclave code
+// Test LVI load hardening on SGX enclave code, specifically that `ret` is rewritten.
 
+//@ add-core-stubs
 //@ assembly-output: emit-asm
-//@ compile-flags: --crate-type staticlib
-//@ only-x86_64-fortanix-unknown-sgx
+//@ compile-flags: --target x86_64-fortanix-unknown-sgx -Copt-level=0
+//@ needs-llvm-components: x86
+
+#![feature(no_core, lang_items, f16)]
+#![crate_type = "lib"]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
 
 #[no_mangle]
-pub extern "C" fn plus_one(r: &mut u64) {
-    *r = *r + 1;
+pub extern "C" fn dereference(a: &mut u64) -> u64 {
+    // CHECK-LABEL: dereference
+    // CHECK: lfence
+    // CHECK: mov
+    // CHECK: popq [[REGISTER:%[a-z]+]]
+    // CHECK-NEXT: lfence
+    // CHECK-NEXT: jmpq *[[REGISTER]]
+    *a
 }
-
-// CHECK: plus_one
-// CHECK: lfence
-// CHECK-NEXT: incq
-// CHECK: popq [[REGISTER:%[a-z]+]]
-// CHECK-NEXT: lfence
-// CHECK-NEXT: jmpq *[[REGISTER]]
diff --git a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-ret.rs b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-ret.rs
index f16d68fa2554..a0cedc3bc2da 100644
--- a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-ret.rs
+++ b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-generic-ret.rs
@@ -1,12 +1,20 @@
 // Test LVI ret hardening on generic rust code
 
+//@ add-core-stubs
 //@ assembly-output: emit-asm
-//@ compile-flags: --crate-type staticlib
-//@ only-x86_64-fortanix-unknown-sgx
+//@ compile-flags: --target x86_64-fortanix-unknown-sgx
+//@ needs-llvm-components: x86
+
+#![feature(no_core, lang_items, f16)]
+#![crate_type = "lib"]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
 
 #[no_mangle]
 pub extern "C" fn myret() {}
-// CHECK: myret:
+// CHECK-LABEL: myret:
 // CHECK: popq [[REGISTER:%[a-z]+]]
 // CHECK-NEXT: lfence
 // CHECK-NEXT: jmpq *[[REGISTER]]
diff --git a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-inline-assembly.rs b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-inline-assembly.rs
index a729df8e1660..215fb4b804ac 100644
--- a/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-inline-assembly.rs
+++ b/tests/assembly-llvm/x86_64-fortanix-unknown-sgx-lvi-inline-assembly.rs
@@ -1,13 +1,22 @@
 // Test LVI load hardening on SGX inline assembly code
 
+//@ add-core-stubs
 //@ assembly-output: emit-asm
-//@ compile-flags: --crate-type staticlib
-//@ only-x86_64-fortanix-unknown-sgx
+//@ compile-flags: --target x86_64-fortanix-unknown-sgx
+//@ needs-llvm-components: x86
 
-use std::arch::asm;
+#![feature(no_core, lang_items, f16)]
+#![crate_type = "lib"]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
 
 #[no_mangle]
 pub extern "C" fn get(ptr: *const u64) -> u64 {
+    // CHECK-LABEL: get
+    // CHECK: movq
+    // CHECK-NEXT: lfence
     let value: u64;
     unsafe {
         asm!("mov {}, [{}]",
@@ -17,18 +26,13 @@ pub extern "C" fn get(ptr: *const u64) -> u64 {
     value
 }
 
-// CHECK: get
-// CHECK: movq
-// CHECK-NEXT: lfence
-
 #[no_mangle]
 pub extern "C" fn myret() {
+    // CHECK-LABEL: myret
+    // CHECK: shlq $0, (%rsp)
+    // CHECK-NEXT: lfence
+    // CHECK-NEXT: retq
     unsafe {
         asm!("ret");
     }
 }
-
-// CHECK: myret
-// CHECK: shlq $0, (%rsp)
-// CHECK-NEXT: lfence
-// CHECK-NEXT: retq
diff --git a/tests/assembly-llvm/x86_64-indirect-branch-cs-prefix.rs b/tests/assembly-llvm/x86_64-indirect-branch-cs-prefix.rs
new file mode 100644
index 000000000000..8e4470ee4514
--- /dev/null
+++ b/tests/assembly-llvm/x86_64-indirect-branch-cs-prefix.rs
@@ -0,0 +1,27 @@
+// Test that the `cs` prefix is (not) added into a `call` and a `jmp` to the
+// indirect thunk when the `-Zindirect-branch-cs-prefix` flag is (not) set.
+
+//@ revisions: unset set
+//@ assembly-output: emit-asm
+//@ compile-flags: -Copt-level=3 -Cunsafe-allow-abi-mismatch=retpoline,retpoline-external-thunk,indirect-branch-cs-prefix -Zretpoline-external-thunk
+//@ [set] compile-flags: -Zindirect-branch-cs-prefix
+//@ only-x86_64
+//@ ignore-apple Symbol is called `___x86_indirect_thunk` (Darwin's extra underscore)
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: foo:
+#[no_mangle]
+pub fn foo(g: fn()) {
+    // unset-NOT: cs
+    // unset: callq {{__x86_indirect_thunk.*}}
+    // set: cs
+    // set-NEXT: callq {{__x86_indirect_thunk.*}}
+    g();
+
+    // unset-NOT: cs
+    // unset: jmp {{__x86_indirect_thunk.*}}
+    // set: cs
+    // set-NEXT: jmp {{__x86_indirect_thunk.*}}
+    g();
+}
diff --git a/tests/assembly-llvm/x86_64-mcount.rs b/tests/assembly-llvm/x86_64-mcount.rs
new file mode 100644
index 000000000000..0428272cfc59
--- /dev/null
+++ b/tests/assembly-llvm/x86_64-mcount.rs
@@ -0,0 +1,24 @@
+//@ assembly-output: emit-asm
+//@ compile-flags: -Zinstrument-mcount=y -Cllvm-args=-x86-asm-syntax=intel
+
+//@ revisions: x86_64-linux
+//@[x86_64-linux] compile-flags: --target=x86_64-unknown-linux-gnu
+//@[x86_64-linux] needs-llvm-components: x86
+//@[x86_64-linux] only-x86_64-unknown-linux-gnu
+
+//@ revisions: x86_64-darwin
+//@[x86_64-darwin] compile-flags: --target=x86_64-apple-darwin
+//@[x86_64-darwin] needs-llvm-components: x86
+//@[x86_64-darwin] only-x86_64-apple-darwin
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: mcount_func:
+#[no_mangle]
+pub fn mcount_func() {
+    // CHECK: call mcount
+
+    std::hint::black_box(());
+
+    // CHECK: ret
+}
diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs
index 47dadd51ce0f..4f4c653cb46e 100644
--- a/tests/auxiliary/minicore.rs
+++ b/tests/auxiliary/minicore.rs
@@ -179,7 +179,14 @@ impl Add for isize {
 
 #[lang = "sync"]
 trait Sync {}
-impl Sync for u8 {}
+impl_marker_trait!(
+    Sync => [
+        char, bool,
+        isize, i8, i16, i32, i64, i128,
+        usize, u8, u16, u32, u64, u128,
+        f16, f32, f64, f128,
+    ]
+);
 
 #[lang = "drop_in_place"]
 fn drop_in_place(_: *mut T) {}
@@ -225,3 +232,10 @@ pub mod mem {
     #[rustc_intrinsic]
     pub unsafe fn transmute(src: Src) -> Dst;
 }
+
+#[lang = "c_void"]
+#[repr(u8)]
+pub enum c_void {
+    __variant1,
+    __variant2,
+}
diff --git a/tests/codegen-llvm/abi-x86-interrupt.rs b/tests/codegen-llvm/abi-x86-interrupt.rs
index 9a1ded2c9e37..b5c495803d89 100644
--- a/tests/codegen-llvm/abi-x86-interrupt.rs
+++ b/tests/codegen-llvm/abi-x86-interrupt.rs
@@ -15,4 +15,4 @@ use minicore::*;
 
 // CHECK: define x86_intrcc void @has_x86_interrupt_abi
 #[no_mangle]
-pub extern "x86-interrupt" fn has_x86_interrupt_abi() {}
+pub extern "x86-interrupt" fn has_x86_interrupt_abi(_p: *const u8) {}
diff --git a/tests/codegen-llvm/addr-of-mutate.rs b/tests/codegen-llvm/addr-of-mutate.rs
index 14bc4b8ab28c..d59d85af62a9 100644
--- a/tests/codegen-llvm/addr-of-mutate.rs
+++ b/tests/codegen-llvm/addr-of-mutate.rs
@@ -5,7 +5,7 @@
 // Test for the absence of `readonly` on the argument when it is mutated via `&raw const`.
 // See .
 
-// CHECK: i8 @foo(ptr noalias{{( nocapture)?}} noundef align 1{{( captures\(none\))?}} dereferenceable(128) %x)
+// CHECK: i8 @foo(ptr{{( dead_on_return)?}} noalias noundef align 1{{( captures\(address\))?}} dereferenceable(128) %x)
 #[no_mangle]
 pub fn foo(x: [u8; 128]) -> u8 {
     let ptr = core::ptr::addr_of!(x).cast_mut();
@@ -15,7 +15,7 @@ pub fn foo(x: [u8; 128]) -> u8 {
     x[0]
 }
 
-// CHECK: i1 @second(ptr noalias{{( nocapture)?}} noundef align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+// CHECK: i1 @second(ptr{{( dead_on_return)?}} noalias noundef align {{[0-9]+}}{{( captures\(address\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
 #[no_mangle]
 pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
     let b_bool_ptr = core::ptr::addr_of!(a_ptr_and_b.1.1).cast_mut();
@@ -24,7 +24,7 @@ pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
 }
 
 // If going through a deref (and there are no other mutating accesses), then `readonly` is fine.
-// CHECK: i1 @third(ptr noalias{{( nocapture)?}} noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+// CHECK: i1 @third(ptr{{( dead_on_return)?}} noalias noundef readonly align {{[0-9]+}}{{( captures\(none\))?}} dereferenceable({{[0-9]+}}) %a_ptr_and_b)
 #[no_mangle]
 pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
     let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut();
diff --git a/tests/codegen-llvm/align-byval-alignment-mismatch.rs b/tests/codegen-llvm/align-byval-alignment-mismatch.rs
index c69fc2de9d25..8eaf2751ed78 100644
--- a/tests/codegen-llvm/align-byval-alignment-mismatch.rs
+++ b/tests/codegen-llvm/align-byval-alignment-mismatch.rs
@@ -55,10 +55,10 @@ extern "C" {
 pub unsafe fn rust_to_c_increases_alignment(x: Align1) {
     // i686-linux: start:
     // i686-linux-NEXT: [[ALLOCA:%[0-9a-z]+]] = alloca [48 x i8], align 4
-    // i686-linux-NEXT: call void @llvm.lifetime.start.p0(i64 48, ptr {{.*}}[[ALLOCA]])
+    // i686-linux-NEXT: call void @llvm.lifetime.start.p0({{(i64 48, )?}}ptr {{.*}}[[ALLOCA]])
     // i686-linux-NEXT: call void @llvm.memcpy.{{.+}}(ptr {{.*}}align 4 {{.*}}[[ALLOCA]], ptr {{.*}}align 1 {{.*}}%x
     // i686-linux-NEXT: call void @extern_c_align1({{.+}} [[ALLOCA]])
-    // i686-linux-NEXT: call void @llvm.lifetime.end.p0(i64 48, ptr {{.*}}[[ALLOCA]])
+    // i686-linux-NEXT: call void @llvm.lifetime.end.p0({{(i64 48, )?}}ptr {{.*}}[[ALLOCA]])
 
     // x86_64-linux: start:
     // x86_64-linux-NEXT: call void @extern_c_align1
diff --git a/tests/codegen-llvm/align-static.rs b/tests/codegen-llvm/align-static.rs
new file mode 100644
index 000000000000..53db998919af
--- /dev/null
+++ b/tests/codegen-llvm/align-static.rs
@@ -0,0 +1,31 @@
+//@ compile-flags: -C no-prepopulate-passes -Z mir-opt-level=0
+
+#![crate_type = "lib"]
+#![feature(static_align)]
+
+// CHECK: @STATIC_ALIGN =
+// CHECK-SAME: align 16
+#[no_mangle]
+#[rustc_align_static(16)]
+pub static STATIC_ALIGN: u64 = 0;
+
+// CHECK: @ALIGN_SPECIFIED_TWICE_1 =
+// CHECK-SAME: align 64
+#[no_mangle]
+#[rustc_align_static(32)]
+#[rustc_align_static(64)]
+pub static ALIGN_SPECIFIED_TWICE_1: u64 = 0;
+
+// CHECK: @ALIGN_SPECIFIED_TWICE_2 =
+// CHECK-SAME: align 128
+#[no_mangle]
+#[rustc_align_static(128)]
+#[rustc_align_static(32)]
+pub static ALIGN_SPECIFIED_TWICE_2: u64 = 0;
+
+// CHECK: @ALIGN_SPECIFIED_TWICE_3 =
+// CHECK-SAME: align 256
+#[no_mangle]
+#[rustc_align_static(32)]
+#[rustc_align_static(256)]
+pub static ALIGN_SPECIFIED_TWICE_3: u64 = 0;
diff --git a/tests/codegen-llvm/amdgpu-addrspacecast.rs b/tests/codegen-llvm/amdgpu-addrspacecast.rs
index 7fe630a7efa0..829133de00d8 100644
--- a/tests/codegen-llvm/amdgpu-addrspacecast.rs
+++ b/tests/codegen-llvm/amdgpu-addrspacecast.rs
@@ -16,3 +16,12 @@ pub fn ref_of_local(f: fn(&i32)) {
     let i = 0;
     f(&i);
 }
+
+// CHECK-LABEL: @ref_of_global
+// CHECK: addrspacecast (ptr addrspace(1) @I to ptr)
+#[no_mangle]
+pub fn ref_of_global(f: fn(&i32)) {
+    #[no_mangle]
+    static I: i32 = 0;
+    f(&I);
+}
diff --git a/tests/codegen-llvm/array-repeat.rs b/tests/codegen-llvm/array-repeat.rs
index 4c755df93903..1c45341d764c 100644
--- a/tests/codegen-llvm/array-repeat.rs
+++ b/tests/codegen-llvm/array-repeat.rs
@@ -1,7 +1,6 @@
 //@ compile-flags: -Copt-level=3
 
 #![crate_type = "lib"]
-#![feature(array_repeat)]
 
 use std::array::repeat;
 
diff --git a/tests/codegen-llvm/asm/powerpc-clobbers.rs b/tests/codegen-llvm/asm/powerpc-clobbers.rs
index f7fc7eea5d50..9d8204fa3be9 100644
--- a/tests/codegen-llvm/asm/powerpc-clobbers.rs
+++ b/tests/codegen-llvm/asm/powerpc-clobbers.rs
@@ -56,13 +56,59 @@ pub unsafe fn v0_clobber() {
     asm!("", out("v0") _, options(nostack, nomem, preserves_flags));
 }
 
-// Output format depends on the availability of altivec.
+// Output format depends on the availability of vsx.
+// CHECK-LABEL: @vs32_clobber
+// powerpc: call void asm sideeffect "", "~{vs32}"()
+// powerpc64: call void asm sideeffect "", "~{vs32}"()
+// powerpc64le: call <4 x i32> asm sideeffect "", "=&{vs32}"()
+// aix64: call <4 x i32> asm sideeffect "", "=&{vs32}"()
+#[no_mangle]
+pub unsafe fn vs32_clobber() {
+    asm!("", out("vs32") _, options(nostack, nomem, preserves_flags));
+}
+
+// Output format depends on the availability of altivec and vsx
 // CHECK-LABEL: @clobber_abi
-// powerpc: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{xer}"()
-// powerpc64: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{xer}"()
-// powerpc64le: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{xer}"()
-// aix64: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{xer}"()
+// powerpc: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},~{vs0},~{vs1},~{vs2},~{vs3},~{vs4},~{vs5},~{vs6},~{vs7},~{vs8},~{vs9},~{vs10},~{vs11},~{vs12},~{vs13},~{vs14},~{vs15},~{vs16},~{vs17},~{vs18},~{vs19},~{vs20},~{vs21},~{vs22},~{vs23},~{vs24},~{vs25},~{vs26},~{vs27},~{vs28},~{vs29},~{vs30},~{vs31},~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// powerpc64: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{vs0},~{vs1},~{vs2},~{vs3},~{vs4},~{vs5},~{vs6},~{vs7},~{vs8},~{vs9},~{vs10},~{vs11},~{vs12},~{vs13},~{vs14},~{vs15},~{vs16},~{vs17},~{vs18},~{vs19},~{vs20},~{vs21},~{vs22},~{vs23},~{vs24},~{vs25},~{vs26},~{vs27},~{vs28},~{vs29},~{vs30},~{vs31},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// powerpc64le: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={vs0},={vs1},={vs2},={vs3},={vs4},={vs5},={vs6},={vs7},={vs8},={vs9},={vs10},={vs11},={vs12},={vs13},={vs14},={vs15},={vs16},={vs17},={vs18},={vs19},={vs20},={vs21},={vs22},={vs23},={vs24},={vs25},={vs26},={vs27},={vs28},={vs29},={vs30},={vs31},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// aix64: asm sideeffect "", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={vs0},={vs1},={vs2},={vs3},={vs4},={vs5},={vs6},={vs7},={vs8},={vs9},={vs10},={vs11},={vs12},={vs13},={vs14},={vs15},={vs16},={vs17},={vs18},={vs19},={vs20},={vs21},={vs22},={vs23},={vs24},={vs25},={vs26},={vs27},={vs28},={vs29},={vs30},={vs31},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
 #[no_mangle]
 pub unsafe fn clobber_abi() {
     asm!("", clobber_abi("C"), options(nostack, nomem, preserves_flags));
 }
+
+// CHECK-LABEL: @clobber_no_preserves_flags
+// CHECK: call void asm sideeffect "nop", ""()
+#[no_mangle]
+pub unsafe fn clobber_no_preserves_flags() {
+    // Use a nop to prevent aliasing of identical functions here.
+    asm!("nop", options(nostack, nomem));
+}
+
+// CHECK-LABEL: @cr0_clobber_no_preserves_flags
+// CHECK: call void asm sideeffect "nop; nop", "~{cr0}"()
+#[no_mangle]
+pub unsafe fn cr0_clobber_no_preserves_flags() {
+    // Use nop; nop to prevent aliasing of identical functions here.
+    asm!("nop; nop", out("cr0") _, options(nostack, nomem));
+}
+
+// CHECK-LABEL: @clobber_preservesflags
+// CHECK: call void asm sideeffect "", "~{memory}"()
+#[no_mangle]
+pub unsafe fn clobber_preservesflags() {
+    asm!("", options(nostack, preserves_flags));
+}
+
+// Output format depends on the availability of altivec and vsx
+// CHECK-LABEL: @clobber_abi_no_preserves_flags
+#[no_mangle]
+// powerpc: asm sideeffect "nop", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},~{vs0},~{vs1},~{vs2},~{vs3},~{vs4},~{vs5},~{vs6},~{vs7},~{vs8},~{vs9},~{vs10},~{vs11},~{vs12},~{vs13},~{vs14},~{vs15},~{vs16},~{vs17},~{vs18},~{vs19},~{vs20},~{vs21},~{vs22},~{vs23},~{vs24},~{vs25},~{vs26},~{vs27},~{vs28},~{vs29},~{vs30},~{vs31},~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// powerpc64: asm sideeffect "nop", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{vs0},~{vs1},~{vs2},~{vs3},~{vs4},~{vs5},~{vs6},~{vs7},~{vs8},~{vs9},~{vs10},~{vs11},~{vs12},~{vs13},~{vs14},~{vs15},~{vs16},~{vs17},~{vs18},~{vs19},~{vs20},~{vs21},~{vs22},~{vs23},~{vs24},~{vs25},~{vs26},~{vs27},~{vs28},~{vs29},~{vs30},~{vs31},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// powerpc64le: asm sideeffect "nop", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={vs0},={vs1},={vs2},={vs3},={vs4},={vs5},={vs6},={vs7},={vs8},={vs9},={vs10},={vs11},={vs12},={vs13},={vs14},={vs15},={vs16},={vs17},={vs18},={vs19},={vs20},={vs21},={vs22},={vs23},={vs24},={vs25},={vs26},={vs27},={vs28},={vs29},={vs30},={vs31},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+// aix64: asm sideeffect "nop", "={r0},={r3},={r4},={r5},={r6},={r7},={r8},={r9},={r10},={r11},={r12},={f0},={f1},={f2},={f3},={f4},={f5},={f6},={f7},={f8},={f9},={f10},={f11},={f12},={f13},={vs0},={vs1},={vs2},={vs3},={vs4},={vs5},={vs6},={vs7},={vs8},={vs9},={vs10},={vs11},={vs12},={vs13},={vs14},={vs15},={vs16},={vs17},={vs18},={vs19},={vs20},={vs21},={vs22},={vs23},={vs24},={vs25},={vs26},={vs27},={vs28},={vs29},={vs30},={vs31},={v0},={v1},={v2},={v3},={v4},={v5},={v6},={v7},={v8},={v9},={v10},={v11},={v12},={v13},={v14},={v15},={v16},={v17},={v18},={v19},~{cr0},~{cr1},~{cr5},~{cr6},~{cr7},~{ctr},~{lr},~{xer}"()
+pub unsafe fn clobber_abi_no_preserves_flags() {
+    // Use a nop to prevent aliasing of identical functions here.
+    asm!("nop", clobber_abi("C"), options(nostack, nomem));
+}
diff --git a/tests/codegen-llvm/asm/readonly-not-pure.rs b/tests/codegen-llvm/asm/readonly-not-pure.rs
new file mode 100644
index 000000000000..a3c0e276c7f5
--- /dev/null
+++ b/tests/codegen-llvm/asm/readonly-not-pure.rs
@@ -0,0 +1,48 @@
+//@ add-core-stubs
+//@ compile-flags: -Copt-level=3 --target x86_64-unknown-linux-gnu
+//@ needs-llvm-components: x86
+
+#![crate_type = "rlib"]
+#![feature(no_core)]
+#![no_core]
+
+// Test that when an inline assembly block specifies `readonly` but not `pure`, a detailed
+// `MemoryEffects` is provided to LLVM: this assembly block is not allowed to perform writes,
+// but it may have side-effects.
+
+extern crate minicore;
+use minicore::*;
+
+pub static mut VAR: i32 = 0;
+
+// CHECK-LABEL: @no_options
+// CHECK: call i32 asm
+#[no_mangle]
+pub unsafe fn no_options() -> i32 {
+    VAR = 1;
+    let _ignored: i32;
+    asm!("mov {0}, 1", out(reg) _ignored);
+    VAR
+}
+
+// CHECK-LABEL: @readonly_pure
+// CHECK-NOT: call i32 asm
+#[no_mangle]
+pub unsafe fn readonly_pure() -> i32 {
+    VAR = 1;
+    let _ignored: i32;
+    asm!("mov {0}, 1", out(reg) _ignored, options(pure, readonly));
+    VAR
+}
+
+// CHECK-LABEL: @readonly_not_pure
+// CHECK: call i32 asm {{.*}} #[[ATTR:[0-9]+]]
+#[no_mangle]
+pub unsafe fn readonly_not_pure() -> i32 {
+    VAR = 1;
+    let _ignored: i32;
+    asm!("mov {0}, 1", out(reg) _ignored, options(readonly));
+    VAR
+}
+
+// CHECK: attributes #[[ATTR]] = { nounwind memory(read, inaccessiblemem: readwrite) }
diff --git a/tests/codegen-llvm/asm/riscv-clobbers.rs b/tests/codegen-llvm/asm/riscv-clobbers.rs
index e55b6731098e..0f235ddcdcc8 100644
--- a/tests/codegen-llvm/asm/riscv-clobbers.rs
+++ b/tests/codegen-llvm/asm/riscv-clobbers.rs
@@ -17,7 +17,7 @@ extern crate minicore;
 use minicore::*;
 
 // CHECK-LABEL: @flags_clobber
-// CHECK: call void asm sideeffect "", "~{vtype},~{vl},~{vxsat},~{vxrm}"()
+// CHECK: call void asm sideeffect "", "~{fflags},~{vtype},~{vl},~{vxsat},~{vxrm}"()
 #[no_mangle]
 pub unsafe fn flags_clobber() {
     asm!("", options(nostack, nomem));
diff --git a/tests/codegen-llvm/async-fn-debug-awaitee-field.rs b/tests/codegen-llvm/async-fn-debug-awaitee-field.rs
index 50860c90662a..b9d3d9ee66e5 100644
--- a/tests/codegen-llvm/async-fn-debug-awaitee-field.rs
+++ b/tests/codegen-llvm/async-fn-debug-awaitee-field.rs
@@ -18,11 +18,11 @@ pub async fn async_fn_test() {
 
 pub async fn foo() {}
 
+// NONMSVC: [[AWAITEE_TYPE:![0-9]*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "{async_fn_env#0}", scope: [[AWAITEE_SCOPE:![0-9]*]],
+// MSVC: [[AWAITEE_TYPE:![0-9]*]] = !DICompositeType(tag: DW_TAG_union_type, name: "enum2$",
+// NONMSVC: [[AWAITEE_SCOPE]] = !DINamespace(name: "foo",
 // NONMSVC: [[GEN:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "{async_fn_env#0}", scope: [[GEN_SCOPE:![0-9]*]],
 // MSVC: [[GEN:!.*]] = !DICompositeType(tag: DW_TAG_union_type, name: "enum2$",
 // NONMSVC: [[GEN_SCOPE:!.*]] = !DINamespace(name: "async_fn_test",
 // CHECK: [[SUSPEND_STRUCT:!.*]] = !DICompositeType(tag: DW_TAG_structure_type, name: "Suspend0", scope: [[GEN]],
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "__awaitee", scope: [[SUSPEND_STRUCT]], {{.*}}, baseType: [[AWAITEE_TYPE:![0-9]*]],
-// NONMSVC: [[AWAITEE_TYPE]] = !DICompositeType(tag: DW_TAG_structure_type, name: "{async_fn_env#0}", scope: [[AWAITEE_SCOPE:![0-9]*]],
-// MSVC: [[AWAITEE_TYPE]] = !DICompositeType(tag: DW_TAG_union_type, name: "enum2$",
-// NONMSVC: [[AWAITEE_SCOPE]] = !DINamespace(name: "foo",
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "__awaitee", scope: [[SUSPEND_STRUCT]], {{.*}}, baseType: [[AWAITEE_TYPE]],
diff --git a/tests/codegen-llvm/atomicptr.rs b/tests/codegen-llvm/atomicptr.rs
index 4819af40ca2d..9d5e618fe76f 100644
--- a/tests/codegen-llvm/atomicptr.rs
+++ b/tests/codegen-llvm/atomicptr.rs
@@ -1,12 +1,11 @@
 // LLVM does not support some atomic RMW operations on pointers, so inside codegen we lower those
-// to integer atomics, surrounded by casts to and from integer type.
+// to integer atomics, followed by an inttoptr cast.
 // This test ensures that we do the round-trip correctly for AtomicPtr::fetch_byte_add, and also
 // ensures that we do not have such a round-trip for AtomicPtr::swap, because LLVM supports pointer
 // arguments to `atomicrmw xchg`.
 
 //@ compile-flags: -Copt-level=3 -Cno-prepopulate-passes
 #![crate_type = "lib"]
-#![feature(strict_provenance_atomic_ptr)]
 
 use std::ptr::without_provenance_mut;
 use std::sync::atomic::AtomicPtr;
@@ -20,8 +19,8 @@ pub fn helper(_: usize) {}
 // CHECK-LABEL: @atomicptr_fetch_byte_add
 #[no_mangle]
 pub fn atomicptr_fetch_byte_add(a: &AtomicPtr, v: usize) -> *mut u8 {
-    // CHECK: %[[INTPTR:.*]] = ptrtoint ptr %{{.*}} to [[USIZE]]
-    // CHECK-NEXT: %[[RET:.*]] = atomicrmw add ptr %{{.*}}, [[USIZE]] %[[INTPTR]]
+    // CHECK: llvm.lifetime.start
+    // CHECK-NEXT: %[[RET:.*]] = atomicrmw add ptr %{{.*}}, [[USIZE]] %v
     // CHECK-NEXT: inttoptr [[USIZE]] %[[RET]] to ptr
     a.fetch_byte_add(v, Relaxed)
 }
diff --git a/tests/codegen-llvm/autodiff/abi_handling.rs b/tests/codegen-llvm/autodiff/abi_handling.rs
new file mode 100644
index 000000000000..5c8126898a8d
--- /dev/null
+++ b/tests/codegen-llvm/autodiff/abi_handling.rs
@@ -0,0 +1,210 @@
+//@ revisions: debug release
+
+//@[debug] compile-flags: -Zautodiff=Enable,NoTT -C opt-level=0 -Clto=fat
+//@[release] compile-flags: -Zautodiff=Enable,NoTT -C opt-level=3 -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+// This test checks that Rust types are lowered to LLVM-IR types in a way
+// we expect and Enzyme can handle. We explicitly check release mode to
+// ensure that LLVM's O3 pipeline doesn't rewrite function signatures
+// into forms that Enzyme can't process correctly.
+
+#![feature(autodiff)]
+
+use std::autodiff::{autodiff_forward, autodiff_reverse};
+
+#[derive(Copy, Clone)]
+struct Input {
+    x: f32,
+    y: f32,
+}
+
+#[derive(Copy, Clone)]
+struct Wrapper {
+    z: f32,
+}
+
+#[derive(Copy, Clone)]
+struct NestedInput {
+    x: f32,
+    y: Wrapper,
+}
+
+fn square(x: f32) -> f32 {
+    x * x
+}
+
+// CHECK-LABEL: ; abi_handling::df1
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (ptr align 4 %x, ptr align 4 %bx_0)
+// release-NEXT: define internal fastcc float
+// release-SAME: (float %x.0.val, float %x.4.val)
+
+// CHECK-LABEL: ; abi_handling::f1
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (ptr align 4 %x)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float %x.0.val, float %x.4.val)
+#[autodiff_forward(df1, Dual, Dual)]
+#[inline(never)]
+fn f1(x: &[f32; 2]) -> f32 {
+    x[0] + x[1]
+}
+
+// CHECK-LABEL: ; abi_handling::df2
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (ptr %f, float %x, float %dret)
+// release-NEXT: define internal fastcc float
+// release-SAME: (float noundef %x)
+
+// CHECK-LABEL: ; abi_handling::f2
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (ptr %f, float %x)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float noundef %x)
+#[autodiff_reverse(df2, Const, Active, Active)]
+#[inline(never)]
+fn f2(f: fn(f32) -> f32, x: f32) -> f32 {
+    f(x)
+}
+
+// CHECK-LABEL: ; abi_handling::df3
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (ptr align 4 %x, ptr align 4 %bx_0, ptr align 4 %y, ptr align 4 %by_0)
+// release-NEXT: define internal fastcc { float, float }
+// release-SAME: (float %x.0.val)
+
+// CHECK-LABEL: ; abi_handling::f3
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (ptr align 4 %x, ptr align 4 %y)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float %x.0.val)
+#[autodiff_forward(df3, Dual, Dual, Dual)]
+#[inline(never)]
+fn f3<'a>(x: &'a f32, y: &'a f32) -> f32 {
+    *x * *y
+}
+
+// CHECK-LABEL: ; abi_handling::df4
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (float %x.0, float %x.1, float %bx_0.0, float %bx_0.1)
+// release-NEXT: define internal fastcc { float, float }
+// release-SAME: (float noundef %x.0, float noundef %x.1)
+
+// CHECK-LABEL: ; abi_handling::f4
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (float %x.0, float %x.1)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float noundef %x.0, float noundef %x.1)
+#[autodiff_forward(df4, Dual, Dual)]
+#[inline(never)]
+fn f4(x: (f32, f32)) -> f32 {
+    x.0 * x.1
+}
+
+// CHECK-LABEL: ; abi_handling::df5
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (float %i.0, float %i.1, float %bi_0.0, float %bi_0.1)
+// release-NEXT: define internal fastcc { float, float }
+// release-SAME: (float noundef %i.0, float noundef %i.1)
+
+// CHECK-LABEL: ; abi_handling::f5
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (float %i.0, float %i.1)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float noundef %i.0, float noundef %i.1)
+#[autodiff_forward(df5, Dual, Dual)]
+#[inline(never)]
+fn f5(i: Input) -> f32 {
+    i.x + i.y
+}
+
+// CHECK-LABEL: ; abi_handling::df6
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (float %i.0, float %i.1, float %bi_0.0, float %bi_0.1)
+// release-NEXT: define internal fastcc { float, float }
+// release-SAME: float noundef %i.0, float noundef %i.1
+// release-SAME: float noundef %bi_0.0, float noundef %bi_0.1
+
+// CHECK-LABEL: ; abi_handling::f6
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (float %i.0, float %i.1)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float noundef %i.0, float noundef %i.1)
+#[autodiff_forward(df6, Dual, Dual)]
+#[inline(never)]
+fn f6(i: NestedInput) -> f32 {
+    i.x + i.y.z * i.y.z
+}
+
+// CHECK-LABEL: ; abi_handling::df7
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal { float, float }
+// debug-SAME: (ptr align 4 %x.0, ptr align 4 %x.1, ptr align 4 %bx_0.0, ptr align 4 %bx_0.1)
+// release-NEXT: define internal fastcc { float, float }
+// release-SAME: (float %x.0.0.val, float %x.1.0.val)
+
+// CHECK-LABEL: ; abi_handling::f7
+// CHECK-NEXT: Function Attrs
+// debug-NEXT: define internal float
+// debug-SAME: (ptr align 4 %x.0, ptr align 4 %x.1)
+// release-NEXT: define internal fastcc noundef float
+// release-SAME: (float %x.0.0.val, float %x.1.0.val)
+#[autodiff_forward(df7, Dual, Dual)]
+#[inline(never)]
+fn f7(x: (&f32, &f32)) -> f32 {
+    x.0 * x.1
+}
+
+fn main() {
+    let x = std::hint::black_box(2.0);
+    let y = std::hint::black_box(3.0);
+    let z = std::hint::black_box(4.0);
+    static Y: f32 = std::hint::black_box(3.2);
+
+    let in_f1 = [x, y];
+    dbg!(f1(&in_f1));
+    let res_f1 = df1(&in_f1, &[1.0, 0.0]);
+    dbg!(res_f1);
+
+    dbg!(f2(square, x));
+    let res_f2 = df2(square, x, 1.0);
+    dbg!(res_f2);
+
+    dbg!(f3(&x, &Y));
+    let res_f3 = df3(&x, &Y, &1.0, &0.0);
+    dbg!(res_f3);
+
+    let in_f4 = (x, y);
+    dbg!(f4(in_f4));
+    let res_f4 = df4(in_f4, (1.0, 0.0));
+    dbg!(res_f4);
+
+    let in_f5 = Input { x, y };
+    dbg!(f5(in_f5));
+    let res_f5 = df5(in_f5, Input { x: 1.0, y: 0.0 });
+    dbg!(res_f5);
+
+    let in_f6 = NestedInput { x, y: Wrapper { z: y } };
+    dbg!(f6(in_f6));
+    let res_f6 = df6(in_f6, NestedInput { x, y: Wrapper { z } });
+    dbg!(res_f6);
+
+    let in_f7 = (&x, &y);
+    dbg!(f7(in_f7));
+    let res_f7 = df7(in_f7, (&1.0, &0.0));
+    dbg!(res_f7);
+}
diff --git a/tests/codegen-llvm/autodiffv2.rs b/tests/codegen-llvm/autodiff/autodiffv2.rs
similarity index 79%
rename from tests/codegen-llvm/autodiffv2.rs
rename to tests/codegen-llvm/autodiff/autodiffv2.rs
index a40d19d3be3a..c24a374148c3 100644
--- a/tests/codegen-llvm/autodiffv2.rs
+++ b/tests/codegen-llvm/autodiff/autodiffv2.rs
@@ -18,20 +18,16 @@
 // but each shadow argument is `width` times larger (thus 16 and 20 elements here).
 // `d_square3` instead takes `width` (4) shadow arguments, which are all the same size as the
 // original function arguments.
-//
-// FIXME(autodiff): We currently can't test `d_square1` and `d_square3` in the same file, since they
-// generate the same dummy functions which get merged by LLVM, breaking pieces of our pipeline which
-// try to rewrite the dummy functions later. We should consider to change to pure declarations both
-// in our frontend and in the llvm backend to avoid these issues.
 
 #![feature(autodiff)]
 
-use std::autodiff::autodiff;
+use std::autodiff::autodiff_forward;
 
+// CHECK: ;
 #[no_mangle]
-//#[autodiff(d_square1, Forward, Dual, Dual)]
-#[autodiff(d_square2, Forward, 4, Dualv, Dualv)]
-#[autodiff(d_square3, Forward, 4, Dual, Dual)]
+#[autodiff_forward(d_square1, Dual, Dual)]
+#[autodiff_forward(d_square2, 4, Dualv, Dualv)]
+#[autodiff_forward(d_square3, 4, Dual, Dual)]
 fn square(x: &[f32], y: &mut [f32]) {
     assert!(x.len() >= 4);
     assert!(y.len() >= 5);
@@ -78,25 +74,25 @@ fn main() {
     let mut dy3_4 = std::hint::black_box(vec![0.0; 5]);
 
     // scalar.
-    //d_square1(&x1, &z1, &mut y1, &mut dy1_1);
-    //d_square1(&x1, &z2, &mut y2, &mut dy1_2);
-    //d_square1(&x1, &z3, &mut y3, &mut dy1_3);
-    //d_square1(&x1, &z4, &mut y4, &mut dy1_4);
+    d_square1(&x1, &z1, &mut y1, &mut dy1_1);
+    d_square1(&x1, &z2, &mut y2, &mut dy1_2);
+    d_square1(&x1, &z3, &mut y3, &mut dy1_3);
+    d_square1(&x1, &z4, &mut y4, &mut dy1_4);
 
     // assert y1 == y2 == y3 == y4
-    //for i in 0..5 {
-    //    assert_eq!(y1[i], y2[i]);
-    //    assert_eq!(y1[i], y3[i]);
-    //    assert_eq!(y1[i], y4[i]);
-    //}
+    for i in 0..5 {
+        assert_eq!(y1[i], y2[i]);
+        assert_eq!(y1[i], y3[i]);
+        assert_eq!(y1[i], y4[i]);
+    }
 
     // batch mode A)
     d_square2(&x1, &z5, &mut y5, &mut dy2);
 
     // assert y1 == y2 == y3 == y4 == y5
-    //for i in 0..5 {
-    //    assert_eq!(y1[i], y5[i]);
-    //}
+    for i in 0..5 {
+        assert_eq!(y1[i], y5[i]);
+    }
 
     // batch mode B)
     d_square3(&x1, &z1, &z2, &z3, &z4, &mut y6, &mut dy3_1, &mut dy3_2, &mut dy3_3, &mut dy3_4);
diff --git a/tests/codegen-llvm/autodiff/batched.rs b/tests/codegen-llvm/autodiff/batched.rs
index d27aed50e6cc..0ff6134bc07d 100644
--- a/tests/codegen-llvm/autodiff/batched.rs
+++ b/tests/codegen-llvm/autodiff/batched.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ compile-flags: -Zautodiff=Enable,NoTT,NoPostopt -C opt-level=3  -Clto=fat
 //@ no-prefer-dynamic
 //@ needs-enzyme
 //
@@ -17,12 +17,13 @@ use std::autodiff::autodiff_forward;
 #[autodiff_forward(d_square2, 4, Dual, DualOnly)]
 #[autodiff_forward(d_square1, 4, Dual, Dual)]
 #[no_mangle]
+#[inline(never)]
 fn square(x: &f32) -> f32 {
     x * x
 }
 
-// d_sqaure2
-// CHECK: define internal fastcc [4 x float] @fwddiffe4square(float %x.0.val, [4 x ptr] %"x'")
+// d_square2
+// CHECK: define internal [4 x float] @fwddiffe4square(ptr noalias noundef readonly align 4 captures(none) dereferenceable(4) %x, [4 x ptr] %"x'")
 // CHECK-NEXT: start:
 // CHECK-NEXT:   %0 = extractvalue [4 x ptr] %"x'", 0
 // CHECK-NEXT:   %"_2'ipl" = load float, ptr %0, align 4
@@ -32,27 +33,28 @@ fn square(x: &f32) -> f32 {
 // CHECK-NEXT:   %"_2'ipl2" = load float, ptr %2, align 4
 // CHECK-NEXT:   %3 = extractvalue [4 x ptr] %"x'", 3
 // CHECK-NEXT:   %"_2'ipl3" = load float, ptr %3, align 4
-// CHECK-NEXT:   %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
-// CHECK-NEXT:   %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
-// CHECK-NEXT:   %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
-// CHECK-NEXT:   %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
-// CHECK-NEXT:   %8 = fadd fast <4 x float> %7, %7
-// CHECK-NEXT:   %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
-// CHECK-NEXT:   %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
-// CHECK-NEXT:   %11 = fmul fast <4 x float> %8, %10
-// CHECK-NEXT:   %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT:   %_2 = load float, ptr %x, align 4
+// CHECK-NEXT:   %4 = fmul fast float %"_2'ipl", %_2
+// CHECK-NEXT:   %5 = fmul fast float %"_2'ipl1", %_2
+// CHECK-NEXT:   %6 = fmul fast float %"_2'ipl2", %_2
+// CHECK-NEXT:   %7 = fmul fast float %"_2'ipl3", %_2
+// CHECK-NEXT:   %8 = fmul fast float %"_2'ipl", %_2
+// CHECK-NEXT:   %9 = fmul fast float %"_2'ipl1", %_2
+// CHECK-NEXT:   %10 = fmul fast float %"_2'ipl2", %_2
+// CHECK-NEXT:   %11 = fmul fast float %"_2'ipl3", %_2
+// CHECK-NEXT:   %12 = fadd fast float %4, %8
 // CHECK-NEXT:   %13 = insertvalue [4 x float] undef, float %12, 0
-// CHECK-NEXT:   %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT:   %14 = fadd fast float %5, %9
 // CHECK-NEXT:   %15 = insertvalue [4 x float] %13, float %14, 1
-// CHECK-NEXT:   %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT:   %16 = fadd fast float %6, %10
 // CHECK-NEXT:   %17 = insertvalue [4 x float] %15, float %16, 2
-// CHECK-NEXT:   %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT:   %18 = fadd fast float %7, %11
 // CHECK-NEXT:   %19 = insertvalue [4 x float] %17, float %18, 3
 // CHECK-NEXT:   ret [4 x float] %19
-// CHECK-NEXT: }
+// CHECK-NEXT:   }
 
 // d_square3, the extra float is the original return value (x * x)
-// CHECK: define internal fastcc { float, [4 x float] } @fwddiffe4square.1(float %x.0.val, [4 x ptr] %"x'")
+// CHECK: define internal { float, [4 x float] } @fwddiffe4square.1(ptr noalias noundef readonly align 4 captures(none) dereferenceable(4) %x, [4 x ptr] %"x'")
 // CHECK-NEXT: start:
 // CHECK-NEXT:   %0 = extractvalue [4 x ptr] %"x'", 0
 // CHECK-NEXT:   %"_2'ipl" = load float, ptr %0, align 4
@@ -62,27 +64,28 @@ fn square(x: &f32) -> f32 {
 // CHECK-NEXT:   %"_2'ipl2" = load float, ptr %2, align 4
 // CHECK-NEXT:   %3 = extractvalue [4 x ptr] %"x'", 3
 // CHECK-NEXT:   %"_2'ipl3" = load float, ptr %3, align 4
-// CHECK-NEXT:   %_0 = fmul float %x.0.val, %x.0.val
-// CHECK-NEXT:   %4 = insertelement <4 x float> poison, float %"_2'ipl", i64 0
-// CHECK-NEXT:   %5 = insertelement <4 x float> %4, float %"_2'ipl1", i64 1
-// CHECK-NEXT:   %6 = insertelement <4 x float> %5, float %"_2'ipl2", i64 2
-// CHECK-NEXT:   %7 = insertelement <4 x float> %6, float %"_2'ipl3", i64 3
-// CHECK-NEXT:   %8 = fadd fast <4 x float> %7, %7
-// CHECK-NEXT:   %9 = insertelement <4 x float> poison, float %x.0.val, i64 0
-// CHECK-NEXT:   %10 = shufflevector <4 x float> %9, <4 x float> poison, <4 x i32> zeroinitializer
-// CHECK-NEXT:   %11 = fmul fast <4 x float> %8, %10
-// CHECK-NEXT:   %12 = extractelement <4 x float> %11, i64 0
+// CHECK-NEXT:   %_2 = load float, ptr %x, align 4
+// CHECK-NEXT:   %_0 = fmul float %_2, %_2
+// CHECK-NEXT:   %4 = fmul fast float %"_2'ipl", %_2
+// CHECK-NEXT:   %5 = fmul fast float %"_2'ipl1", %_2
+// CHECK-NEXT:   %6 = fmul fast float %"_2'ipl2", %_2
+// CHECK-NEXT:   %7 = fmul fast float %"_2'ipl3", %_2
+// CHECK-NEXT:   %8 = fmul fast float %"_2'ipl", %_2
+// CHECK-NEXT:   %9 = fmul fast float %"_2'ipl1", %_2
+// CHECK-NEXT:   %10 = fmul fast float %"_2'ipl2", %_2
+// CHECK-NEXT:   %11 = fmul fast float %"_2'ipl3", %_2
+// CHECK-NEXT:   %12 = fadd fast float %4, %8
 // CHECK-NEXT:   %13 = insertvalue [4 x float] undef, float %12, 0
-// CHECK-NEXT:   %14 = extractelement <4 x float> %11, i64 1
+// CHECK-NEXT:   %14 = fadd fast float %5, %9
 // CHECK-NEXT:   %15 = insertvalue [4 x float] %13, float %14, 1
-// CHECK-NEXT:   %16 = extractelement <4 x float> %11, i64 2
+// CHECK-NEXT:   %16 = fadd fast float %6, %10
 // CHECK-NEXT:   %17 = insertvalue [4 x float] %15, float %16, 2
-// CHECK-NEXT:   %18 = extractelement <4 x float> %11, i64 3
+// CHECK-NEXT:   %18 = fadd fast float %7, %11
 // CHECK-NEXT:   %19 = insertvalue [4 x float] %17, float %18, 3
 // CHECK-NEXT:   %20 = insertvalue { float, [4 x float] } undef, float %_0, 0
 // CHECK-NEXT:   %21 = insertvalue { float, [4 x float] } %20, [4 x float] %19, 1
 // CHECK-NEXT:   ret { float, [4 x float] } %21
-// CHECK-NEXT: }
+// CHECK-NEXT:   }
 
 fn main() {
     let x = std::hint::black_box(3.0);
diff --git a/tests/codegen-llvm/autodiff/generic.rs b/tests/codegen-llvm/autodiff/generic.rs
index 2f674079be02..6f56460a2b6d 100644
--- a/tests/codegen-llvm/autodiff/generic.rs
+++ b/tests/codegen-llvm/autodiff/generic.rs
@@ -6,19 +6,11 @@
 use std::autodiff::autodiff_reverse;
 
 #[autodiff_reverse(d_square, Duplicated, Active)]
+#[inline(never)]
 fn square + Copy>(x: &T) -> T {
     *x * *x
 }
 
-// Ensure that `d_square::` code is generated even if `square::` was never called
-//
-// CHECK: ; generic::square
-// CHECK-NEXT: ; Function Attrs:
-// CHECK-NEXT: define internal {{.*}} double
-// CHECK-NEXT: start:
-// CHECK-NOT: ret
-// CHECK: fmul double
-
 // Ensure that `d_square::` code is generated
 //
 // CHECK: ; generic::square
@@ -28,6 +20,15 @@ fn square + Copy>(x: &T) -> T {
 // CHECK-NOT: ret
 // CHECK: fmul float
 
+// Ensure that `d_square::` code is generated even if `square::` was never called
+//
+// CHECK: ; generic::square
+// CHECK-NEXT: ; Function Attrs:
+// CHECK-NEXT: define internal {{.*}} double
+// CHECK-NEXT: start:
+// CHECK-NOT: ret
+// CHECK: fmul double
+
 fn main() {
     let xf32: f32 = std::hint::black_box(3.0);
     let xf64: f64 = std::hint::black_box(3.0);
diff --git a/tests/codegen-llvm/autodiff/identical_fnc.rs b/tests/codegen-llvm/autodiff/identical_fnc.rs
index 1c25b3d09ab0..1c18e7acc4b6 100644
--- a/tests/codegen-llvm/autodiff/identical_fnc.rs
+++ b/tests/codegen-llvm/autodiff/identical_fnc.rs
@@ -14,25 +14,27 @@
 use std::autodiff::autodiff_reverse;
 
 #[autodiff_reverse(d_square, Duplicated, Active)]
+#[inline(never)]
 fn square(x: &f64) -> f64 {
     x * x
 }
 
 #[autodiff_reverse(d_square2, Duplicated, Active)]
+#[inline(never)]
 fn square2(x: &f64) -> f64 {
     x * x
 }
 
 // CHECK:; identical_fnc::main
 // CHECK-NEXT:; Function Attrs:
-// CHECK-NEXT:define internal void @_ZN13identical_fnc4main17hf4dbc69c8d2f9130E()
+// CHECK-NEXT:define internal void @_ZN13identical_fnc4main17h6009e4f751bf9407E()
 // CHECK-NEXT:start:
 // CHECK-NOT:br
 // CHECK-NOT:ret
 // CHECK:; call identical_fnc::d_square
-// CHECK-NEXT:  call fastcc void @_ZN13identical_fnc8d_square17h4c364207a2f8e06dE(double %x.val, ptr noalias noundef nonnull align 8 dereferenceable(8) %dx1)
-// CHECK-NEXT:; call identical_fnc::d_square
-// CHECK-NEXT:  call fastcc void @_ZN13identical_fnc8d_square17h4c364207a2f8e06dE(double %x.val, ptr noalias noundef nonnull align 8 dereferenceable(8) %dx2)
+// CHECK-NEXT:call fastcc void @_ZN13identical_fnc8d_square[[HASH:.+]](double %x.val, ptr noalias noundef align 8 dereferenceable(8) %dx1)
+// CHECK:; call identical_fnc::d_square
+// CHECK-NEXT:call fastcc void @_ZN13identical_fnc8d_square[[HASH]](double %x.val, ptr noalias noundef align 8 dereferenceable(8) %dx2)
 
 fn main() {
     let x = std::hint::black_box(3.0);
diff --git a/tests/codegen-llvm/autodiff/inline.rs b/tests/codegen-llvm/autodiff/inline.rs
deleted file mode 100644
index 65bed170207c..000000000000
--- a/tests/codegen-llvm/autodiff/inline.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat -Zautodiff=NoPostopt
-//@ no-prefer-dynamic
-//@ needs-enzyme
-
-#![feature(autodiff)]
-
-use std::autodiff::autodiff_reverse;
-
-#[autodiff_reverse(d_square, Duplicated, Active)]
-fn square(x: &f64) -> f64 {
-    x * x
-}
-
-// CHECK: ; inline::d_square
-// CHECK-NEXT: ; Function Attrs: alwaysinline
-// CHECK-NOT: noinline
-// CHECK-NEXT: define internal fastcc void @_ZN6inline8d_square17h021c74e92c259cdeE
-fn main() {
-    let x = std::hint::black_box(3.0);
-    let mut dx1 = std::hint::black_box(1.0);
-    let _ = d_square(&x, &mut dx1, 1.0);
-    assert_eq!(dx1, 6.0);
-}
diff --git a/tests/codegen-llvm/autodiff/scalar.rs b/tests/codegen-llvm/autodiff/scalar.rs
index 096b4209e84a..53672a89230a 100644
--- a/tests/codegen-llvm/autodiff/scalar.rs
+++ b/tests/codegen-llvm/autodiff/scalar.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ compile-flags: -Zautodiff=Enable,NoTT -C opt-level=3  -Clto=fat
 //@ no-prefer-dynamic
 //@ needs-enzyme
 #![feature(autodiff)]
@@ -7,11 +7,12 @@ use std::autodiff::autodiff_reverse;
 
 #[autodiff_reverse(d_square, Duplicated, Active)]
 #[no_mangle]
+#[inline(never)]
 fn square(x: &f64) -> f64 {
     x * x
 }
 
-// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nocapture nonnull align 8 %"x'"
+// CHECK:define internal fastcc double @diffesquare(double %x.0.val, ptr nonnull align 8 captures(none) %"x'")
 // CHECK-NEXT:invertstart:
 // CHECK-NEXT:  %_0 = fmul double %x.0.val, %x.0.val
 // CHECK-NEXT:  %0 = fadd fast double %x.0.val, %x.0.val
diff --git a/tests/codegen-llvm/autodiff/sret.rs b/tests/codegen-llvm/autodiff/sret.rs
index d2fa85e3e378..498cd3fea012 100644
--- a/tests/codegen-llvm/autodiff/sret.rs
+++ b/tests/codegen-llvm/autodiff/sret.rs
@@ -1,4 +1,4 @@
-//@ compile-flags: -Zautodiff=Enable -C opt-level=3  -Clto=fat
+//@ compile-flags: -Zautodiff=Enable,NoTT -C opt-level=3  -Clto=fat
 //@ no-prefer-dynamic
 //@ needs-enzyme
 
@@ -13,30 +13,30 @@ use std::autodiff::autodiff_reverse;
 
 #[no_mangle]
 #[autodiff_reverse(df, Active, Active, Active)]
+#[inline(never)]
 fn primal(x: f32, y: f32) -> f64 {
     (x * x * y) as f64
 }
 
-// CHECK:define internal fastcc void @_ZN4sret2df17h93be4316dd8ea006E(ptr dead_on_unwind noalias nocapture noundef nonnull writable writeonly align 8 dereferenceable(16) initializes((0, 16)) %_0, float noundef %x, float noundef %y)
-// CHECK-NEXT:start:
-// CHECK-NEXT:  %0 = tail call fastcc { double, float, float } @diffeprimal(float %x, float %y)
-// CHECK-NEXT:  %.elt = extractvalue { double, float, float } %0, 0
-// CHECK-NEXT:  store double %.elt, ptr %_0, align 8
-// CHECK-NEXT:  %_0.repack1 = getelementptr inbounds nuw i8, ptr %_0, i64 8
-// CHECK-NEXT:  %.elt2 = extractvalue { double, float, float } %0, 1
-// CHECK-NEXT:  store float %.elt2, ptr %_0.repack1, align 8
-// CHECK-NEXT:  %_0.repack3 = getelementptr inbounds nuw i8, ptr %_0, i64 12
-// CHECK-NEXT:  %.elt4 = extractvalue { double, float, float } %0, 2
-// CHECK-NEXT:  store float %.elt4, ptr %_0.repack3, align 4
-// CHECK-NEXT:  ret void
-// CHECK-NEXT:}
+// CHECK: define internal fastcc { double, float, float } @diffeprimal(float noundef %x, float noundef %y)
+// CHECK-NEXT: invertstart:
+// CHECK-NEXT: %_4 = fmul float %x, %x
+// CHECK-NEXT: %_3 = fmul float %_4, %y
+// CHECK-NEXT: %_0 = fpext float %_3 to double
+// CHECK-NEXT: %0 = fadd fast float %y, %y
+// CHECK-NEXT: %1 = fmul fast float %0, %x
+// CHECK-NEXT: %2 = insertvalue { double, float, float } undef, double %_0, 0
+// CHECK-NEXT: %3 = insertvalue { double, float, float } %2, float %1, 1
+// CHECK-NEXT: %4 = insertvalue { double, float, float } %3, float %_4, 2
+// CHECK-NEXT: ret { double, float, float } %4
+// CHECK-NEXT: }
 
 fn main() {
     let x = std::hint::black_box(3.0);
     let y = std::hint::black_box(2.5);
     let scalar = std::hint::black_box(1.0);
     let (r1, r2, r3) = df(x, y, scalar);
-    // 3*3*1.5 = 22.5
+    // 3*3*2.5 = 22.5
     assert_eq!(r1, 22.5);
     // 2*x*y = 2*3*2.5 = 15.0
     assert_eq!(r2, 15.0);
diff --git a/tests/codegen-llvm/autodiff/trait.rs b/tests/codegen-llvm/autodiff/trait.rs
new file mode 100644
index 000000000000..701f3a9e843b
--- /dev/null
+++ b/tests/codegen-llvm/autodiff/trait.rs
@@ -0,0 +1,30 @@
+//@ compile-flags: -Zautodiff=Enable -Zautodiff=NoPostopt -C opt-level=3 -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+// Just check it does not crash for now
+// CHECK: ;
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+struct Foo {
+    a: f64,
+}
+
+trait MyTrait {
+    fn f(&self, x: f64) -> f64;
+    fn df(&self, x: f64, seed: f64) -> (f64, f64);
+}
+
+impl MyTrait for Foo {
+    #[autodiff_reverse(df, Const, Active, Active)]
+    fn f(&self, x: f64) -> f64 {
+        self.a * 0.25 * (x * x - 1.0 - 2.0 * x.ln())
+    }
+}
+
+fn main() {
+    let foo = Foo { a: 3.0f64 };
+    dbg!(foo.df(1.0, 1.0));
+}
diff --git a/tests/codegen-llvm/autodiff/typetree.rs b/tests/codegen-llvm/autodiff/typetree.rs
new file mode 100644
index 000000000000..1cb0c2fb68be
--- /dev/null
+++ b/tests/codegen-llvm/autodiff/typetree.rs
@@ -0,0 +1,33 @@
+//@ compile-flags: -Zautodiff=Enable -C opt-level=3 -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+// Test that basic autodiff still works with our TypeTree infrastructure
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_simple, Duplicated, Active)]
+#[no_mangle]
+#[inline(never)]
+fn simple(x: &f64) -> f64 {
+    2.0 * x
+}
+
+// CHECK-LABEL: @simple
+// CHECK: fmul double
+
+// The derivative function should be generated normally
+// CHECK-LABEL: diffesimple
+// CHECK: fadd fast double
+
+fn main() {
+    let x = std::hint::black_box(3.0);
+    let output = simple(&x);
+    assert_eq!(6.0, output);
+
+    let mut df_dx = 0.0;
+    let output_ = d_simple(&x, &mut df_dx, 1.0);
+    assert_eq!(output, output_);
+    assert_eq!(2.0, df_dx);
+}
diff --git a/tests/codegen-llvm/autodiff/void_ret.rs b/tests/codegen-llvm/autodiff/void_ret.rs
new file mode 100644
index 000000000000..98c6b98eef4e
--- /dev/null
+++ b/tests/codegen-llvm/autodiff/void_ret.rs
@@ -0,0 +1,41 @@
+//@ compile-flags: -Zautodiff=Enable,NoTT,NoPostopt -C no-prepopulate-passes -C opt-level=3  -Clto=fat
+//@ no-prefer-dynamic
+//@ needs-enzyme
+
+#![feature(autodiff)]
+use std::autodiff::*;
+
+// Usually we would store the return value of the differentiated function.
+// However, if the return type is void or an empty struct,
+// we don't need to store anything. Verify this, since it caused a bug.
+
+// CHECK:; void_ret::main
+// CHECK-NEXT:    ; Function Attrs:
+// CHECK-NEXT:    define internal
+// CHECK-NOT: store {} undef, ptr undef
+// CHECK: ret void
+
+#[autodiff_reverse(bar, Duplicated, Duplicated)]
+pub fn foo(r: &[f64; 10], res: &mut f64) {
+    let mut output = [0.0; 10];
+    output[0] = r[0];
+    output[1] = r[1] * r[2];
+    output[2] = r[4] * r[5];
+    output[3] = r[2] * r[6];
+    output[4] = r[1] * r[7];
+    output[5] = r[2] * r[8];
+    output[6] = r[1] * r[9];
+    output[7] = r[5] * r[6];
+    output[8] = r[5] * r[7];
+    output[9] = r[4] * r[8];
+    *res = output.iter().sum();
+}
+fn main() {
+    let inputs = Box::new([3.1; 10]);
+    let mut d_inputs = Box::new([0.0; 10]);
+    let mut res = Box::new(0.0);
+    let mut d_res = Box::new(1.0);
+
+    bar(&inputs, &mut d_inputs, &mut res, &mut d_res);
+    dbg!(&d_inputs);
+}
diff --git a/tests/codegen-llvm/auxiliary/darwin_objc_aux.rs b/tests/codegen-llvm/auxiliary/darwin_objc_aux.rs
new file mode 100644
index 000000000000..3c35d003c8af
--- /dev/null
+++ b/tests/codegen-llvm/auxiliary/darwin_objc_aux.rs
@@ -0,0 +1,27 @@
+#![crate_type = "lib"]
+#![feature(darwin_objc)]
+
+use std::os::darwin::objc;
+
+#[link(name = "Foundation", kind = "framework")]
+unsafe extern "C" {}
+
+#[inline(always)]
+pub fn inline_get_object_class() -> objc::Class {
+    objc::class!("NSObject")
+}
+
+#[inline(always)]
+pub fn inline_get_alloc_selector() -> objc::SEL {
+    objc::selector!("alloc")
+}
+
+#[inline(never)]
+pub fn never_inline_get_string_class() -> objc::Class {
+    objc::class!("NSString")
+}
+
+#[inline(never)]
+pub fn never_inline_get_init_selector() -> objc::SEL {
+    objc::selector!("init")
+}
diff --git a/tests/codegen-llvm/become-musttail.rs b/tests/codegen-llvm/become-musttail.rs
new file mode 100644
index 000000000000..07f335719104
--- /dev/null
+++ b/tests/codegen-llvm/become-musttail.rs
@@ -0,0 +1,18 @@
+//@ compile-flags: -C opt-level=0 -Cpanic=abort -C no-prepopulate-passes
+//@ needs-unwind
+
+#![crate_type = "lib"]
+#![feature(explicit_tail_calls)]
+
+// CHECK-LABEL: define {{.*}}@fibonacci(
+#[no_mangle]
+#[inline(never)]
+pub fn fibonacci(n: u64, a: u64, b: u64) -> u64 {
+    // CHECK: musttail call {{.*}}@fibonacci(
+    // CHECK-NEXT: ret i64
+    match n {
+        0 => a,
+        1 => b,
+        _ => become fibonacci(n - 1, b, a + b),
+    }
+}
diff --git a/tests/codegen-llvm/bigint-helpers.rs b/tests/codegen-llvm/bigint-helpers.rs
index 355cccb81504..ec70a3eabedb 100644
--- a/tests/codegen-llvm/bigint-helpers.rs
+++ b/tests/codegen-llvm/bigint-helpers.rs
@@ -3,11 +3,20 @@
 #![crate_type = "lib"]
 #![feature(bigint_helper_methods)]
 
+// Note that there's also an assembly test for this, which is what checks for
+// the `ADC` (Add with Carry) instruction on x86 now that the IR we emit uses
+// the preferred instruction phrasing instead of the intrinsic.
+
 // CHECK-LABEL: @u32_carrying_add
 #[no_mangle]
 pub fn u32_carrying_add(a: u32, b: u32, c: bool) -> (u32, bool) {
-    // CHECK: @llvm.uadd.with.overflow.i32
-    // CHECK: @llvm.uadd.with.overflow.i32
-    // CHECK: or disjoint i1
+    // CHECK: %[[AB:.+]] = add i32 {{%a, %b|%b, %a}}
+    // CHECK: %[[O1:.+]] = icmp ult i32 %[[AB]], %a
+    // CHECK: %[[CEXT:.+]] = zext i1 %c to i32
+    // CHECK: %[[ABC:.+]] = add i32 %[[AB]], %[[CEXT]]
+    // CHECK: %[[O2:.+]] = icmp ult i32 %[[ABC]], %[[AB]]
+    // CHECK: %[[O:.+]] = or disjoint i1 %[[O1]], %[[O2]]
+    // CHECK: insertvalue {{.+}}, i32 %[[ABC]], 0
+    // CHECK: insertvalue {{.+}}, i1 %[[O]], 1
     u32::carrying_add(a, b, c)
 }
diff --git a/tests/codegen-llvm/binary-search-index-no-bound-check.rs b/tests/codegen-llvm/binary-search-index-no-bound-check.rs
index d59c0beec641..8322c4179bd1 100644
--- a/tests/codegen-llvm/binary-search-index-no-bound-check.rs
+++ b/tests/codegen-llvm/binary-search-index-no-bound-check.rs
@@ -8,8 +8,7 @@
 #[no_mangle]
 pub fn binary_search_index_no_bounds_check(s: &[u8]) -> u8 {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     if let Ok(idx) = s.binary_search(&b'\\') { s[idx] } else { 42 }
 }
diff --git a/tests/codegen-llvm/bounds-check-elision-slice-min.rs b/tests/codegen-llvm/bounds-check-elision-slice-min.rs
new file mode 100644
index 000000000000..e160e5da50f9
--- /dev/null
+++ b/tests/codegen-llvm/bounds-check-elision-slice-min.rs
@@ -0,0 +1,19 @@
+//! Regression test for #:
+//! Multiple bounds checking elision failures
+//! (ensures bounds checks are properly elided,
+//! with no calls to panic_bounds_check in the LLVM IR).
+
+//@ compile-flags: -C opt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @foo
+// CHECK-NOT: panic_bounds_check
+#[no_mangle]
+pub fn foo(buf: &[u8], alloced_size: usize) -> &[u8] {
+    if alloced_size.checked_add(1).map(|total| buf.len() < total).unwrap_or(true) {
+        return &[];
+    }
+    let size = buf[0];
+    &buf[1..1 + usize::min(alloced_size, usize::from(size))]
+}
diff --git a/tests/codegen-llvm/bpf-abi-indirect-return.rs b/tests/codegen-llvm/bpf-abi-indirect-return.rs
new file mode 100644
index 000000000000..b5787cc83a2b
--- /dev/null
+++ b/tests/codegen-llvm/bpf-abi-indirect-return.rs
@@ -0,0 +1,43 @@
+// Checks that results larger than one register are returned indirectly
+//@ only-bpf
+//@ needs-llvm-components: bpf
+//@ compile-flags: --target bpfel-unknown-none
+
+#![no_std]
+#![no_main]
+
+#[no_mangle]
+fn outer(a: u64) -> u64 {
+    let v = match inner_res(a) {
+        Ok(v) => v,
+        Err(()) => 0,
+    };
+
+    inner_big(v).a[0] as u64
+}
+
+// CHECK-LABEL: define {{.*}} @_ZN{{.*}}inner_res{{.*}}E(
+// CHECK-SAME:   ptr{{[^,]*}},
+// CHECK-SAME:   i64{{[^)]*}}
+#[inline(never)]
+fn inner_res(a: u64) -> Result {
+    if a == 0 { Err(()) } else { Ok(a + 1) }
+}
+
+struct Big {
+    a: [u16; 32],
+    b: u64,
+}
+
+// CHECK-LABEL: define {{.*}} @_ZN{{.*}}inner_big{{.*}}E(
+// CHECK-SAME:   ptr{{[^,]*}},
+// CHECK-SAME:   i64{{[^)]*}}
+#[inline(never)]
+fn inner_big(a: u64) -> Big {
+    Big { a: [a as u16; 32], b: 42 }
+}
+
+#[panic_handler]
+fn panic(_info: &core::panic::PanicInfo) -> ! {
+    loop {}
+}
diff --git a/tests/codegen-llvm/branch-protection.rs b/tests/codegen-llvm/branch-protection.rs
index d67e494cc0d6..f92259c941ce 100644
--- a/tests/codegen-llvm/branch-protection.rs
+++ b/tests/codegen-llvm/branch-protection.rs
@@ -1,9 +1,10 @@
 // Test that the correct module flags are emitted with different branch protection flags.
 
 //@ add-core-stubs
-//@ revisions: BTI PACRET LEAF BKEY PAUTHLR PAUTHLR_BKEY PAUTHLR_LEAF PAUTHLR_BTI NONE
+//@ revisions: BTI GCS PACRET LEAF BKEY PAUTHLR PAUTHLR_BKEY PAUTHLR_LEAF PAUTHLR_BTI NONE
 //@ needs-llvm-components: aarch64
 //@ [BTI] compile-flags: -Z branch-protection=bti
+//@ [GCS] compile-flags: -Z branch-protection=gcs
 //@ [PACRET] compile-flags: -Z branch-protection=pac-ret
 //@ [LEAF] compile-flags: -Z branch-protection=pac-ret,leaf
 //@ [BKEY] compile-flags: -Z branch-protection=pac-ret,b-key
@@ -32,6 +33,9 @@ pub fn test() {}
 // BTI: !"sign-return-address-all", i32 0
 // BTI: !"sign-return-address-with-bkey", i32 0
 
+// GCS: attributes [[ATTR]] = {{.*}} "guarded-control-stack"
+// GCS: !"guarded-control-stack", i32 1
+
 // PACRET: attributes [[ATTR]] = {{.*}} "sign-return-address"="non-leaf"
 // PACRET-SAME: "sign-return-address-key"="a_key"
 // PACRET: !"branch-target-enforcement", i32 0
diff --git a/tests/codegen-llvm/c-variadic-lifetime.rs b/tests/codegen-llvm/c-variadic-lifetime.rs
new file mode 100644
index 000000000000..c6d3602ef51a
--- /dev/null
+++ b/tests/codegen-llvm/c-variadic-lifetime.rs
@@ -0,0 +1,21 @@
+//@ add-core-stubs
+//@ compile-flags: -Copt-level=3
+#![feature(c_variadic)]
+#![crate_type = "lib"]
+
+// Check that `%args` explicitly has its lifetime start and end. Being explicit can improve
+// instruction and register selection, see e.g. https://github.com/rust-lang/rust/pull/144549
+
+#[unsafe(no_mangle)]
+unsafe extern "C" fn variadic(a: f64, mut args: ...) -> f64 {
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 [0-9]+, )?}}ptr nonnull %args)
+    // CHECK: call void @llvm.va_start.p0(ptr nonnull %args)
+
+    let b = args.arg::();
+    let c = args.arg::();
+
+    a + b + c
+
+    // CHECK: call void @llvm.va_end.p0(ptr nonnull %args)
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 [0-9]+, )?}}ptr nonnull %args)
+}
diff --git a/tests/codegen-llvm/call-tmps-lifetime.rs b/tests/codegen-llvm/call-tmps-lifetime.rs
index 7b7b6e17bddd..0d7657ed758b 100644
--- a/tests/codegen-llvm/call-tmps-lifetime.rs
+++ b/tests/codegen-llvm/call-tmps-lifetime.rs
@@ -16,14 +16,14 @@ use minicore::*;
 // CHECK-NEXT: start:
 // CHECK-NEXT: [[B:%.*]] = alloca
 // CHECK-NEXT: [[A:%.*]] = alloca
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[A]])
 // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 4 {{.*}}, i32 4096, i1 false)
 // CHECK-NEXT: call void %h(ptr {{.*}} [[A]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[A]])
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[B]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 4096, )?}}ptr [[B]])
 // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[B]], ptr align 4 {{.*}}, i32 4096, i1 false)
 // CHECK-NEXT: call void %h(ptr {{.*}} [[B]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[B]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 4096, )?}}ptr [[B]])
 #[no_mangle]
 pub fn const_indirect(h: extern "C" fn([u32; 1024])) {
     const C: [u32; 1024] = [0; 1024];
@@ -42,12 +42,12 @@ pub struct Str {
 // CHECK-LABEL: define void @immediate_indirect(ptr {{.*}}%s.0, i32 {{.*}}%s.1, ptr {{.*}}%g)
 // CHECK-NEXT: start:
 // CHECK-NEXT: [[A:%.*]] = alloca
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr [[A]])
 // CHECK-NEXT: store ptr %s.0, ptr [[A]]
 // CHECK-NEXT: [[B:%.]] = getelementptr inbounds i8, ptr [[A]], i32 4
 // CHECK-NEXT: store i32 %s.1, ptr [[B]]
 // CHECK-NEXT: call void %g(ptr {{.*}} [[A]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr [[A]])
 #[no_mangle]
 pub fn immediate_indirect(s: Str, g: extern "C" fn(Str)) {
     g(s);
@@ -58,10 +58,10 @@ pub fn immediate_indirect(s: Str, g: extern "C" fn(Str)) {
 // CHECK-LABEL: define void @align_indirect(ptr{{.*}} align 1{{.*}} %a, ptr{{.*}} %fun)
 // CHECK-NEXT: start:
 // CHECK-NEXT: [[A:%.*]] = alloca [1024 x i8], align 4
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 1024, ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 1024, )?}}ptr [[A]])
 // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[A]], ptr align 1 %a, i32 1024, i1 false)
 // CHECK-NEXT: call void %fun(ptr {{.*}} [[A]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1024, ptr [[A]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0({{(i64 1024, )?}}ptr [[A]])
 #[no_mangle]
 pub fn align_indirect(a: [u8; 1024], fun: extern "C" fn([u8; 1024])) {
     fun(a);
diff --git a/tests/codegen-llvm/cast-target-abi.rs b/tests/codegen-llvm/cast-target-abi.rs
index 28d3ad5f61c7..090de00df936 100644
--- a/tests/codegen-llvm/cast-target-abi.rs
+++ b/tests/codegen-llvm/cast-target-abi.rs
@@ -171,7 +171,15 @@ pub extern "C" fn receives_doubledouble(x: DoubleDouble) {
 
     // CHECK: [[RUST_ALLOCA:%.+]] = alloca [16 x i8], align [[RUST_ALIGN:8]]
 
-    // CHECK: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // aarch64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = extractvalue [[ABI_TYPE]] [[ABI_VALUE:%.+]], 0
+    // loongarch64: [[ABI_VALUE_1:%.+]] = extractvalue [[ABI_TYPE]] [[ABI_VALUE:%.+]], 1
+    // loongarch64: store double [[ABI_VALUE_0]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: store double [[ABI_VALUE_1]], ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
+    // powerpc64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // sparc64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // x86_64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // CHECK: call void @llvm.memcpy.{{.+}}(ptr align [[RUST_ALIGN]] [[RUST_ALLOCA]], ptr align [[ABI_ALIGN]] [[ABI_ALLOCA]], i64 16, i1 false)
 }
@@ -190,7 +198,11 @@ pub extern "C" fn returns_doubledouble() -> DoubleDouble {
     // x86_64:      [[ABI_ALLOCA:%.+]] = alloca [16 x i8], align [[ABI_ALIGN:8]]
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x double\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = load double, ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: [[ABI_VALUE_1:%.+]] = load double, ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_2:%.+]] = insertvalue [[ABI_TYPE:{ double, double }]] poison, double [[ABI_VALUE_0]], 0
+    // loongarch64: [[ABI_VALUE:%.+]] = insertvalue { double, double } [[ABI_VALUE_2]], double [[ABI_VALUE_1]], 1
     // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
@@ -269,7 +281,11 @@ pub extern "C" fn receives_doublefloat(x: DoubleFloat) {
     // x86_64:      [[RUST_ALLOCA:%.+]] = alloca [16 x i8], align [[RUST_ALIGN:8]]
 
     // aarch64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = extractvalue { double, float } [[ABI_VALUE]], 0
+    // loongarch64: [[ABI_VALUE_1:%.+]] = extractvalue { double, float } [[ABI_VALUE]], 1
+    // loongarch64: store double [[ABI_VALUE_0]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: store float [[ABI_VALUE_1]], ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
     // powerpc64:   store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
@@ -297,7 +313,11 @@ pub extern "C" fn returns_doublefloat() -> DoubleFloat {
     // x86_64:      [[ABI_ALLOCA:%.+]] = alloca [16 x i8], align [[ABI_ALIGN:8]]
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, float }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = load double, ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: [[ABI_VALUE_1:%.+]] = load float, ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_2:%.+]] = insertvalue [[ABI_TYPE:{ double, float }]] poison, double [[ABI_VALUE_0]], 0
+    // loongarch64: [[ABI_VALUE:%.+]] = insertvalue { double, float } [[ABI_VALUE_2]], float [[ABI_VALUE_1]], 1
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // aarch64:     ret [[ABI_TYPE]] [[ABI_VALUE]]
@@ -429,7 +449,11 @@ pub fn call_doubledouble() {
     // CHECK: call void @llvm.memcpy.{{.+}}(ptr align [[ABI_ALIGN]] [[ABI_ALLOCA]], ptr align [[RUST_ALIGN]] [[RUST_ALLOCA]], i64 16, i1 false)
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x double\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = load double, ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: [[ABI_VALUE_1:%.+]] = load double, ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_2:%.+]] = insertvalue [[ABI_TYPE:{ double, double }]] poison, double [[ABI_VALUE_0]], 0
+    // loongarch64: [[ABI_VALUE:%.+]] = insertvalue { double, double } [[ABI_VALUE_2]], double [[ABI_VALUE_1]], 1
     // powerpc64:   [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // sparc64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
@@ -465,7 +489,11 @@ pub fn return_doubledouble() -> DoubleDouble {
     // x86_64:      [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ double, double }]] @returns_doubledouble()
 
     // aarch64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = extractvalue { double, double } [[ABI_VALUE]], 0
+    // loongarch64: [[ABI_VALUE_1:%.+]] = extractvalue { double, double } [[ABI_VALUE]], 1
+    // loongarch64: store double [[ABI_VALUE_0]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: store double [[ABI_VALUE_1]], ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
     // sparc64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
@@ -500,7 +528,11 @@ pub fn call_doublefloat() {
     // x86_64:      call void @llvm.memcpy.{{.+}}(ptr align [[ABI_ALIGN]] [[ABI_ALLOCA]], ptr align [[RUST_ALIGN]] [[RUST_ALLOCA]], i64 16, i1 false)
 
     // aarch64:     [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, float }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = load double, ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: [[ABI_VALUE_1:%.+]] = load float, ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_2:%.+]] = insertvalue [[ABI_TYPE:{ double, float }]] poison, double [[ABI_VALUE_0]], 0
+    // loongarch64: [[ABI_VALUE:%.+]] = insertvalue { double, float } [[ABI_VALUE_2]], float [[ABI_VALUE_1]], 1
     // powerpc64:   [[ABI_VALUE:%.+]] = load [[ABI_TYPE:\[2 x i64\]]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
     // x86_64:      [[ABI_VALUE:%.+]] = load [[ABI_TYPE:{ double, double }]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
@@ -540,7 +572,11 @@ pub fn return_doublefloat() -> DoubleFloat {
     // x86_64:      [[ABI_VALUE:%.+]] = call [[ABI_TYPE:{ double, double }]] @returns_doublefloat()
 
     // aarch64:     store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
-    // loongarch64: store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_VALUE_0:%.+]] = extractvalue { double, float } [[ABI_VALUE]], 0
+    // loongarch64: [[ABI_VALUE_1:%.+]] = extractvalue { double, float } [[ABI_VALUE]], 1
+    // loongarch64: store double [[ABI_VALUE_0]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
+    // loongarch64: [[ABI_ALLOCA_1:%.+]] = getelementptr inbounds i8, ptr [[ABI_ALLOCA]], i64 8
+    // loongarch64: store float [[ABI_VALUE_1]], ptr [[ABI_ALLOCA_1]], align [[ABI_ALIGN]]
     // x86_64:      store [[ABI_TYPE]] [[ABI_VALUE]], ptr [[ABI_ALLOCA]], align [[ABI_ALIGN]]
 
     // aarch64:     call void @llvm.memcpy.{{.+}}(ptr align [[RUST_ALIGN]] [[RUST_ALLOCA]], ptr align [[ABI_ALIGN]] [[ABI_ALLOCA]], i64 16, i1 false)
diff --git a/tests/codegen-llvm/cffi/c-variadic-ffi.rs b/tests/codegen-llvm/cffi/c-variadic-ffi.rs
index 3e99c9fb84e4..1dee477e9ed7 100644
--- a/tests/codegen-llvm/cffi/c-variadic-ffi.rs
+++ b/tests/codegen-llvm/cffi/c-variadic-ffi.rs
@@ -14,7 +14,7 @@
 //@[arm32] needs-llvm-components: arm
 #![crate_type = "lib"]
 #![feature(no_core)]
-#![feature(extended_varargs_abi_support, extern_system_varargs)]
+#![feature(extern_system_varargs)]
 #![no_core]
 
 extern crate minicore;
diff --git a/tests/codegen-llvm/cffi/c-variadic-inline.rs b/tests/codegen-llvm/cffi/c-variadic-inline.rs
new file mode 100644
index 000000000000..369b7e571cab
--- /dev/null
+++ b/tests/codegen-llvm/cffi/c-variadic-inline.rs
@@ -0,0 +1,47 @@
+//@ compile-flags: -C opt-level=3
+#![feature(c_variadic)]
+
+// Test that the inline attributes are accepted on C-variadic functions.
+//
+// Currently LLVM is unable to inline C-variadic functions, but that is valid because despite
+// the name even `#[inline(always)]` is just a hint.
+
+#[inline(always)]
+unsafe extern "C" fn inline_always(mut ap: ...) -> u32 {
+    ap.arg::()
+}
+
+#[inline]
+unsafe extern "C" fn inline(mut ap: ...) -> u32 {
+    ap.arg::()
+}
+
+#[inline(never)]
+unsafe extern "C" fn inline_never(mut ap: ...) -> u32 {
+    ap.arg::()
+}
+
+#[cold]
+unsafe extern "C" fn cold(mut ap: ...) -> u32 {
+    ap.arg::()
+}
+
+#[unsafe(no_mangle)]
+#[inline(never)]
+fn helper() {
+    // CHECK-LABEL: helper
+    // CHECK-LABEL: call c_variadic_inline::inline_always
+    // CHECK-LABEL: call c_variadic_inline::inline
+    // CHECK-LABEL: call c_variadic_inline::inline_never
+    // CHECK-LABEL: call c_variadic_inline::cold
+    unsafe {
+        inline_always(1);
+        inline(2);
+        inline_never(3);
+        cold(4);
+    }
+}
+
+fn main() {
+    helper()
+}
diff --git a/tests/codegen-llvm/cold-attribute.rs b/tests/codegen-llvm/cold-attribute.rs
new file mode 100644
index 000000000000..92b4280eaf27
--- /dev/null
+++ b/tests/codegen-llvm/cold-attribute.rs
@@ -0,0 +1,78 @@
+// Checks that the cold attribute adds the llvm cold attribute.
+//
+//@ reference: attributes.codegen.cold.intro
+//@ reference: attributes.codegen.cold.trait
+//@ edition:2024
+//@ compile-flags: -Copt-level=0
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: ; cold_attribute::free_function
+// CHECK-NEXT: Function Attrs: cold {{.*}}
+#[cold]
+pub fn free_function() {}
+
+// CHECK-LABEL: ; cold_attribute::async_block
+// CHECK-NEXT: Function Attrs: cold {{.*}}
+#[cold]
+pub async fn async_block() {
+    async fn x(f: impl Future) {
+        f.await;
+    }
+    x(
+        // CHECK-LABEL: ; cold_attribute::async_block::{{{{closure}}}}::{{{{closure}}}}
+        // CHECK-NEXT: Function Attrs: cold {{.*}}
+        #[cold]
+        async {},
+    )
+    .await;
+}
+
+pub fn closure() {
+    fn x(f: impl Fn()) {
+        f()
+    }
+    x(
+        // CHECK-LABEL: ; cold_attribute::closure::{{{{closure}}}}
+        // CHECK-NEXT: Function Attrs: cold {{.*}}
+        #[cold]
+        || {},
+    );
+}
+
+pub struct S;
+
+impl S {
+    // CHECK-LABEL: ; cold_attribute::S::method
+    // CHECK-NEXT: Function Attrs: cold {{.*}}
+    #[cold]
+    pub fn method(&self) {}
+}
+
+pub trait Trait {
+    // CHECK-LABEL: ; cold_attribute::Trait::trait_fn
+    // CHECK-NEXT: Function Attrs: cold {{.*}}
+    #[cold]
+    fn trait_fn(&self) {}
+
+    #[cold]
+    fn trait_fn_overridden(&self) {}
+
+    fn impl_fn(&self);
+}
+
+impl Trait for S {
+    // CHECK-LABEL: ; ::impl_fn
+    // CHECK-NEXT: Function Attrs: cold {{.*}}
+    #[cold]
+    fn impl_fn(&self) {
+        self.trait_fn();
+    }
+
+    // This does not have #[cold], and does not inherit the cold attribute from the trait.
+    // CHECK-LABEL: ; ::trait_fn_overridden
+    // CHECK: ; Function Attrs:
+    // CHECK-NOT: cold
+    // CHECK: define
+    fn trait_fn_overridden(&self) {}
+}
diff --git a/tests/codegen-llvm/common_prim_int_ptr.rs b/tests/codegen-llvm/common_prim_int_ptr.rs
index 53716adccbf2..2c906e86bd73 100644
--- a/tests/codegen-llvm/common_prim_int_ptr.rs
+++ b/tests/codegen-llvm/common_prim_int_ptr.rs
@@ -11,7 +11,7 @@
 #[no_mangle]
 pub fn insert_int(x: usize) -> Result> {
     // CHECK: start:
-    // CHECK-NEXT: %[[WO_PROV:.+]] = getelementptr i8, ptr null, [[USIZE:i[0-9]+]] %x
+    // CHECK-NEXT: %[[WO_PROV:.+]] = inttoptr [[USIZE:i[0-9]+]] %x to ptr
     // CHECK-NEXT: %[[R:.+]] = insertvalue { [[USIZE]], ptr } { [[USIZE]] 0, ptr poison }, ptr %[[WO_PROV]], 1
     // CHECK-NEXT: ret { [[USIZE]], ptr } %[[R]]
     Ok(x)
diff --git a/tests/codegen-llvm/comparison-operators-2-struct.rs b/tests/codegen-llvm/comparison-operators-2-struct.rs
index e179066ebfd7..d44f92f511b6 100644
--- a/tests/codegen-llvm/comparison-operators-2-struct.rs
+++ b/tests/codegen-llvm/comparison-operators-2-struct.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -C opt-level=1
-//@ min-llvm-version: 20
 
 // The `derive(PartialOrd)` for a 2-field type doesn't override `lt`/`le`/`gt`/`ge`.
 // This double-checks that the `Option` intermediate values used
diff --git a/tests/codegen-llvm/comparison-operators-2-tuple.rs b/tests/codegen-llvm/comparison-operators-2-tuple.rs
index 6a7e489c82dd..37a7c5dfdaf0 100644
--- a/tests/codegen-llvm/comparison-operators-2-tuple.rs
+++ b/tests/codegen-llvm/comparison-operators-2-tuple.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -C opt-level=1 -Z merge-functions=disabled
-//@ min-llvm-version: 20
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/cross-crate-inlining/auxiliary/leaf.rs b/tests/codegen-llvm/cross-crate-inlining/auxiliary/leaf.rs
index d059a3d0a73b..7b5679c3f4de 100644
--- a/tests/codegen-llvm/cross-crate-inlining/auxiliary/leaf.rs
+++ b/tests/codegen-llvm/cross-crate-inlining/auxiliary/leaf.rs
@@ -18,3 +18,8 @@ pub fn stem_fn() -> String {
 fn inner() -> String {
     String::from("test")
 }
+
+// This function's optimized MIR contains a call, but it is to an intrinsic.
+pub fn leaf_with_intrinsic(a: &[u64; 2], b: &[u64; 2]) -> bool {
+    a == b
+}
diff --git a/tests/codegen-llvm/cross-crate-inlining/leaf-inlining.rs b/tests/codegen-llvm/cross-crate-inlining/leaf-inlining.rs
index 37132312ca94..5e7912791ad6 100644
--- a/tests/codegen-llvm/cross-crate-inlining/leaf-inlining.rs
+++ b/tests/codegen-llvm/cross-crate-inlining/leaf-inlining.rs
@@ -18,3 +18,10 @@ pub fn stem_outer() -> String {
     // CHECK: call {{.*}}stem_fn
     leaf::stem_fn()
 }
+
+// Check that we inline functions that call intrinsics
+#[no_mangle]
+pub fn leaf_with_intrinsic_outer(a: &[u64; 2], b: &[u64; 2]) -> bool {
+    // CHECK-NOT: call {{.*}}leaf_with_intrinsic
+    leaf::leaf_with_intrinsic(a, b)
+}
diff --git a/tests/codegen-llvm/darwin-no-objc.rs b/tests/codegen-llvm/darwin-no-objc.rs
new file mode 100644
index 000000000000..fda3671fb6d8
--- /dev/null
+++ b/tests/codegen-llvm/darwin-no-objc.rs
@@ -0,0 +1,52 @@
+// Test that we don't generate Objective-C definitions or image info unnecessarily.
+
+//@ add-core-stubs
+//@ revisions: i686_apple_darwin
+//@ [i686_apple_darwin] compile-flags: --target i686-apple-darwin
+//@ [i686_apple_darwin] needs-llvm-components: x86
+//@ revisions: x86_64_macos
+//@ [x86_64_macos] compile-flags: --target x86_64-apple-darwin
+//@ [x86_64_macos] needs-llvm-components: x86
+//@ revisions: aarch64_macos
+//@ [aarch64_macos] compile-flags: --target aarch64-apple-darwin
+//@ [aarch64_macos] needs-llvm-components: aarch64
+//@ revisions: i386_ios
+//@ [i386_ios] compile-flags: --target i386-apple-ios
+//@ [i386_ios] needs-llvm-components: x86
+//@ revisions: x86_64_ios
+//@ [x86_64_ios] compile-flags: --target x86_64-apple-ios
+//@ [x86_64_ios] needs-llvm-components: x86
+//@ revisions: armv7s_ios
+//@ [armv7s_ios] compile-flags: --target armv7s-apple-ios
+//@ [armv7s_ios] needs-llvm-components: arm
+//@ revisions: aarch64_ios
+//@ [aarch64_ios] compile-flags: --target aarch64-apple-ios
+//@ [aarch64_ios] needs-llvm-components: aarch64
+//@ revisions: aarch64_ios_sim
+//@ [aarch64_ios_sim] compile-flags: --target aarch64-apple-ios-sim
+//@ [aarch64_ios_sim] needs-llvm-components: aarch64
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[no_mangle]
+pub fn foo() {}
+
+// CHECK-NOT: %struct._class_t
+// CHECK-NOT: %struct._objc_module
+// CHECK-NOT: @OBJC_CLASS_NAME_
+// CHECK-NOT: @"OBJC_CLASS_$_{{[0-9A-Z_a-z]+}}"
+// CHECK-NOT: @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}"
+// CHECK-NOT: @OBJC_METH_VAR_NAME_
+// CHECK-NOT: @OBJC_SELECTOR_REFERENCES_
+// CHECK-NOT: @OBJC_MODULES
+
+// CHECK-NOT: !"Objective-C Version"
+// CHECK-NOT: !"Objective-C Image Info Version"
+// CHECK-NOT: !"Objective-C Image Info Section"
+// CHECK-NOT: !"Objective-C Is Simulated"
+// CHECK-NOT: !"Objective-C Class Properties"
diff --git a/tests/codegen-llvm/darwin-objc-abi-v1.rs b/tests/codegen-llvm/darwin-objc-abi-v1.rs
new file mode 100644
index 000000000000..0fc1de9332ac
--- /dev/null
+++ b/tests/codegen-llvm/darwin-objc-abi-v1.rs
@@ -0,0 +1,100 @@
+// ignore-tidy-linelength
+//@ add-core-stubs
+//@ revisions: i686_apple_darwin
+//@ [i686_apple_darwin] compile-flags: --target i686-apple-darwin
+//@ [i686_apple_darwin] needs-llvm-components: x86
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items, rustc_attrs)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[no_mangle]
+pub fn get_class() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_class = "MyClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_class_again() -> *mut () {
+    // Codegen should de-duplicate this class with the one from get_class above.
+    unsafe extern "C" {
+        #[rustc_objc_class = "MyClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_selector() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_selector = "myMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_selector_again() -> *mut () {
+    // Codegen should de-duplicate this selector with the one from get_selector above.
+    unsafe extern "C" {
+        #[rustc_objc_selector = "myMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_other_class() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_class = "OtherClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_other_selector() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_selector = "otherMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+// CHECK: %struct._objc_module = type { i32, i32, ptr, ptr }
+
+// CHECK: @OBJC_CLASS_NAME_.{{[0-9]+}} = private unnamed_addr constant [8 x i8] c"MyClass\00", section "__TEXT,__cstring,cstring_literals", align 1
+// CHECK: @OBJC_CLASS_REFERENCES_.{{[0-9]+}} = private global ptr @OBJC_CLASS_NAME_.{{[0-9]+}}, section "__OBJC,__cls_refs,literal_pointers,no_dead_strip", align 4
+// CHECK-NOT: @OBJC_CLASS_NAME_
+// CHECK-NOT: @OBJC_CLASS_REFERENCES_
+
+// CHECK: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [9 x i8] c"myMethod\00", section "__TEXT,__cstring,cstring_literals", align 1
+// CHECK: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = private externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}, section "__OBJC,__message_refs,literal_pointers,no_dead_strip", align 4
+// CHECK-NOT: @OBJC_METH_VAR_NAME_
+// CHECK-NOT: @OBJC_SELECTOR_REFERENCES_
+
+// CHECK: @OBJC_CLASS_NAME_.{{[0-9]+}} = private unnamed_addr constant [11 x i8] c"OtherClass\00", section "__TEXT,__cstring,cstring_literals", align 1
+// CHECK: @OBJC_CLASS_REFERENCES_.{{[0-9]+}} = private global ptr @OBJC_CLASS_NAME_.{{[0-9]+}}, section "__OBJC,__cls_refs,literal_pointers,no_dead_strip", align 4
+
+// CHECK: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [12 x i8] c"otherMethod\00", section "__TEXT,__cstring,cstring_literals", align 1
+// CHECK: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = private externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}, section "__OBJC,__message_refs,literal_pointers,no_dead_strip", align 4
+
+// CHECK: @OBJC_CLASS_NAME_.{{[0-9]+}} = private unnamed_addr constant [1 x i8] zeroinitializer, section "__TEXT,__cstring,cstring_literals", align 1
+// CHECK: @OBJC_MODULES = private global %struct._objc_module { i32 7, i32 16, ptr @OBJC_CLASS_NAME_.{{[0-9]+}}, ptr null }, section "__OBJC,__module_info,regular,no_dead_strip", align 4
+
+// CHECK: load ptr, ptr @OBJC_CLASS_REFERENCES_.{{[0-9]+}}, align 4
+// CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}}, align 4
+// CHECK: load ptr, ptr @OBJC_CLASS_REFERENCES_.{{[0-9]+}}, align 4
+// CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}}, align 4
+
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Version", i32 1}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Version", i32 0}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Section", !"__OBJC,__image_info,regular"}
+// CHECK-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Class Properties", i32 64}
diff --git a/tests/codegen-llvm/darwin-objc-abi-v2.rs b/tests/codegen-llvm/darwin-objc-abi-v2.rs
new file mode 100644
index 000000000000..f142371d5825
--- /dev/null
+++ b/tests/codegen-llvm/darwin-objc-abi-v2.rs
@@ -0,0 +1,185 @@
+// ignore-tidy-linelength
+//@ add-core-stubs
+//@ revisions: x86_64_macos
+//@ [x86_64_macos] compile-flags: --target x86_64-apple-darwin
+//@ [x86_64_macos] needs-llvm-components: x86
+//@ revisions: aarch64_macos
+//@ [aarch64_macos] compile-flags: --target aarch64-apple-darwin
+//@ [aarch64_macos] needs-llvm-components: aarch64
+//@ revisions: i386_ios
+//@ [i386_ios] compile-flags: --target i386-apple-ios
+//@ [i386_ios] needs-llvm-components: x86
+//@ revisions: x86_64_ios
+//@ [x86_64_ios] compile-flags: --target x86_64-apple-ios
+//@ [x86_64_ios] needs-llvm-components: x86
+//@ revisions: armv7s_ios
+//@ [armv7s_ios] compile-flags: --target armv7s-apple-ios
+//@ [armv7s_ios] needs-llvm-components: arm
+//@ revisions: aarch64_ios
+//@ [aarch64_ios] compile-flags: --target aarch64-apple-ios
+//@ [aarch64_ios] needs-llvm-components: aarch64
+//@ revisions: aarch64_ios_sim
+//@ [aarch64_ios_sim] compile-flags: --target aarch64-apple-ios-sim
+//@ [aarch64_ios_sim] needs-llvm-components: aarch64
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items, rustc_attrs)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[no_mangle]
+pub fn get_class() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_class = "MyClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_class_again() -> *mut () {
+    // Codegen should de-duplicate this class with the one from get_class above.
+    unsafe extern "C" {
+        #[rustc_objc_class = "MyClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_selector() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_selector = "myMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_selector_again() -> *mut () {
+    // Codegen should de-duplicate this selector with the one from get_selector above.
+    unsafe extern "C" {
+        #[rustc_objc_selector = "myMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_other_class() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_class = "OtherClass"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+#[no_mangle]
+pub fn get_other_selector() -> *mut () {
+    unsafe extern "C" {
+        #[rustc_objc_selector = "otherMethod"]
+        safe static VAL: *mut ();
+    }
+    VAL
+}
+
+// CHECK: %struct._class_t = type { ptr, ptr, ptr, ptr, ptr }
+
+// CHECK: @"OBJC_CLASS_$_MyClass" = external global %struct._class_t
+// CHECK: @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}" = internal global ptr @"OBJC_CLASS_$_MyClass", section "__DATA,__objc_classrefs,regular,no_dead_strip",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+// CHECK-NOT: @"OBJC_CLASS_$_MyClass"
+// CHECK-NOT: @"OBJC_CLASSLIST_REFERENCES_$_
+
+// CHECK: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [9 x i8] c"myMethod\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+// CHECK: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+// CHECK-NOT: @OBJC_METH_VAR_NAME_
+// CHECK-NOT: @OBJC_SELECTOR_REFERENCES_
+
+// CHECK: @"OBJC_CLASS_$_OtherClass" = external global %struct._class_t
+// CHECK: @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}" = internal global ptr @"OBJC_CLASS_$_OtherClass", section "__DATA,__objc_classrefs,regular,no_dead_strip",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [12 x i8] c"otherMethod\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+// CHECK: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK-NOT: @OBJC_CLASS_NAME_
+// CHECK-NOT: @OBJC_MODULES
+
+// CHECK: load ptr, ptr @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}},
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK: load ptr, ptr @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}",
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}},
+// x86_64_macos-SAME: align 8
+// aarch64_macos-SAME: align 8
+// i386_ios-SAME: align 4
+// x86_64_ios-SAME: align 8
+// armv7s_ios-SAME: align 4
+// aarch64_ios-SAME: align 8
+// aarch64_ios_sim-SAME: align 8
+
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Version", i32 2}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Version", i32 0}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Section", !"__DATA,__objc_imageinfo,regular,no_dead_strip"}
+
+// x86_64_macos-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// aarch64_macos-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// i386_ios: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// x86_64_ios: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// armv7s_ios-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// aarch64_ios-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// aarch64_ios_sim: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Class Properties", i32 64}
diff --git a/tests/codegen-llvm/darwin-objc-cross-crate.rs b/tests/codegen-llvm/darwin-objc-cross-crate.rs
new file mode 100644
index 000000000000..74ad9a27346d
--- /dev/null
+++ b/tests/codegen-llvm/darwin-objc-cross-crate.rs
@@ -0,0 +1,58 @@
+// Test that Objective-C class and selector references inlined across crates
+// get defined in this CGU but non-inline references don't.
+
+// ignore-tidy-linelength
+//@ aux-build: darwin_objc_aux.rs
+//@ revisions: x86_64_macos aarch64_macos
+//@ [x86_64_macos] only-x86_64-apple-darwin
+//@ [aarch64_macos] only-aarch64-apple-darwin
+
+#![crate_type = "lib"]
+#![feature(darwin_objc)]
+
+use std::os::darwin::objc;
+
+extern crate darwin_objc_aux as aux;
+
+#[no_mangle]
+pub fn get_object_class() -> objc::Class {
+    aux::inline_get_object_class()
+}
+
+#[no_mangle]
+pub fn get_alloc_selector() -> objc::SEL {
+    aux::inline_get_alloc_selector()
+}
+
+#[no_mangle]
+pub fn get_string_class() -> objc::Class {
+    aux::never_inline_get_string_class()
+}
+
+#[no_mangle]
+pub fn get_init_selector() -> objc::SEL {
+    aux::never_inline_get_init_selector()
+}
+
+// CHECK: %struct._class_t = type { ptr, ptr, ptr, ptr, ptr }
+
+// CHECK: @"OBJC_CLASS_$_NSObject" = external global %struct._class_t
+// CHECK: @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}" = internal global ptr @"OBJC_CLASS_$_NSObject", section "__DATA,__objc_classrefs,regular,no_dead_strip", align 8
+
+// CHECK: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [6 x i8] c"alloc\00", section "__TEXT,__objc_methname,cstring_literals", align 1
+// CHECK: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}, section "__DATA,__objc_selrefs,literal_pointers,no_dead_strip", align 8
+
+// CHECK-NOT: @"OBJC_CLASS_$_NSString" = external global %struct._class_t
+// CHECK-NOT: @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}" = internal global ptr @"OBJC_CLASS_$_NSString"
+
+// CHECK-NOT: @OBJC_METH_VAR_NAME_.{{[0-9]+}} = private unnamed_addr constant [5 x i8] c"init\00"
+// CHECK-NOT: @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}} = internal externally_initialized global ptr @OBJC_METH_VAR_NAME_.{{[0-9]+}}
+
+// CHECK: load ptr, ptr @"OBJC_CLASSLIST_REFERENCES_$_.{{[0-9]+}}", align 8
+// CHECK: load ptr, ptr @OBJC_SELECTOR_REFERENCES_.{{[0-9]+}}, align 8
+
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Version", i32 2}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Version", i32 0}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Image Info Section", !"__DATA,__objc_imageinfo,regular,no_dead_strip"}
+// CHECK-NOT: !{{[0-9]+}} = !{i32 1, !"Objective-C Is Simulated", i32 32}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"Objective-C Class Properties", i32 64}
diff --git a/tests/codegen-llvm/dead_on_return.rs b/tests/codegen-llvm/dead_on_return.rs
new file mode 100644
index 000000000000..3c1940d6ba7c
--- /dev/null
+++ b/tests/codegen-llvm/dead_on_return.rs
@@ -0,0 +1,31 @@
+//@ compile-flags: -C opt-level=3
+//@ min-llvm-version: 21
+
+#![crate_type = "lib"]
+#![allow(unused_assignments, unused_variables)]
+
+// Check that the old string is deallocated, but a new one is not initialized.
+#[unsafe(no_mangle)]
+pub fn test_str_new(mut s: String) {
+    // CHECK-LABEL: @test_str_new
+    // CHECK: __rust_dealloc
+    // CHECK-NOT: store
+    s = String::new();
+}
+
+#[unsafe(no_mangle)]
+pub fn test_str_take(mut x: String) -> String {
+    // CHECK-LABEL: @test_str_take
+    // CHECK-NEXT: {{.*}}:
+    // CHECK-NEXT: call void @llvm.memcpy
+    // CHECK-NEXT: ret
+    core::mem::take(&mut x)
+}
+
+#[unsafe(no_mangle)]
+pub fn test_array_store(mut x: [u32; 100]) {
+    // CHECK-LABEL: @test_array_store
+    // CHECK-NEXT: {{.*}}:
+    // CHECK-NEXT: ret
+    x[0] = 1;
+}
diff --git a/tests/codegen-llvm/debug-fndef-size.rs b/tests/codegen-llvm/debug-fndef-size.rs
index 8f716c34e7b7..02629bd748c4 100644
--- a/tests/codegen-llvm/debug-fndef-size.rs
+++ b/tests/codegen-llvm/debug-fndef-size.rs
@@ -16,5 +16,5 @@ pub fn main() {
 
 // CHECK: %compare.dbg.spill = alloca [0 x i8], align 1
 // CHECK: dbg{{.}}declare({{(metadata )?}}ptr %compare.dbg.spill, {{(metadata )?}}![[VAR:.*]], {{(metadata )?}}!DIExpression()
-// CHECK: ![[TYPE:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, name: "fn(&i32, &i32) -> core::cmp::Ordering", baseType: !{{.*}}, align: 8, dwarfAddressSpace: {{.*}})
-// CHECK: ![[VAR]] = !DILocalVariable(name: "compare", scope: !{{.*}}, file: !{{.*}}, line: {{.*}}, type: ![[TYPE]], align: 8)
+// CHECK-DAG: ![[TYPE:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, name: "fn(&i32, &i32) -> core::cmp::Ordering", baseType: !{{.*}}, align: 8, dwarfAddressSpace: {{.*}})
+// CHECK-DAG: ![[VAR]] = !DILocalVariable(name: "compare", scope: !{{.*}}, file: !{{.*}}, line: {{.*}}, type: ![[TYPE]], align: 8)
diff --git a/tests/codegen-llvm/debuginfo-dse.rs b/tests/codegen-llvm/debuginfo-dse.rs
new file mode 100644
index 000000000000..fd0c9f1c676f
--- /dev/null
+++ b/tests/codegen-llvm/debuginfo-dse.rs
@@ -0,0 +1,362 @@
+//@ compile-flags: -Copt-level=3 -g -Zverify-llvm-ir -Zmerge-functions=disabled
+//@ revisions: CODEGEN OPTIMIZED
+//@[CODEGEN] compile-flags: -Cno-prepopulate-passes
+//@ only-64bit
+// ignore-tidy-linelength
+
+#![crate_type = "lib"]
+#![feature(repr_simd, rustc_attrs)]
+
+// The pass mode is direct and the backend represent is scalar.
+type Scalar = i32; // scalar(i32)
+type Scalar_Ref = &'static i32; // scalar(ptr)
+
+// The pass modes are pair and the backend represents are scalar pair.
+type Tuple_Scalar_Scalar = (i32, i32);
+struct Tuple_Ref_Scalar(&'static i32, i32);
+struct Tuple_ArrayRef_Scalar(&'static [i32; 16], i32); // pair(ptr, i32)
+impl Default for Tuple_ArrayRef_Scalar {
+    fn default() -> Tuple_ArrayRef_Scalar {
+        Tuple_ArrayRef_Scalar(&[0; 16], 0)
+    }
+}
+struct Tuple_Scalar_ArrayRef(i32, &'static [i32; 16]); // pair(i32, ptr)
+impl Default for Tuple_Scalar_ArrayRef {
+    fn default() -> Tuple_Scalar_ArrayRef {
+        Tuple_Scalar_ArrayRef(0, &[0; 16])
+    }
+}
+// The pass mode is indirect and the backend represent is memory.
+type Tuple_SliceRef_Scalar = (&'static [i32], i32);
+
+// The pass mode is pair and the backend represent is scalar pair.
+type SliceRef = &'static [i32]; // pair(ptr, i32)
+// The pass mode is indirect and the backend represent is memory.
+type Array = [i32; 16];
+// The pass mode is direct and the backend represent is scalar.
+type ArrayRef = &'static [i32; 16];
+
+// The pass mode is indirect and the backend represent is memory.
+type Typle_i32_i64_i8 = (i32, i64, i8);
+// The pass mode is indirect and the backend represent is memory.
+#[repr(C)]
+struct Aggregate_i32_Array_i8(i32, &'static [i32; 16], i8);
+
+type ZST = ();
+
+impl Default for Aggregate_i32_Array_i8 {
+    fn default() -> Aggregate_i32_Array_i8 {
+        Aggregate_i32_Array_i8(0, &[0; 16], 0)
+    }
+}
+// The pass mode is cast and the backend represent is scalar.
+#[derive(Default)]
+struct Aggregate_4xi8(i8, i8, i8, i8); // scalar(i32)
+
+// The pass mode is indirect and the backend represent is simd vector.
+#[repr(simd)]
+struct Simd_i32x4([i32; 4]);
+
+unsafe extern "Rust" {
+    #[rustc_nounwind]
+    safe fn opaque_fn();
+    #[rustc_nounwind]
+    safe fn opaque_ptr(_: *const core::ffi::c_void);
+}
+
+#[inline(never)]
+#[rustc_nounwind]
+fn opaque_use(p: &T) {
+    opaque_ptr(&raw const p as *const _);
+}
+
+#[inline(never)]
+#[rustc_nounwind]
+fn opaque_read() -> T {
+    core::hint::black_box(T::default())
+}
+
+#[unsafe(no_mangle)]
+fn local_var() {
+    // CHECK-LABEL: define{{( dso_local)?}} void @local_var
+    let local_var_scalar: Scalar = opaque_read();
+    opaque_use(&local_var_scalar);
+    let dead_local_var_scalar: Scalar = opaque_read();
+    let local_var_aggregate_4xi8: Aggregate_4xi8 = opaque_read();
+    opaque_use(&local_var_aggregate_4xi8);
+    let local_var_aggregate_i32_array_i8: Aggregate_i32_Array_i8 = opaque_read();
+    opaque_use(&local_var_aggregate_i32_array_i8);
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr %local_var_scalar, [[ref_local_var_scalar:![0-9]+]], !DIExpression()
+    let ref_local_var_scalar = &local_var_scalar;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_dead_local_var_scalar:![0-9]+]], !DIExpression()
+    let ref_dead_local_var_scalar = &dead_local_var_scalar;
+    // CHECK-NEXT: #dbg_value(ptr %local_var_aggregate_4xi8, [[ref_local_var_aggregate_4xi8:![0-9]+]], !DIExpression()
+    let ref_local_var_aggregate_4xi8 = &local_var_aggregate_4xi8;
+    // CHECK-NEXT: #dbg_value(ptr %local_var_aggregate_4xi8, [[ref_0_local_var_aggregate_4xi8:![0-9]+]], !DIExpression()
+    let ref_0_local_var_aggregate_4xi8 = &local_var_aggregate_4xi8.0;
+    // CHECK-NEXT: #dbg_value(ptr %local_var_aggregate_4xi8, [[ref_2_local_var_aggregate_4xi8:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 2, DW_OP_stack_value)
+    let ref_2_local_var_aggregate_4xi8 = &local_var_aggregate_4xi8.2;
+    // This introduces an extra load instruction.
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_1_1_local_var_aggregate_i32_array_i8:![0-9]+]], !DIExpression()
+    let ref_1_1_local_var_aggregate_i32_array_i8 = &local_var_aggregate_i32_array_i8.1[1];
+    // CHECK-NEXT: #dbg_value(ptr %local_var_aggregate_i32_array_i8, [[ref_2_local_var_aggregate_i32_array_i8:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 16, DW_OP_stack_value)
+    let ref_2_local_var_aggregate_i32_array_i8 = &local_var_aggregate_i32_array_i8.2;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+#[unsafe(no_mangle)]
+fn zst(zst: ZST, zst_ref: &ZST) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @zst
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_zst:![0-9]+]], !DIExpression()
+    let ref_zst = &zst;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_zst_ref:![0-9]+]], !DIExpression()
+    let ref_zst_ref = &zst_ref;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+// It only makes sense if the argument is a reference and it refer to projections.
+#[unsafe(no_mangle)]
+fn direct(
+    scalar: Scalar,
+    scalar_ref: Scalar_Ref,
+    array_ref: ArrayRef,
+    aggregate_4xi8_ref: &Aggregate_4xi8,
+) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @direct
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_scalar:![0-9]+]], !DIExpression()
+    let ref_scalar = &scalar;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_scalar_ref:![0-9]+]], !DIExpression()
+    let ref_scalar_ref = &scalar_ref;
+    // CHECK-NEXT: #dbg_value(ptr %array_ref, [[ref_0_array_ref:![0-9]+]], !DIExpression()
+    let ref_0_array_ref = &array_ref[0];
+    // CHECK-NEXT: #dbg_value(ptr %array_ref, [[ref_1_array_ref:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value)
+    let ref_1_array_ref = &array_ref[1];
+    // CHECK-NEXT: #dbg_value(ptr %aggregate_4xi8_ref, [[ref_1_aggregate_4xi8_ref:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)
+    let ref_1_aggregate_4xi8_ref = &aggregate_4xi8_ref.1;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+// Arguments are passed through registers, the final values are poison.
+#[unsafe(no_mangle)]
+fn cast(aggregate_4xi8: Aggregate_4xi8) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @cast(i32 %0)
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // The temporary allocated variable is eliminated.
+    // CODEGEN-NEXT: #dbg_value(ptr %aggregate_4xi8, [[ref_aggregate_4xi8:![0-9]+]], !DIExpression()
+    // OPTIMIZED-NEXT: #dbg_value(ptr undef, [[ref_aggregate_4xi8:![0-9]+]], !DIExpression()
+    let ref_aggregate_4xi8 = &aggregate_4xi8;
+    // CODEGEN-NEXT: #dbg_value(ptr %aggregate_4xi8, [[ref_0_aggregate_4xi8:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)
+    // OPTIMIZED-NEXT: #dbg_value(ptr undef, [[ref_0_aggregate_4xi8:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value)
+    let ref_0_aggregate_4xi8 = &aggregate_4xi8.1;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+// Arguments are passed indirectly via a pointer.
+// The reference of argument is the pointer itself.
+#[unsafe(no_mangle)]
+fn indirect(
+    tuple_sliceref_scalar: Tuple_SliceRef_Scalar,
+    array: Array,
+    typle_i32_i64_i8: Typle_i32_i64_i8,
+    simd_i32x4: Simd_i32x4,
+) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @indirect
+    // CHECK-SAME: (ptr{{.*}} %tuple_sliceref_scalar, ptr{{.*}} %array, ptr{{.*}} %typle_i32_i64_i8, ptr{{.*}} %simd_i32x4)
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr %tuple_sliceref_scalar, [[ref_tuple_sliceref_scalar:![0-9]+]], !DIExpression()
+    let ref_tuple_sliceref_scalar = &tuple_sliceref_scalar;
+    // CHECK-NEXT: #dbg_value(ptr %tuple_sliceref_scalar, [[ref_1_tuple_sliceref_scalar:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 16, DW_OP_stack_value)
+    let ref_1_tuple_sliceref_scalar = &tuple_sliceref_scalar.1;
+    // CHECK-NEXT: #dbg_value(ptr %array, [[ref_1_array:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value)
+    let ref_1_array = &array[1];
+    // CHECK-NEXT: #dbg_value(ptr %typle_i32_i64_i8, [[ref_1_typle_i32_i64_i8:![0-9]+]], !DIExpression()
+    let ref_1_typle_i32_i64_i8 = &typle_i32_i64_i8.1;
+    // CHECK-NEXT: #dbg_value(ptr %simd_i32x4, [[ref_simd_i32x4:![0-9]+]], !DIExpression()
+    let ref_simd_i32x4 = &simd_i32x4;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+// They are different MIR statements, but they have the same LLVM IR statement due to the ABI of arguments.
+// Both `direct_ref` and `indirect_byval` are passed as a pointer here.
+#[unsafe(no_mangle)]
+fn direct_ref_and_indirect(
+    direct_ref: &Aggregate_i32_Array_i8,
+    indirect_byval: Aggregate_i32_Array_i8,
+) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @direct_ref_and_indirect
+    // CHECK-SAME: (ptr{{.*}} %direct_ref, ptr{{.*}} %indirect_byval)
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_direct_ref:![0-9]+]], !DIExpression()
+    let ref_direct_ref: &&Aggregate_i32_Array_i8 = &direct_ref;
+    // CHECK-NEXT: #dbg_value(ptr %direct_ref, [[ref_1_direct_ref:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 8, DW_OP_stack_value)
+    let ref_1_direct_ref = &direct_ref.1;
+    // CHECK-NEXT: #dbg_value(ptr %indirect_byval, [[ref_indirect_byval:![0-9]+]], !DIExpression()
+    let ref_indirect_byval: &Aggregate_i32_Array_i8 = &indirect_byval;
+    // CHECK-NEXT: #dbg_value(ptr %indirect_byval, [[ref_1_indirect_byval:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 8, DW_OP_stack_value)
+    let ref_1_indirect_byval = &indirect_byval.1;
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+#[unsafe(no_mangle)]
+fn pair(
+    tuple_scalar_scalar: Tuple_Scalar_Scalar,
+    tuple_ref_scalar: Tuple_Ref_Scalar,
+    tuple_arrayref_scalar: Tuple_ArrayRef_Scalar,
+    tuple_scalar_arrayref: Tuple_Scalar_ArrayRef,
+    sliceref: SliceRef,
+) {
+    // CHECK-LABEL: define{{( dso_local)?}} void @pair
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_0_tuple_scalar_scalar:![0-9]+]], !DIExpression()
+    let ref_0_tuple_scalar_scalar = &tuple_scalar_scalar.0;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_0_tuple_ref_scalar:![0-9]+]], !DIExpression()
+    let ref_0_tuple_ref_scalar = &tuple_ref_scalar.0;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_1_tuple_ref_scalar:![0-9]+]], !DIExpression()
+    let ref_1_tuple_ref_scalar = &tuple_ref_scalar.1;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_0_tuple_arrayref_scalar:![0-9]+]], !DIExpression()
+    let ref_0_tuple_arrayref_scalar = &tuple_arrayref_scalar.0;
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_1_tuple_arrayref_scalar:![0-9]+]], !DIExpression()
+    let ref_1_tuple_arrayref_scalar = &tuple_arrayref_scalar.1;
+    // FIXME: This can be a valid value.
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_0_1_tuple_arrayref_scalar:![0-9]+]], !DIExpression()
+    let ref_0_1_tuple_arrayref_scalar = &tuple_arrayref_scalar.0[1];
+    // FIXME: This can be a valid value.
+    // CHECK-NEXT: #dbg_value(ptr poison, [[ref_1_1_tuple_scalar_arrayref:![0-9]+]], !DIExpression()
+    let ref_1_1_tuple_scalar_arrayref = &tuple_scalar_arrayref.1[1];
+    // CHECK: #dbg_value(ptr %sliceref.0, [[ref_1_sliceref:![0-9]+]], !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value)
+    let ref_1_sliceref = &sliceref[1];
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+}
+
+#[repr(C)]
+#[derive(Clone, Copy)]
+pub struct Foo(i32, i64, i32);
+
+#[repr(C)]
+pub struct Bar<'a> {
+    a: i32,
+    b: i64,
+    foo: &'a Foo,
+}
+
+#[unsafe(no_mangle)]
+pub fn dead_first(dead_first_foo: &Foo) -> &i32 {
+    // CHECK-LABEL: def {{.*}} ptr @dead_first
+    // CHECK-SAME: (ptr {{.*}} [[ARG_dead_first_foo:%.*]])
+    // CODEGEN: #dbg_declare(ptr %dead_first_foo.dbg.spill, [[ARG_dead_first_foo:![0-9]+]], !DIExpression()
+    // OPTIMIZED: #dbg_value(ptr %dead_first_foo, [[ARG_dead_first_foo:![0-9]+]], !DIExpression()
+    // CHECK: #dbg_value(ptr %dead_first_foo, [[VAR_dead_first_v0:![0-9]+]], !DIExpression()
+    // CHECK: %dead_first_v0 = getelementptr{{.*}} i8, ptr %dead_first_foo, i64 16
+    // CODEGEN: #dbg_declare(ptr %dead_first_v0.dbg.spill, [[VAR_dead_first_v0]], !DIExpression()
+    // OPTIMIZED: #dbg_value(ptr %dead_first_v0, [[VAR_dead_first_v0]], !DIExpression()
+    let mut dead_first_v0 = &dead_first_foo.0;
+    dead_first_v0 = &dead_first_foo.2;
+    dead_first_v0
+}
+
+#[unsafe(no_mangle)]
+pub fn fragment(fragment_v1: Foo, mut fragment_v2: Foo) -> Foo {
+    // CHECK-LABEL: define{{( dso_local)?}} void @fragment
+    // CHECK-SAME: (ptr {{.*}}, ptr {{.*}} [[ARG_fragment_v1:%.*]], ptr {{.*}} [[ARG_fragment_v2:%.*]])
+    // CHECK: #dbg_declare(ptr [[ARG_fragment_v1]]
+    // CHECK-NEXT: #dbg_declare(ptr [[ARG_fragment_v2]]
+    // CHECK-NEXT: #dbg_value(ptr [[ARG_fragment_v2]], [[VAR_fragment_f:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+    // CHECK-NEXT: #dbg_value(ptr [[ARG_fragment_v1]], [[VAR_fragment_f:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+    let fragment_f = || {
+        fragment_v2 = fragment_v1;
+    };
+    fragment_v2 = fragment_v1;
+    fragment_v2
+}
+
+#[unsafe(no_mangle)]
+pub fn deref(bar: Bar) -> i32 {
+    // CHECK-LABEL: define{{.*}} i32 @deref
+    // We are unable to represent dereference within this expression.
+    // CHECK: #dbg_value(ptr poison, [[VAR_deref_dead:![0-9]+]], !DIExpression()
+    let deref_dead = &bar.foo.2;
+    bar.a
+}
+
+#[unsafe(no_mangle)]
+fn index(slice: &[i32; 4], idx: usize) -> i32 {
+    // CHECK-LABEL: define{{.*}} i32 @index
+    // CHECK: call void @opaque_fn()
+    opaque_fn();
+    // CHECK: #dbg_value(ptr poison, [[VAR_index_from_var:![0-9]+]], !DIExpression()
+    let index_from_var = &slice[idx];
+    // CHECK: #dbg_value(ptr %slice, [[VAR_const_index_from_start:![0-9]+]], !DIExpression()
+    // CHECK-NEXT: #dbg_value(ptr poison, [[VAR_const_index_from_end:![0-9]+]], !DIExpression()
+    let [ref const_index_from_start, .., ref const_index_from_end] = slice[..] else {
+        return 0;
+    };
+    slice[0]
+}
+
+// CHECK-DAG: [[ref_local_var_scalar]] = !DILocalVariable(name: "ref_local_var_scalar"
+// CHECK-DAG: [[ref_dead_local_var_scalar]] = !DILocalVariable(name: "ref_dead_local_var_scalar"
+// CHECK-DAG: [[ref_local_var_aggregate_4xi8]] = !DILocalVariable(name: "ref_local_var_aggregate_4xi8"
+// CHECK-DAG: [[ref_0_local_var_aggregate_4xi8]] = !DILocalVariable(name: "ref_0_local_var_aggregate_4xi8"
+// CHECK-DAG: [[ref_2_local_var_aggregate_4xi8]] = !DILocalVariable(name: "ref_2_local_var_aggregate_4xi8"
+// CHECK-DAG: [[ref_1_1_local_var_aggregate_i32_array_i8]] = !DILocalVariable(name: "ref_1_1_local_var_aggregate_i32_array_i8"
+// CHECK-DAG: [[ref_2_local_var_aggregate_i32_array_i8]] = !DILocalVariable(name: "ref_2_local_var_aggregate_i32_array_i8"
+
+// CHECK-DAG: [[ref_zst]] = !DILocalVariable(name: "ref_zst"
+// CHECK-DAG: [[ref_zst_ref]] = !DILocalVariable(name: "ref_zst_ref"
+
+// CHECK-DAG: [[ref_scalar]] = !DILocalVariable(name: "ref_scalar"
+// CHECK-DAG: [[ref_scalar_ref]] = !DILocalVariable(name: "ref_scalar_ref"
+// CHECK-DAG: [[ref_0_array_ref]] = !DILocalVariable(name: "ref_0_array_ref"
+// CHECK-DAG: [[ref_1_array_ref]] = !DILocalVariable(name: "ref_1_array_ref"
+// CHECK-DAG: [[ref_1_aggregate_4xi8_ref]] = !DILocalVariable(name: "ref_1_aggregate_4xi8_ref"
+
+// CHECK-DAG: [[ref_aggregate_4xi8]] = !DILocalVariable(name: "ref_aggregate_4xi8"
+// CHECK-DAG: [[ref_0_aggregate_4xi8]] = !DILocalVariable(name: "ref_0_aggregate_4xi8"
+
+// CHECK-DAG: [[ref_tuple_sliceref_scalar]] = !DILocalVariable(name: "ref_tuple_sliceref_scalar"
+// CHECK-DAG: [[ref_1_tuple_sliceref_scalar]] = !DILocalVariable(name: "ref_1_tuple_sliceref_scalar"
+// CHECK-DAG: [[ref_1_array]] = !DILocalVariable(name: "ref_1_array"
+// CHECK-DAG: [[ref_1_typle_i32_i64_i8]] = !DILocalVariable(name: "ref_1_typle_i32_i64_i8"
+// CHECK-DAG: [[ref_simd_i32x4]] = !DILocalVariable(name: "ref_simd_i32x4"
+
+// CHECK-DAG: [[ref_direct_ref]] = !DILocalVariable(name: "ref_direct_ref"
+// CHECK-DAG: [[ref_1_direct_ref]] = !DILocalVariable(name: "ref_1_direct_ref"
+// CHECK-DAG: [[ref_indirect_byval]] = !DILocalVariable(name: "ref_indirect_byval"
+// CHECK-DAG: [[ref_1_indirect_byval]] = !DILocalVariable(name: "ref_1_indirect_byval"
+
+// CHECK-DAG: [[ref_0_tuple_scalar_scalar]] = !DILocalVariable(name: "ref_0_tuple_scalar_scalar"
+// CHECK-DAG: [[ref_0_tuple_ref_scalar]] = !DILocalVariable(name: "ref_0_tuple_ref_scalar"
+// CHECK-DAG: [[ref_1_tuple_ref_scalar]] = !DILocalVariable(name: "ref_1_tuple_ref_scalar"
+// CHECK-DAG: [[ref_0_tuple_arrayref_scalar]] = !DILocalVariable(name: "ref_0_tuple_arrayref_scalar"
+// CHECK-DAG: [[ref_1_tuple_arrayref_scalar]] = !DILocalVariable(name: "ref_1_tuple_arrayref_scalar"
+// CHECK-DAG: [[ref_0_1_tuple_arrayref_scalar]] = !DILocalVariable(name: "ref_0_1_tuple_arrayref_scalar"
+// CHECK-DAG: [[ref_1_1_tuple_scalar_arrayref]] = !DILocalVariable(name: "ref_1_1_tuple_scalar_arrayref"
+// CHECK-DAG: [[ref_1_sliceref]] = !DILocalVariable(name: "ref_1_sliceref"
+
+// CHECK-DAG: [[ARG_dead_first_foo]] = !DILocalVariable(name: "dead_first_foo"
+// CHECK-DAG: [[VAR_dead_first_v0]] = !DILocalVariable(name: "dead_first_v0"
+
+// CHECK-DAG: [[VAR_fragment_f]] = !DILocalVariable(name: "fragment_f"
+
+// CHECK-DAG: [[VAR_deref_dead]] = !DILocalVariable(name: "deref_dead"
+
+// CHECK-DAG: [[VAR_index_from_var]] = !DILocalVariable(name: "index_from_var"
+// CHECK-DAG: [[VAR_const_index_from_start]] = !DILocalVariable(name: "const_index_from_start"
+// CHECK-DAG: [[VAR_const_index_from_end]] = !DILocalVariable(name: "const_index_from_end"
diff --git a/tests/codegen-llvm/deduced-param-attrs.rs b/tests/codegen-llvm/deduced-param-attrs.rs
index 34504c80fad1..eec15a434645 100644
--- a/tests/codegen-llvm/deduced-param-attrs.rs
+++ b/tests/codegen-llvm/deduced-param-attrs.rs
@@ -1,60 +1,180 @@
-//@ compile-flags: -Copt-level=3
-
+//@ compile-flags: -Copt-level=3 -Cno-prepopulate-passes
+//@ compile-flags: -Cpanic=abort -Csymbol-mangling-version=v0
+//@ revisions: LLVM21 LLVM20
+//@ [LLVM21] min-llvm-version: 21
+//@ [LLVM20] max-llvm-major-version: 20
+#![feature(custom_mir, core_intrinsics, unboxed_closures)]
 #![crate_type = "lib"]
-#![allow(internal_features)]
-#![feature(unsized_fn_params)]
-
+extern crate core;
+use core::intrinsics::mir::*;
 use std::cell::Cell;
-use std::hint;
+use std::hint::black_box;
+use std::mem::ManuallyDrop;
 
-// Check to make sure that we can deduce the `readonly` attribute from function bodies for
-// parameters passed indirectly.
-
-pub struct BigStruct {
-    blah: [i32; 1024],
+pub struct Big {
+    pub blah: [i32; 1024],
 }
 
-pub struct BigCellContainer {
-    blah: [Cell; 1024],
+pub struct BigCell {
+    pub blah: [Cell; 1024],
 }
 
-// The by-value parameter for this big struct can be marked readonly.
+pub struct BigDrop {
+    pub blah: [u8; 1024],
+}
+
+impl Drop for BigDrop {
+    #[inline(never)]
+    fn drop(&mut self) {}
+}
+
+// CHECK-LABEL: @mutate(
+// CHECK-NOT: readonly
+// CHECK-SAME: %b)
+#[unsafe(no_mangle)]
+pub fn mutate(mut b: Big) {
+    b.blah[987] = 654;
+    black_box(&b);
+}
+
+// LLVM21-LABEL: @deref_mut({{.*}}readonly {{.*}}captures(none) {{.*}}%c)
+// LLVM20-LABEL: @deref_mut({{.*}}readonly                      {{.*}}%c)
+#[unsafe(no_mangle)]
+pub fn deref_mut(c: (BigCell, &mut usize)) {
+    *c.1 = 42;
+}
+
+// LLVM21-LABEL: @call_copy_arg(ptr {{.*}}readonly {{.*}}captures(none){{.*}})
+// LLVM20-LABEL: @call_copy_arg(ptr {{.*}}readonly                     {{.*}})
+#[unsafe(no_mangle)]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub fn call_copy_arg(a: Big) {
+    mir! {
+        {
+            Call(RET = call_copy_arg(a), ReturnTo(bb1), UnwindUnreachable())
+        }
+        bb1 = {
+            Return()
+        }
+    }
+}
+
+// CHECK-LABEL: @call_move_arg(
+// CHECK-NOT:   readonly
+// LLVM21-SAME: captures(address)
+// CHECK-SAME:  )
+#[unsafe(no_mangle)]
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+pub fn call_move_arg(a: Big) {
+    mir! {
+        {
+            Call(RET = call_move_arg(Move(a)), ReturnTo(bb1), UnwindUnreachable())
+        }
+        bb1 = {
+            Return()
+        }
+    }
+}
+
+fn shared_borrow(a: T) {
+    black_box(&a);
+}
+
+// Freeze parameter cannot be mutated through a shared borrow.
 //
-// CHECK: @use_big_struct_immutably({{.*}} readonly {{.*}} %big_struct)
-#[no_mangle]
-pub fn use_big_struct_immutably(big_struct: BigStruct) {
-    hint::black_box(&big_struct);
-}
+// CHECK-LABEL: ; deduced_param_attrs::shared_borrow::
+// CHECK-NEXT:  ;
+// LLVM21-NEXT: (ptr {{.*}}readonly {{.*}}captures(address) {{.*}}%a)
+// LLVM20-NEXT: (ptr {{.*}}readonly                         {{.*}}%a)
+pub static A0: fn(Big) = shared_borrow;
 
-// The by-value parameter for this big struct can't be marked readonly, because we mutate it.
+// !Freeze parameter can be mutated through a shared borrow.
 //
-// CHECK-NOT: @use_big_struct_mutably({{.*}} readonly {{.*}} %big_struct)
-#[no_mangle]
-pub fn use_big_struct_mutably(mut big_struct: BigStruct) {
-    big_struct.blah[987] = 654;
-    hint::black_box(&big_struct);
-}
+// CHECK-LABEL: ; deduced_param_attrs::shared_borrow::
+// CHECK-NEXT:  ;
+// CHECK-NOT:   readonly
+// CHECK-NEXT:  %a)
+pub static A1: fn(BigCell) = shared_borrow;
 
-// The by-value parameter for this big struct can't be marked readonly, because it contains
-// UnsafeCell.
+// The parameter can be mutated through a raw const borrow.
 //
-// CHECK-NOT: @use_big_cell_container({{.*}} readonly {{.*}} %big_cell_container)
-#[no_mangle]
-pub fn use_big_cell_container(big_cell_container: BigCellContainer) {
-    hint::black_box(&big_cell_container);
-}
-
-// Make sure that we don't mistakenly mark a big struct as `readonly` when passed through a generic
-// type parameter if it contains UnsafeCell.
-//
-// CHECK-NOT: @use_something({{.*}} readonly {{.*}} %something)
-#[no_mangle]
+// CHECK-LABEL: ; deduced_param_attrs::raw_const_borrow
+// CHECK-NOT:   readonly
+// CHECK-NEXT : %a)
 #[inline(never)]
-pub fn use_something(something: T) {
-    hint::black_box(&something);
+pub fn raw_const_borrow(a: Big) {
+    black_box(&raw const a);
 }
 
-#[no_mangle]
-pub fn forward_big_cell_container(big_cell_container: BigCellContainer) {
-    use_something(big_cell_container)
+fn consume(_: T) {}
+
+// The parameter doesn't need to be dropped.
+//
+// CHECK-LABEL: ; deduced_param_attrs::consume::
+// CHECK-NEXT:  ;
+// LLVM21-NEXT: (ptr {{.*}}readonly {{.*}}captures(none) {{.*}})
+// LLVM20-NEXT: (ptr {{.*}}readonly                      {{.*}})
+pub static B0: fn(BigCell) = consume;
+
+// The parameter needs to be dropped.
+//
+// CHECK-LABEL: ; deduced_param_attrs::consume::
+// CHECK-NEXT:  ;
+// LLVM21-NEXT: (ptr {{.*}}captures(address) {{.*}})
+// LLVM20-NEXT: (ptr                         {{.*}})
+pub static B1: fn(BigDrop) = consume;
+
+fn consume_parts(t: (T, T)) {
+    let (_t0, ..) = t;
 }
+
+// In principle it would be possible to deduce readonly here.
+//
+// CHECK-LABEL: ; deduced_param_attrs::consume_parts::<[u8; 40]>
+// CHECK-NEXT:  ;
+// CHECK-NOT:   readonly
+// CHECK-NEXT:  %t)
+pub static C1: fn(([u8; 40], [u8; 40])) = consume_parts;
+
+// The inner field of ManuallyDrop needs to be dropped.
+//
+// CHECK-LABEL: @manually_drop_field(
+// CHECK-NOT:   readonly
+// CHECK-SAME:  %b)
+#[unsafe(no_mangle)]
+pub fn manually_drop_field(a: fn() -> BigDrop, mut b: ManuallyDrop) {
+    // FIXME(tmiasko) replace with custom MIR, instead of expecting MIR optimizations to turn this
+    // into: drop((_2.0: BigDrop))
+    *b = a();
+    unsafe { core::intrinsics::unreachable() }
+}
+
+// `readonly` is omitted from the return place, even when applicable.
+//
+// CHECK-LABEL: @never_returns(
+// CHECK-NOT:   readonly
+// CHECK-SAME:  %_0)
+#[unsafe(no_mangle)]
+pub fn never_returns() -> [u8; 80] {
+    loop {}
+}
+
+// LLVM21-LABEL: @not_captured_return_place(ptr{{.*}} captures(none) {{.*}}%_0)
+#[unsafe(no_mangle)]
+pub fn not_captured_return_place() -> [u8; 80] {
+    [0u8; 80]
+}
+
+// LLVM21-LABEL: @captured_return_place(ptr{{.*}} captures(address) {{.*}}%_0)
+#[unsafe(no_mangle)]
+pub fn captured_return_place() -> [u8; 80] {
+    black_box([0u8; 80])
+}
+
+// Arguments spread at ABI level are unsupported.
+//
+// CHECK-LABEL: @spread_arg(
+// CHECK-NOT: readonly
+// CHECK-SAME: )
+#[no_mangle]
+pub extern "rust-call" fn spread_arg(_: (Big, Big, Big)) {}
diff --git a/tests/codegen-llvm/default-visibility.rs b/tests/codegen-llvm/default-visibility.rs
index 88ff9fee2544..28238e5ef129 100644
--- a/tests/codegen-llvm/default-visibility.rs
+++ b/tests/codegen-llvm/default-visibility.rs
@@ -32,6 +32,7 @@ pub static tested_symbol: [u8; 6] = *b"foobar";
 // INTERPOSABLE: @{{.*}}default_visibility{{.*}}tested_symbol{{.*}} = constant
 // DEFAULT:      @{{.*}}default_visibility{{.*}}tested_symbol{{.*}} = constant
 
+#[inline(never)]
 pub fn do_memcmp(left: &[u8], right: &[u8]) -> i32 {
     left.cmp(right) as i32
 }
diff --git a/tests/codegen-llvm/emscripten-catch-unwind-js-eh.rs b/tests/codegen-llvm/emscripten-catch-unwind-js-eh.rs
index f43869cf2189..dfe154c1c831 100644
--- a/tests/codegen-llvm/emscripten-catch-unwind-js-eh.rs
+++ b/tests/codegen-llvm/emscripten-catch-unwind-js-eh.rs
@@ -25,7 +25,7 @@ trait Copy {}
 impl Copy for *mut T {}
 
 #[rustc_intrinsic]
-fn size_of() -> usize {
+const fn size_of() -> usize {
     loop {}
 }
 
@@ -40,7 +40,7 @@ unsafe fn catch_unwind(
 #[no_mangle]
 pub fn ptr_size() -> usize {
     // CHECK: ret [[PTR_SIZE:.*]]
-    size_of::<*mut u8>()
+    const { size_of::<*mut u8>() }
 }
 
 // CHECK-LABEL: @test_catch_unwind
diff --git a/tests/codegen-llvm/emscripten-catch-unwind-wasm-eh.rs b/tests/codegen-llvm/emscripten-catch-unwind-wasm-eh.rs
index b0750d52268a..88c95b088aaa 100644
--- a/tests/codegen-llvm/emscripten-catch-unwind-wasm-eh.rs
+++ b/tests/codegen-llvm/emscripten-catch-unwind-wasm-eh.rs
@@ -24,7 +24,7 @@ trait Copy {}
 impl Copy for *mut T {}
 
 #[rustc_intrinsic]
-fn size_of() -> usize {
+const fn size_of() -> usize {
     loop {}
 }
 #[rustc_intrinsic]
@@ -38,7 +38,7 @@ unsafe fn catch_unwind(
 #[no_mangle]
 pub fn ptr_size() -> usize {
     // CHECK: ret [[PTR_SIZE:.*]]
-    size_of::<*mut u8>()
+    const { size_of::<*mut u8>() }
 }
 
 // CHECK-LABEL: @test_catch_unwind
diff --git a/tests/codegen-llvm/enum/enum-aggregate.rs b/tests/codegen-llvm/enum/enum-aggregate.rs
index 0161e5f3fa1a..89517008c637 100644
--- a/tests/codegen-llvm/enum/enum-aggregate.rs
+++ b/tests/codegen-llvm/enum/enum-aggregate.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
-//@ min-llvm-version: 19
 //@ only-64bit
 
 #![crate_type = "lib"]
@@ -27,7 +26,7 @@ fn make_none_bool() -> Option {
 
 #[no_mangle]
 fn make_some_ordering(x: Ordering) -> Option {
-    // CHECK-LABEL: i8 @make_some_ordering(i8 %x)
+    // CHECK-LABEL: i8 @make_some_ordering(i8{{( signext)?}} %x)
     // CHECK-NEXT: start:
     // CHECK-NEXT: ret i8 %x
     Some(x)
@@ -35,7 +34,7 @@ fn make_some_ordering(x: Ordering) -> Option {
 
 #[no_mangle]
 fn make_some_u16(x: u16) -> Option {
-    // CHECK-LABEL: { i16, i16 } @make_some_u16(i16 %x)
+    // CHECK-LABEL: { i16, i16 } @make_some_u16(i16{{( zeroext)?}} %x)
     // CHECK-NEXT: start:
     // CHECK-NEXT: %0 = insertvalue { i16, i16 } { i16 1, i16 poison }, i16 %x, 1
     // CHECK-NEXT: ret { i16, i16 } %0
@@ -52,7 +51,7 @@ fn make_none_u16() -> Option {
 
 #[no_mangle]
 fn make_some_nzu32(x: NonZero) -> Option> {
-    // CHECK-LABEL: i32 @make_some_nzu32(i32 %x)
+    // CHECK-LABEL: i32 @make_some_nzu32(i32{{( signext)?}} %x)
     // CHECK-NEXT: start:
     // CHECK-NEXT: ret i32 %x
     Some(x)
@@ -71,7 +70,7 @@ fn make_ok_ptr(x: NonNull) -> Result, usize> {
 fn make_ok_int(x: usize) -> Result> {
     // CHECK-LABEL: { i64, ptr } @make_ok_int(i64 %x)
     // CHECK-NEXT: start:
-    // CHECK-NEXT: %[[NOPROV:.+]] = getelementptr i8, ptr null, i64 %x
+    // CHECK-NEXT: %[[NOPROV:.+]] = inttoptr i64 %x to ptr
     // CHECK-NEXT: %[[R:.+]] = insertvalue { i64, ptr } { i64 0, ptr poison }, ptr %[[NOPROV]], 1
     // CHECK-NEXT: ret { i64, ptr } %[[R]]
     Ok(x)
@@ -114,7 +113,7 @@ fn make_uninhabited_err_indirectly(n: Never) -> Result {
 fn make_fully_uninhabited_result(v: u32, n: Never) -> Result<(u32, Never), (Never, u32)> {
     // Actually reaching this would be UB, so we don't actually build a result.
 
-    // CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32 %v)
+    // CHECK-LABEL: { i32, i32 } @make_fully_uninhabited_result(i32{{( signext)?}} %v)
     // CHECK-NEXT: start:
     // CHECK-NEXT: call void @llvm.trap()
     // CHECK-NEXT: call void @llvm.trap()
diff --git a/tests/codegen-llvm/enum/enum-discriminant-eq.rs b/tests/codegen-llvm/enum/enum-discriminant-eq.rs
index d599685c2e51..68cd58643e84 100644
--- a/tests/codegen-llvm/enum/enum-discriminant-eq.rs
+++ b/tests/codegen-llvm/enum/enum-discriminant-eq.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -Copt-level=3 -Zmerge-functions=disabled
-//@ min-llvm-version: 20
 //@ only-64bit
 //@ revisions: LLVM20 LLVM21
 //@ [LLVM21] min-llvm-version: 21
@@ -91,18 +90,23 @@ pub enum Mid {
 pub fn mid_bool_eq_discr(a: Mid, b: Mid) -> bool {
     // CHECK-LABEL: @mid_bool_eq_discr(
 
-    // CHECK: %[[A_REL_DISCR:.+]] = add nsw i8 %a, -2
-    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i8 %a, 1
-    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %[[A_REL_DISCR]], 1
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, 3
     // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
-    // CHECK: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %[[A_REL_DISCR]], i8 1
+    // LLVM20: %[[A_REL_DISCR:.+]] = add nsw i8 %a, -2
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i8 %a, 1
+    // LLVM20: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %[[A_REL_DISCR]], i8 1
 
-    // CHECK: %[[B_REL_DISCR:.+]] = add nsw i8 %b, -2
-    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i8 %b, 1
-    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %[[B_REL_DISCR]], 1
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, 3
     // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
-    // CHECK: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %[[B_REL_DISCR]], i8 1
+    // LLVM20: %[[B_REL_DISCR:.+]] = add nsw i8 %b, -2
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i8 %b, 1
+    // LLVM20: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %[[B_REL_DISCR]], i8 1
 
+    // LLVM21: %[[A_MOD_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 3
+    // LLVM21: %[[B_MOD_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %b, i8 3
+
+    // LLVM20: %[[R:.+]] = icmp eq i8 %[[A_DISCR]], %[[B_DISCR]]
+    // LLVM21: %[[R:.+]] = icmp eq i8 %[[A_MOD_DISCR]], %[[B_MOD_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
@@ -111,19 +115,23 @@ pub fn mid_bool_eq_discr(a: Mid, b: Mid) -> bool {
 pub fn mid_ord_eq_discr(a: Mid, b: Mid) -> bool {
     // CHECK-LABEL: @mid_ord_eq_discr(
 
-    // CHECK: %[[A_REL_DISCR:.+]] = add nsw i8 %a, -2
-    // CHECK: %[[A_IS_NICHE:.+]] = icmp sgt i8 %a, 1
-    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %[[A_REL_DISCR]], 1
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, 3
     // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
-    // CHECK: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %[[A_REL_DISCR]], i8 1
+    // LLVM20: %[[A_REL_DISCR:.+]] = add nsw i8 %a, -2
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp sgt i8 %a, 1
+    // LLVM20: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %[[A_REL_DISCR]], i8 1
 
-    // CHECK: %[[B_REL_DISCR:.+]] = add nsw i8 %b, -2
-    // CHECK: %[[B_IS_NICHE:.+]] = icmp sgt i8 %b, 1
-    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %[[B_REL_DISCR]], 1
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, 3
     // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
-    // CHECK: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %[[B_REL_DISCR]], i8 1
+    // LLVM20: %[[B_REL_DISCR:.+]] = add nsw i8 %b, -2
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp sgt i8 %b, 1
+    // LLVM20: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %[[B_REL_DISCR]], i8 1
 
-    // CHECK: %[[R:.+]] = icmp eq i8 %[[A_DISCR]], %[[B_DISCR]]
+    // LLVM21: %[[A_MOD_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 3
+    // LLVM21: %[[B_MOD_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %b, i8 3
+
+    // LLVM20: %[[R:.+]] = icmp eq i8 %[[A_DISCR]], %[[B_DISCR]]
+    // LLVM21: %[[R:.+]] = icmp eq i8 %[[A_MOD_DISCR]], %[[B_MOD_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
@@ -140,16 +148,16 @@ pub fn mid_nz32_eq_discr(a: Mid>, b: Mid>) -> bool {
 pub fn mid_ac_eq_discr(a: Mid, b: Mid) -> bool {
     // CHECK-LABEL: @mid_ac_eq_discr(
 
-    // LLVM20: %[[A_REL_DISCR:.+]] = xor i8 %a, -128
-    // CHECK: %[[A_IS_NICHE:.+]] = icmp slt i8 %a, 0
     // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i8 %a, -127
     // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
+    // LLVM20: %[[A_REL_DISCR:.+]] = xor i8 %a, -128
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp slt i8 %a, 0
     // LLVM20: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %[[A_REL_DISCR]], i8 1
 
-    // LLVM20: %[[B_REL_DISCR:.+]] = xor i8 %b, -128
-    // CHECK: %[[B_IS_NICHE:.+]] = icmp slt i8 %b, 0
     // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i8 %b, -127
     // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
+    // LLVM20: %[[B_REL_DISCR:.+]] = xor i8 %b, -128
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp slt i8 %b, 0
     // LLVM20: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i8 %[[B_REL_DISCR]], i8 1
 
     // LLVM21: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i8 %a, i8 -127
@@ -166,21 +174,25 @@ pub fn mid_ac_eq_discr(a: Mid, b: Mid) -> bool {
 pub fn mid_giant_eq_discr(a: Mid, b: Mid) -> bool {
     // CHECK-LABEL: @mid_giant_eq_discr(
 
-    // CHECK: %[[A_TRUNC:.+]] = trunc nuw nsw i128 %a to i64
-    // CHECK: %[[A_REL_DISCR:.+]] = add nsw i64 %[[A_TRUNC]], -5
-    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i128 %a, 4
-    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i64 %[[A_REL_DISCR]], 1
+    // CHECK: %[[A_NOT_HOLE:.+]] = icmp ne i128 %a, 6
     // CHECK: tail call void @llvm.assume(i1 %[[A_NOT_HOLE]])
-    // CHECK: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i64 %[[A_REL_DISCR]], i64 1
+    // CHECK: %[[A_TRUNC:.+]] = trunc nuw nsw i128 %a to i64
+    // LLVM20: %[[A_REL_DISCR:.+]] = add nsw i64 %[[A_TRUNC]], -5
+    // CHECK: %[[A_IS_NICHE:.+]] = icmp samesign ugt i128 %a, 4
+    // LLVM20: %[[A_DISCR:.+]] = select i1 %[[A_IS_NICHE]], i64 %[[A_REL_DISCR]], i64 1
 
-    // CHECK: %[[B_TRUNC:.+]] = trunc nuw nsw i128 %b to i64
-    // CHECK: %[[B_REL_DISCR:.+]] = add nsw i64 %[[B_TRUNC]], -5
-    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i128 %b, 4
-    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i64 %[[B_REL_DISCR]], 1
+    // CHECK: %[[B_NOT_HOLE:.+]] = icmp ne i128 %b, 6
     // CHECK: tail call void @llvm.assume(i1 %[[B_NOT_HOLE]])
-    // CHECK: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i64 %[[B_REL_DISCR]], i64 1
+    // CHECK: %[[B_TRUNC:.+]] = trunc nuw nsw i128 %b to i64
+    // LLVM20: %[[B_REL_DISCR:.+]] = add nsw i64 %[[B_TRUNC]], -5
+    // CHECK: %[[B_IS_NICHE:.+]] = icmp samesign ugt i128 %b, 4
+    // LLVM20: %[[B_DISCR:.+]] = select i1 %[[B_IS_NICHE]], i64 %[[B_REL_DISCR]], i64 1
 
-    // CHECK: %[[R:.+]] = icmp eq i64 %[[A_DISCR]], %[[B_DISCR]]
+    // LLVM21: %[[A_MODIFIED_TAG:.+]] = select i1 %[[A_IS_NICHE]], i64 %[[A_TRUNC]], i64 6
+    // LLVM21: %[[B_MODIFIED_TAG:.+]] = select i1 %[[B_IS_NICHE]], i64 %[[B_TRUNC]], i64 6
+    // LLVM21: %[[R:.+]] = icmp eq i64 %[[A_MODIFIED_TAG]], %[[B_MODIFIED_TAG]]
+
+    // LLVM20: %[[R:.+]] = icmp eq i64 %[[A_DISCR]], %[[B_DISCR]]
     // CHECK: ret i1 %[[R]]
     discriminant_value(&a) == discriminant_value(&b)
 }
diff --git a/tests/codegen-llvm/enum/enum-match.rs b/tests/codegen-llvm/enum/enum-match.rs
index 57db44ec74e8..20e2006e3eb7 100644
--- a/tests/codegen-llvm/enum/enum-match.rs
+++ b/tests/codegen-llvm/enum/enum-match.rs
@@ -138,18 +138,18 @@ pub fn match3(e: Option<&u8>) -> i16 {
 
 #[derive(PartialEq)]
 pub enum MiddleNiche {
-    A,
-    B,
-    C(bool),
-    D,
-    E,
+    A,       // tag 2
+    B,       // tag 3
+    C(bool), // untagged
+    D,       // tag 5
+    E,       // tag 6
 }
 
 // CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 -?[0-9]+, -?[0-9]+\))?}} i8 @match4(i8{{.+}}%0)
 // CHECK-NEXT: start:
-// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
-// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %[[REL_VAR]], 2
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %0, 4
 // CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
+// CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
 // CHECK-NEXT: %[[NOT_NICHE:.+]] = icmp{{( samesign)?}} ult i8 %0, 2
 // CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[NOT_NICHE]], i8 2, i8 %[[REL_VAR]]
 // CHECK-NEXT: switch i8 %[[DISCR]]
@@ -443,19 +443,19 @@ pub enum HugeVariantIndex {
     V255(Never),
     V256(Never),
 
-    Possible257,
-    Bool258(bool),
-    Possible259,
+    Possible257,   // tag 2
+    Bool258(bool), // untagged
+    Possible259,   // tag 4
 }
 
 // CHECK-LABEL: define{{( dso_local)?}} noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @match5(i8{{.+}}%0)
 // CHECK-NEXT: start:
+// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i8 %0, 3
+// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
 // CHECK-NEXT: %[[REL_VAR:.+]] = add{{( nsw)?}} i8 %0, -2
 // CHECK-NEXT: %[[REL_VAR_WIDE:.+]] = zext i8 %[[REL_VAR]] to i64
 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp{{( samesign)?}} ugt i8 %0, 1
 // CHECK-NEXT: %[[NICHE_DISCR:.+]] = add nuw nsw i64 %[[REL_VAR_WIDE]], 257
-// CHECK-NEXT: %[[NOT_IMPOSSIBLE:.+]] = icmp ne i64 %[[NICHE_DISCR]], 258
-// CHECK-NEXT: call void @llvm.assume(i1 %[[NOT_IMPOSSIBLE]])
 // CHECK-NEXT: %[[DISCR:.+]] = select i1 %[[IS_NICHE]], i64 %[[NICHE_DISCR]], i64 258
 // CHECK-NEXT: switch i64 %[[DISCR]],
 // CHECK-NEXT:   i64 257,
@@ -739,7 +739,7 @@ pub enum Tricky {
 
 const _: () = assert!(std::intrinsics::discriminant_value(&Tricky::V100) == 100);
 
-// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @discriminant6(i8 noundef %e)
+// CHECK-LABEL: define noundef{{( range\(i8 [0-9]+, [0-9]+\))?}} i8 @discriminant6(i8 noundef{{( zeroext)?}} %e)
 // CHECK-NEXT: start:
 // CHECK-NEXT: %[[REL_VAR:.+]] = add i8 %e, -66
 // CHECK-NEXT: %[[IS_NICHE:.+]] = icmp ult i8 %[[REL_VAR]], -56
diff --git a/tests/codegen-llvm/enum/enum-transparent-extract.rs b/tests/codegen-llvm/enum/enum-transparent-extract.rs
index c5efb8d472b0..1435e6ec8022 100644
--- a/tests/codegen-llvm/enum/enum-transparent-extract.rs
+++ b/tests/codegen-llvm/enum/enum-transparent-extract.rs
@@ -9,7 +9,7 @@ pub enum Never {}
 
 #[no_mangle]
 pub fn make_unmake_result_never(x: i32) -> i32 {
-    // CHECK-LABEL: define i32 @make_unmake_result_never(i32 %x)
+    // CHECK-LABEL: define i32 @make_unmake_result_never(i32{{( signext)?}} %x)
     // CHECK: start:
     // CHECK-NEXT: ret i32 %x
 
diff --git a/tests/codegen-llvm/function-arguments.rs b/tests/codegen-llvm/function-arguments.rs
index c8cd8526ae51..aaa1d57592a8 100644
--- a/tests/codegen-llvm/function-arguments.rs
+++ b/tests/codegen-llvm/function-arguments.rs
@@ -80,7 +80,7 @@ pub fn option_nonzero_int(x: Option>) -> Option> {
     x
 }
 
-// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1)
+// CHECK: @readonly_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1)
 // FIXME #25759 This should also have `nocapture`
 #[no_mangle]
 pub fn readonly_borrow(_: &i32) {}
@@ -91,12 +91,12 @@ pub fn readonly_borrow_ret() -> &'static i32 {
     loop {}
 }
 
-// CHECK: @static_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1)
+// CHECK: @static_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1)
 // static borrow may be captured
 #[no_mangle]
 pub fn static_borrow(_: &'static i32) {}
 
-// CHECK: @named_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1)
+// CHECK: @named_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1)
 // borrow with named lifetime may be captured
 #[no_mangle]
 pub fn named_borrow<'r>(_: &'r i32) {}
@@ -129,21 +129,21 @@ pub fn mutable_borrow_ret() -> &'static mut i32 {
 // .
 pub fn mutable_notunpin_borrow(_: &mut NotUnpin) {}
 
-// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4 dereferenceable(4) %_1)
+// CHECK: @notunpin_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(4) %_1)
 // But `&NotUnpin` behaves perfectly normal.
 #[no_mangle]
 pub fn notunpin_borrow(_: &NotUnpin) {}
 
-// CHECK: @indirect_struct(ptr noalias{{( nocapture)?}} noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1)
+// CHECK: @indirect_struct(ptr{{( dead_on_return)?}} noalias noundef readonly align 4{{( captures\(none\))?}} dereferenceable(32) %_1)
 #[no_mangle]
 pub fn indirect_struct(_: S) {}
 
-// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4 dereferenceable(32) %_1)
+// CHECK: @borrowed_struct(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable(32) %_1)
 // FIXME #25759 This should also have `nocapture`
 #[no_mangle]
 pub fn borrowed_struct(_: &S) {}
 
-// CHECK: @option_borrow(ptr noalias noundef readonly align 4 dereferenceable_or_null(4) %_x)
+// CHECK: @option_borrow(ptr noalias noundef readonly align 4{{( captures\(address, read_provenance\))?}} dereferenceable_or_null(4) %_x)
 #[no_mangle]
 pub fn option_borrow(_x: Option<&i32>) {}
 
@@ -185,7 +185,7 @@ pub fn _box(x: Box) -> Box {
 // With a custom allocator, it should *not* have `noalias`. (See
 //  for why.) The second argument is the allocator,
 // which is a reference here that still carries `noalias` as usual.
-// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1 %x.1)
+// CHECK: @_box_custom(ptr noundef nonnull align 4 %x.0, ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %x.1)
 #[no_mangle]
 pub fn _box_custom(x: Box) {
     drop(x)
@@ -197,7 +197,7 @@ pub fn notunpin_box(x: Box) -> Box {
     x
 }
 
-// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias{{( nocapture)?}} noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(none\))?}} dereferenceable(32){{( %_0)?}})
+// CHECK: @struct_return(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([32 x i8]) align 4{{( captures\(none\))?}} dereferenceable(32){{( %_0)?}})
 #[no_mangle]
 pub fn struct_return() -> S {
     S { _field: [0, 0, 0, 0, 0, 0, 0, 0] }
@@ -208,7 +208,7 @@ pub fn struct_return() -> S {
 #[no_mangle]
 pub fn helper(_: usize) {}
 
-// CHECK: @slice(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1)
+// CHECK: @slice(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1)
 // FIXME #25759 This should also have `nocapture`
 #[no_mangle]
 pub fn slice(_: &[u8]) {}
@@ -227,7 +227,7 @@ pub fn unsafe_slice(_: &[UnsafeInner]) {}
 #[no_mangle]
 pub fn raw_slice(_: *const [u8]) {}
 
-// CHECK: @str(ptr noalias noundef nonnull readonly align 1 %_1.0, [[USIZE]] noundef %_1.1)
+// CHECK: @str(ptr noalias noundef nonnull readonly align 1{{( captures\(address, read_provenance\))?}} %_1.0, [[USIZE]] noundef %_1.1)
 // FIXME #25759 This should also have `nocapture`
 #[no_mangle]
 pub fn str(_: &[u8]) {}
@@ -259,7 +259,7 @@ pub fn trait_option(x: Option>) -> Option &[u16] {
     x
diff --git a/tests/codegen-llvm/global-allocator-attributes.rs b/tests/codegen-llvm/global-allocator-attributes.rs
new file mode 100644
index 000000000000..472ca7720750
--- /dev/null
+++ b/tests/codegen-llvm/global-allocator-attributes.rs
@@ -0,0 +1,41 @@
+//@ compile-flags: -C opt-level=3
+#![crate_type = "lib"]
+
+mod foobar {
+    use std::alloc::{GlobalAlloc, Layout};
+
+    struct Allocator;
+
+    unsafe impl GlobalAlloc for Allocator {
+        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+            // CHECK-LABEL: ; __rustc::__rust_alloc
+            // CHECK-NEXT: ; Function Attrs: {{.*}}allockind("alloc,uninitialized,aligned") allocsize(0){{.*}}
+            // CHECK-NEXT: define{{.*}} noalias{{.*}} ptr @{{.*}}__rust_alloc(i[[SIZE:[0-9]+]] {{.*}}%size, i[[SIZE]] allocalign{{.*}} %align)
+            panic!()
+        }
+
+        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
+            // CHECK-LABEL: ; __rustc::__rust_dealloc
+            // CHECK-NEXT: ; Function Attrs: {{.*}}allockind("free"){{.*}}
+            // CHECK-NEXT: define{{.*}} void @{{.*}}__rust_dealloc(ptr allocptr{{.*}} %ptr, i[[SIZE]] {{.*}} %size, i[[SIZE]] {{.*}} %align)
+            panic!()
+        }
+
+        unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+            // CHECK-LABEL: ; __rustc::__rust_realloc
+            // CHECK-NEXT: ; Function Attrs: {{.*}}allockind("realloc,aligned") allocsize(3){{.*}}
+            // CHECK-NEXT: define{{.*}} noalias{{.*}} ptr @{{.*}}__rust_realloc(ptr allocptr{{.*}} %ptr, i[[SIZE]] {{.*}} %size, i[[SIZE]] allocalign{{.*}} %align, i[[SIZE]] {{.*}} %new_size)
+            panic!()
+        }
+
+        unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
+            // CHECK-LABEL: ; __rustc::__rust_alloc_zeroed
+            // CHECK-NEXT: ; Function Attrs: {{.*}}allockind("alloc,zeroed,aligned") allocsize(0){{.*}}
+            // CHECK-NEXT: define{{.*}} noalias{{.*}} ptr @{{.*}}__rust_alloc_zeroed(i[[SIZE]] {{.*}} %size, i[[SIZE]] allocalign{{.*}} %align)
+            panic!()
+        }
+    }
+
+    #[global_allocator]
+    static GLOBAL: Allocator = Allocator;
+}
diff --git a/tests/codegen-llvm/gpu-kernel-abi.rs b/tests/codegen-llvm/gpu-kernel-abi.rs
index 8ac376d93381..d5a357ef6552 100644
--- a/tests/codegen-llvm/gpu-kernel-abi.rs
+++ b/tests/codegen-llvm/gpu-kernel-abi.rs
@@ -1,7 +1,9 @@
 // Checks that the gpu-kernel calling convention correctly translates to LLVM calling conventions.
 
 //@ add-core-stubs
-//@ revisions: nvptx
+//@ revisions: amdgpu nvptx
+//@ [amdgpu] compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900
+//@ [amdgpu] needs-llvm-components: amdgpu
 //@ [nvptx] compile-flags: --crate-type=rlib --target=nvptx64-nvidia-cuda
 //@ [nvptx] needs-llvm-components: nvptx
 #![feature(no_core, lang_items, abi_gpu_kernel)]
@@ -10,6 +12,7 @@
 extern crate minicore;
 use minicore::*;
 
+// amdgpu: define amdgpu_kernel void @fun(i32
 // nvptx: define ptx_kernel void @fun(i32
 #[no_mangle]
 pub extern "gpu-kernel" fn fun(_: i32) {}
diff --git a/tests/codegen-llvm/gpu_offload/gpu_host.rs b/tests/codegen-llvm/gpu_offload/gpu_host.rs
index 513e27426bc0..fac4054d1b7f 100644
--- a/tests/codegen-llvm/gpu_offload/gpu_host.rs
+++ b/tests/codegen-llvm/gpu_offload/gpu_host.rs
@@ -21,16 +21,15 @@ fn main() {
 }
 
 // CHECK: %struct.__tgt_offload_entry = type { i64, i16, i16, i32, ptr, ptr, i64, i64, ptr }
-// CHECK: %struct.__tgt_kernel_arguments = type { i32, i32, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, [3 x i32], [3 x i32], i32 }
 // CHECK: %struct.ident_t = type { i32, i32, i32, i32, ptr }
 // CHECK: %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr }
+// CHECK: %struct.__tgt_kernel_arguments = type { i32, i32, ptr, ptr, ptr, ptr, ptr, ptr, i64, i64, [3 x i32], [3 x i32], i32 }
 
 // CHECK: @.offload_sizes.1 = private unnamed_addr constant [1 x i64] [i64 1024]
-// CHECK: @.offload_maptypes.1 = private unnamed_addr constant [1 x i64] [i64 3]
+// CHECK: @.offload_maptypes.1 = private unnamed_addr constant [1 x i64] [i64 35]
 // CHECK: @.kernel_1.region_id = weak unnamed_addr constant i8 0
 // CHECK: @.offloading.entry_name.1 = internal unnamed_addr constant [9 x i8] c"kernel_1\00", section ".llvm.rodata.offloading", align 1
-// CHECK: @.offloading.entry.kernel_1 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.kernel_1.region_id, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section ".omp_offloading_entries", align 1
-// CHECK: @my_struct_global2 = external global %struct.__tgt_kernel_arguments
+// CHECK: @.offloading.entry.kernel_1 = weak constant %struct.__tgt_offload_entry { i64 0, i16 1, i16 1, i32 0, ptr @.kernel_1.region_id, ptr @.offloading.entry_name.1, i64 0, i64 0, ptr null }, section "llvm_offload_entries", align 8
 // CHECK: @0 = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
 // CHECK: @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8
 
@@ -43,34 +42,61 @@ fn main() {
 // CHECK-NEXT:   %.offload_baseptrs = alloca [1 x ptr], align 8
 // CHECK-NEXT:   %.offload_ptrs = alloca [1 x ptr], align 8
 // CHECK-NEXT:   %.offload_sizes = alloca [1 x i64], align 8
-// CHECK-NEXT:   %x.addr = alloca ptr, align 8
-// CHECK-NEXT:   store ptr %x, ptr %x.addr, align 8
-// CHECK-NEXT:   %1 = load ptr, ptr %x.addr, align 8
-// CHECK-NEXT:   %2 = getelementptr inbounds float, ptr %1, i32 0
+// CHECK-NEXT:   %kernel_args = alloca %struct.__tgt_kernel_arguments, align 8
 // CHECK:        call void @llvm.memset.p0.i64(ptr align 8 %EmptyDesc, i8 0, i64 32, i1 false)
+// CHECK-NEXT:   %1 = getelementptr inbounds float, ptr %x, i32 0
 // CHECK-NEXT:   call void @__tgt_register_lib(ptr %EmptyDesc)
 // CHECK-NEXT:   call void @__tgt_init_all_rtls()
-// CHECK-NEXT:   %3 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK-NEXT:   %2 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK-NEXT:   store ptr %x, ptr %2, align 8
+// CHECK-NEXT:   %3 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
 // CHECK-NEXT:   store ptr %1, ptr %3, align 8
-// CHECK-NEXT:   %4 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-// CHECK-NEXT:   store ptr %2, ptr %4, align 8
-// CHECK-NEXT:   %5 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
-// CHECK-NEXT:   store i64 1024, ptr %5, align 8
-// CHECK-NEXT:   %6 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
-// CHECK-NEXT:   %7 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-// CHECK-NEXT:   %8 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
-// CHECK-NEXT:   call void @__tgt_target_data_begin_mapper(ptr @1, i64 -1, i32 1, ptr %6, ptr %7, ptr %8, ptr @.offload_maptypes.1, ptr null, ptr null)
-// CHECK-NEXT:   call void @kernel_1(ptr noalias noundef nonnull align 4 dereferenceable(1024) %x)
-// CHECK-NEXT:   %9 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
-// CHECK-NEXT:   %10 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
-// CHECK-NEXT:   %11 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
-// CHECK-NEXT:   call void @__tgt_target_data_end_mapper(ptr @1, i64 -1, i32 1, ptr %9, ptr %10, ptr %11, ptr @.offload_maptypes.1, ptr null, ptr null)
+// CHECK-NEXT:   %4 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
+// CHECK-NEXT:   store i64 1024, ptr %4, align 8
+// CHECK-NEXT:   %5 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK-NEXT:   %6 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// CHECK-NEXT:   %7 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
+// CHECK-NEXT:   call void @__tgt_target_data_begin_mapper(ptr @1, i64 -1, i32 1, ptr %5, ptr %6, ptr %7, ptr @.offload_maptypes.1, ptr null, ptr null)
+// CHECK-NEXT:   %8 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 0
+// CHECK-NEXT:   store i32 3, ptr %8, align 4
+// CHECK-NEXT:   %9 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 1
+// CHECK-NEXT:   store i32 1, ptr %9, align 4
+// CHECK-NEXT:   %10 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 2
+// CHECK-NEXT:   store ptr %5, ptr %10, align 8
+// CHECK-NEXT:   %11 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 3
+// CHECK-NEXT:   store ptr %6, ptr %11, align 8
+// CHECK-NEXT:   %12 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 4
+// CHECK-NEXT:   store ptr %7, ptr %12, align 8
+// CHECK-NEXT:   %13 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 5
+// CHECK-NEXT:   store ptr @.offload_maptypes.1, ptr %13, align 8
+// CHECK-NEXT:   %14 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 6
+// CHECK-NEXT:   store ptr null, ptr %14, align 8
+// CHECK-NEXT:   %15 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 7
+// CHECK-NEXT:   store ptr null, ptr %15, align 8
+// CHECK-NEXT:   %16 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 8
+// CHECK-NEXT:   store i64 0, ptr %16, align 8
+// CHECK-NEXT:   %17 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 9
+// CHECK-NEXT:   store i64 0, ptr %17, align 8
+// CHECK-NEXT:   %18 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 10
+// CHECK-NEXT:   store [3 x i32] [i32 2097152, i32 0, i32 0], ptr %18, align 4
+// CHECK-NEXT:   %19 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 11
+// CHECK-NEXT:   store [3 x i32] [i32 256, i32 0, i32 0], ptr %19, align 4
+// CHECK-NEXT:   %20 = getelementptr inbounds %struct.__tgt_kernel_arguments, ptr %kernel_args, i32 0, i32 12
+// CHECK-NEXT:   store i32 0, ptr %20, align 4
+// CHECK-NEXT:   %21 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args)
+// CHECK-NEXT:   %22 = getelementptr inbounds [1 x ptr], ptr %.offload_baseptrs, i32 0, i32 0
+// CHECK-NEXT:   %23 = getelementptr inbounds [1 x ptr], ptr %.offload_ptrs, i32 0, i32 0
+// CHECK-NEXT:   %24 = getelementptr inbounds [1 x i64], ptr %.offload_sizes, i32 0, i32 0
+// CHECK-NEXT:   call void @__tgt_target_data_end_mapper(ptr @1, i64 -1, i32 1, ptr %22, ptr %23, ptr %24, ptr @.offload_maptypes.1, ptr null, ptr null)
 // CHECK-NEXT:   call void @__tgt_unregister_lib(ptr %EmptyDesc)
 // CHECK:        store ptr %x, ptr %0, align 8
 // CHECK-NEXT:   call void asm sideeffect "", "r,~{memory}"(ptr nonnull %0)
 // CHECK:        ret void
 // CHECK-NEXT: }
 
+// CHECK: Function Attrs: nounwind
+// CHECK: declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr)
+
 #[unsafe(no_mangle)]
 #[inline(never)]
 pub fn kernel_1(x: &mut [f32; 256]) {
diff --git a/tests/codegen-llvm/indirect-branch-cs-prefix.rs b/tests/codegen-llvm/indirect-branch-cs-prefix.rs
new file mode 100644
index 000000000000..df25008d5f09
--- /dev/null
+++ b/tests/codegen-llvm/indirect-branch-cs-prefix.rs
@@ -0,0 +1,18 @@
+// Test that the `indirect_branch_cs_prefix` module attribute is (not)
+// emitted when the `-Zindirect-branch-cs-prefix` flag is (not) set.
+
+//@ add-core-stubs
+//@ revisions: unset set
+//@ needs-llvm-components: x86
+//@ compile-flags: --target x86_64-unknown-linux-gnu
+//@ [set] compile-flags: -Zindirect-branch-cs-prefix
+
+#![crate_type = "lib"]
+#![feature(no_core, lang_items)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// unset-NOT: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1}
+// set: !{{[0-9]+}} = !{i32 4, !"indirect_branch_cs_prefix", i32 1}
diff --git a/tests/codegen-llvm/int-ptr-int-enum-miscompile.rs b/tests/codegen-llvm/int-ptr-int-enum-miscompile.rs
new file mode 100644
index 000000000000..299c3bf8847b
--- /dev/null
+++ b/tests/codegen-llvm/int-ptr-int-enum-miscompile.rs
@@ -0,0 +1,22 @@
+// This is a regression test for https://github.com/rust-lang/rust/issues/147265.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn mk_result(a: usize) -> Result {
+    // CHECK-LABEL: @mk_result
+    // CHECK-NOT: unreachable
+    // CHECK: load i8,
+    // CHECK-NOT: unreachable
+    match g(a) {
+        Ok(b) => Ok(unsafe { *(b as *const u8) }),
+        Err(c) => Err(c),
+    }
+}
+
+#[cold]
+fn g(a: usize) -> Result {
+    Ok(a)
+}
diff --git a/tests/codegen-llvm/integer-cmp.rs b/tests/codegen-llvm/integer-cmp.rs
index 812fa8e4a424..2233a575f8e7 100644
--- a/tests/codegen-llvm/integer-cmp.rs
+++ b/tests/codegen-llvm/integer-cmp.rs
@@ -1,9 +1,6 @@
 // This is test for more optimal Ord implementation for integers.
 // See  for more info.
 
-//@ revisions: llvm-pre-20 llvm-20
-//@ [llvm-20] min-llvm-version: 20
-//@ [llvm-pre-20] max-llvm-major-version: 19
 //@ compile-flags: -C opt-level=3 -Zmerge-functions=disabled
 
 #![crate_type = "lib"]
@@ -13,50 +10,29 @@ use std::cmp::Ordering;
 // CHECK-LABEL: @cmp_signed
 #[no_mangle]
 pub fn cmp_signed(a: i64, b: i64) -> Ordering {
-    // llvm-20: call{{.*}} i8 @llvm.scmp.i8.i64
-    // llvm-pre-20: icmp slt
-    // llvm-pre-20: icmp ne
-    // llvm-pre-20: zext i1
-    // llvm-pre-20: select i1
+    // CHECK: call{{.*}} i8 @llvm.scmp.i8.i64
     a.cmp(&b)
 }
 
 // CHECK-LABEL: @cmp_unsigned
 #[no_mangle]
 pub fn cmp_unsigned(a: u32, b: u32) -> Ordering {
-    // llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
-    // llvm-pre-20: icmp ult
-    // llvm-pre-20: icmp ne
-    // llvm-pre-20: zext i1
-    // llvm-pre-20: select i1
+    // CHECK: call{{.*}} i8 @llvm.ucmp.i8.i32
     a.cmp(&b)
 }
 
 // CHECK-LABEL: @cmp_char
 #[no_mangle]
 pub fn cmp_char(a: char, b: char) -> Ordering {
-    // llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
-    // llvm-pre-20: icmp ult
-    // llvm-pre-20: icmp ne
-    // llvm-pre-20: zext i1
-    // llvm-pre-20: select i1
+    // CHECK: call{{.*}} i8 @llvm.ucmp.i8.i32
     a.cmp(&b)
 }
 
 // CHECK-LABEL: @cmp_tuple
 #[no_mangle]
 pub fn cmp_tuple(a: (i16, u16), b: (i16, u16)) -> Ordering {
-    // llvm-20-DAG: call{{.*}} i8 @llvm.ucmp.i8.i16
-    // llvm-20-DAG: call{{.*}} i8 @llvm.scmp.i8.i16
-    // llvm-20: ret i8
-    // llvm-pre-20: icmp slt
-    // llvm-pre-20: icmp ne
-    // llvm-pre-20: zext i1
-    // llvm-pre-20: select i1
-    // llvm-pre-20: icmp ult
-    // llvm-pre-20: icmp ne
-    // llvm-pre-20: zext i1
-    // llvm-pre-20: select i1
-    // llvm-pre-20: select i1
+    // CHECK-DAG: call{{.*}} i8 @llvm.ucmp.i8.i16
+    // CHECK-DAG: call{{.*}} i8 @llvm.scmp.i8.i16
+    // CHECK: ret i8
     a.cmp(&b)
 }
diff --git a/tests/codegen-llvm/integer-overflow.rs b/tests/codegen-llvm/integer-overflow.rs
index 80362247a86f..df7845be06d1 100644
--- a/tests/codegen-llvm/integer-overflow.rs
+++ b/tests/codegen-llvm/integer-overflow.rs
@@ -10,7 +10,7 @@ pub struct S1<'a> {
 // CHECK-LABEL: @slice_no_index_order
 #[no_mangle]
 pub fn slice_no_index_order<'a>(s: &'a mut S1, n: usize) -> &'a [u8] {
-    // CHECK-NOT: slice_index_order_fail
+    // CHECK-COUNT-1: slice_index_fail
     let d = &s.data[s.position..s.position + n];
     s.position += n;
     return d;
@@ -19,6 +19,6 @@ pub fn slice_no_index_order<'a>(s: &'a mut S1, n: usize) -> &'a [u8] {
 // CHECK-LABEL: @test_check
 #[no_mangle]
 pub fn test_check<'a>(s: &'a mut S1, x: usize, y: usize) -> &'a [u8] {
-    // CHECK: slice_index_order_fail
+    // CHECK-COUNT-1: slice_index_fail
     &s.data[x..y]
 }
diff --git a/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs b/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs
index 4bec579831dc..255f20e6ff64 100644
--- a/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs
+++ b/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs
@@ -7,7 +7,5 @@ use std::intrinsics::sqrtf32;
 // CHECK: @llvm.sqrt.f32(float) #{{[0-9]*}}
 
 fn main() {
-    unsafe {
-        sqrtf32(0.0f32);
-    }
+    sqrtf32(0.0f32);
 }
diff --git a/tests/codegen-llvm/intrinsics/prefetch.rs b/tests/codegen-llvm/intrinsics/prefetch.rs
index 3f9f21c85cbf..41877872019e 100644
--- a/tests/codegen-llvm/intrinsics/prefetch.rs
+++ b/tests/codegen-llvm/intrinsics/prefetch.rs
@@ -9,56 +9,48 @@ use std::intrinsics::{
 
 #[no_mangle]
 pub fn check_prefetch_read_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
-        prefetch_read_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
-        prefetch_read_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
-        prefetch_read_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
-        prefetch_read_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 1)
+    prefetch_read_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 1)
+    prefetch_read_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 1)
+    prefetch_read_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 1)
+    prefetch_read_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_data(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
-        prefetch_write_data(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
-        prefetch_write_data(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
-        prefetch_write_data(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
-        prefetch_write_data(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 1)
+    prefetch_write_data::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 1)
+    prefetch_write_data::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 1)
+    prefetch_write_data::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 1)
+    prefetch_write_data::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_read_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
-        prefetch_read_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 0, i32 0)
+    prefetch_read_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 1, i32 0)
+    prefetch_read_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 2, i32 0)
+    prefetch_read_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 0, i32 3, i32 0)
+    prefetch_read_instruction::<_, 3>(data.as_ptr());
 }
 
 #[no_mangle]
 pub fn check_prefetch_write_instruction(data: &[i8]) {
-    unsafe {
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 0);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 1);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 2);
-        // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
-        prefetch_write_instruction(data.as_ptr(), 3);
-    }
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 0, i32 0)
+    prefetch_write_instruction::<_, 0>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 1, i32 0)
+    prefetch_write_instruction::<_, 1>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 2, i32 0)
+    prefetch_write_instruction::<_, 2>(data.as_ptr());
+    // CHECK: call void @llvm.prefetch{{.*}}({{.*}}, i32 1, i32 3, i32 0)
+    prefetch_write_instruction::<_, 3>(data.as_ptr());
 }
diff --git a/tests/codegen-llvm/intrinsics/three_way_compare.rs b/tests/codegen-llvm/intrinsics/three_way_compare.rs
index 95fcb636f7ca..89bf69561e9a 100644
--- a/tests/codegen-llvm/intrinsics/three_way_compare.rs
+++ b/tests/codegen-llvm/intrinsics/three_way_compare.rs
@@ -2,7 +2,6 @@
 //@ [DEBUG] compile-flags: -C opt-level=0
 //@ [OPTIM] compile-flags: -C opt-level=3
 //@ compile-flags: -C no-prepopulate-passes
-//@ min-llvm-version: 20
 
 #![crate_type = "lib"]
 #![feature(core_intrinsics)]
diff --git a/tests/codegen-llvm/intrinsics/transmute-niched.rs b/tests/codegen-llvm/intrinsics/transmute-niched.rs
index a886d9eee590..8c647655f65b 100644
--- a/tests/codegen-llvm/intrinsics/transmute-niched.rs
+++ b/tests/codegen-llvm/intrinsics/transmute-niched.rs
@@ -249,7 +249,7 @@ pub unsafe fn check_four_or_eight_to_nonnull(x: FourOrEight) -> NonNull {
     // OPT: call void @llvm.assume(i1 %1)
     // CHECK-NOT: icmp
     // CHECK-NOT: assume
-    // CHECK: %[[RET:.+]] = getelementptr i8, ptr null, i64 %x
+    // CHECK: %[[RET:.+]] = inttoptr i64 %x to ptr
     // CHECK-NEXT: ret ptr %[[RET]]
 
     transmute(x)
diff --git a/tests/codegen-llvm/intrinsics/transmute-simd.rs b/tests/codegen-llvm/intrinsics/transmute-simd.rs
index e34b27e13335..92af16945abf 100644
--- a/tests/codegen-llvm/intrinsics/transmute-simd.rs
+++ b/tests/codegen-llvm/intrinsics/transmute-simd.rs
@@ -97,10 +97,10 @@ pub extern "C" fn float_ptr_same_lanes(v: f64x2) -> PtrX2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <2 x double> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x ptr> %[[RET]]
     unsafe { transmute(v) }
 }
@@ -111,10 +111,10 @@ pub extern "C" fn ptr_float_same_lanes(v: PtrX2) -> f64x2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <2 x ptr> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x double>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x double> %[[RET]]
     unsafe { transmute(v) }
 }
@@ -125,10 +125,10 @@ pub extern "C" fn int_ptr_same_lanes(v: i64x2) -> PtrX2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <2 x i64> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x ptr> %[[RET]]
     unsafe { transmute(v) }
 }
@@ -139,10 +139,10 @@ pub extern "C" fn ptr_int_same_lanes(v: PtrX2) -> i64x2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <2 x ptr> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x i64>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x i64> %[[RET]]
     unsafe { transmute(v) }
 }
@@ -153,10 +153,10 @@ pub extern "C" fn float_ptr_widen(v: f32x4) -> PtrX2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <4 x float> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x ptr> %[[RET]]
     unsafe { transmute(v) }
 }
@@ -167,10 +167,10 @@ pub extern "C" fn int_ptr_widen(v: i32x4) -> PtrX2 {
     // CHECK-NOT: alloca
     // CHECK: %[[TEMP:.+]] = alloca [16 x i8]
     // CHECK-NOT: alloca
-    // CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: store <4 x i32> %v, ptr %[[TEMP]]
     // CHECK: %[[RET:.+]] = load <2 x ptr>, ptr %[[TEMP]]
-    // CHECK: call void @llvm.lifetime.end.p0(i64 16, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 16, )?}}ptr %[[TEMP]])
     // CHECK: ret <2 x ptr> %[[RET]]
     unsafe { transmute(v) }
 }
diff --git a/tests/codegen-llvm/intrinsics/transmute.rs b/tests/codegen-llvm/intrinsics/transmute.rs
index 91cff38773d7..b86c6df3267e 100644
--- a/tests/codegen-llvm/intrinsics/transmute.rs
+++ b/tests/codegen-llvm/intrinsics/transmute.rs
@@ -192,12 +192,12 @@ pub unsafe fn check_byte_from_bool(x: bool) -> u8 {
 #[no_mangle]
 pub unsafe fn check_to_pair(x: u64) -> Option {
     // CHECK: %[[TEMP:.+]] = alloca [8 x i8], align 8
-    // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr %[[TEMP]])
     // CHECK: store i64 %x, ptr %[[TEMP]], align 8
     // CHECK: %[[PAIR0:.+]] = load i32, ptr %[[TEMP]], align 8
     // CHECK: %[[PAIR1P:.+]] = getelementptr inbounds i8, ptr %[[TEMP]], i64 4
     // CHECK: %[[PAIR1:.+]] = load i32, ptr %[[PAIR1P]], align 4
-    // CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr %[[TEMP]])
     // CHECK: insertvalue {{.+}}, i32 %[[PAIR0]], 0
     // CHECK: insertvalue {{.+}}, i32 %[[PAIR1]], 1
     transmute(x)
@@ -207,12 +207,12 @@ pub unsafe fn check_to_pair(x: u64) -> Option {
 #[no_mangle]
 pub unsafe fn check_from_pair(x: Option) -> u64 {
     // CHECK: %[[TEMP:.+]] = alloca [8 x i8], align 8
-    // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.start.p0({{(i64 8, )?}}ptr %[[TEMP]])
     // CHECK: store i32 %x.0, ptr %[[TEMP]], align 8
     // CHECK: %[[PAIR1P:.+]] = getelementptr inbounds i8, ptr %[[TEMP]], i64 4
     // CHECK: store i32 %x.1, ptr %[[PAIR1P]], align 4
     // CHECK: %[[R:.+]] = load i64, ptr %[[TEMP]], align 8
-    // CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TEMP]])
+    // CHECK: call void @llvm.lifetime.end.p0({{(i64 8, )?}}ptr %[[TEMP]])
     // CHECK: ret i64 %[[R]]
     transmute(x)
 }
@@ -303,7 +303,7 @@ pub unsafe fn check_pair_with_bool(x: (u8, bool)) -> (bool, i8) {
 pub unsafe fn check_float_to_pointer(x: f64) -> *const () {
     // CHECK-NOT: alloca
     // CHECK: %0 = bitcast double %x to i64
-    // CHECK: %_0 = getelementptr i8, ptr null, i64 %0
+    // CHECK: %_0 = inttoptr i64 %0 to ptr
     // CHECK: ret ptr %_0
     transmute(x)
 }
@@ -378,7 +378,7 @@ pub unsafe fn check_issue_110005(x: (usize, bool)) -> Option> {
 // CHECK-LABEL: @check_pair_to_dst_ref(
 #[no_mangle]
 pub unsafe fn check_pair_to_dst_ref<'a>(x: (usize, usize)) -> &'a [u8] {
-    // CHECK: %_0.0 = getelementptr i8, ptr null, i64 %x.0
+    // CHECK: %_0.0 = inttoptr i64 %x.0 to ptr
     // CHECK: %0 = icmp ne ptr %_0.0, null
     // CHECK: call void @llvm.assume(i1 %0)
     // CHECK: %1 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0
diff --git a/tests/codegen-llvm/issues/and-masked-comparison-131162.rs b/tests/codegen-llvm/issues/and-masked-comparison-131162.rs
new file mode 100644
index 000000000000..fc4b0341a31b
--- /dev/null
+++ b/tests/codegen-llvm/issues/and-masked-comparison-131162.rs
@@ -0,0 +1,16 @@
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @issue_131162
+#[no_mangle]
+pub fn issue_131162(a1: usize, a2: usize) -> bool {
+    const MASK: usize = 1;
+
+    // CHECK-NOT: xor
+    // CHECK-NOT: trunc
+    // CHECK-NOT: and i1
+    // CHECK: icmp
+    // CHECK-NEXT: ret
+    (a1 & !MASK) == (a2 & !MASK) && (a1 & MASK) == (a2 & MASK)
+}
diff --git a/tests/codegen-llvm/issues/assert-for-loop-bounds-check-71997.rs b/tests/codegen-llvm/issues/assert-for-loop-bounds-check-71997.rs
new file mode 100644
index 000000000000..a0c64d607a8f
--- /dev/null
+++ b/tests/codegen-llvm/issues/assert-for-loop-bounds-check-71997.rs
@@ -0,0 +1,18 @@
+// Tests that there's no bounds check within for-loop after asserting that
+// the range start and end are within bounds.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @no_bounds_check_after_assert
+#[no_mangle]
+fn no_bounds_check_after_assert(slice: &[u64], start: usize, end: usize) -> u64 {
+    // CHECK-NOT: panic_bounds_check
+    let mut total = 0;
+    assert!(start < end && start < slice.len() && end <= slice.len());
+    for i in start..end {
+        total += slice[i];
+    }
+    total
+}
diff --git a/tests/codegen-llvm/issues/cows-dont-have-branches-117763.rs b/tests/codegen-llvm/issues/cows-dont-have-branches-117763.rs
new file mode 100644
index 000000000000..b97729fa1465
--- /dev/null
+++ b/tests/codegen-llvm/issues/cows-dont-have-branches-117763.rs
@@ -0,0 +1,17 @@
+//@ compile-flags: -Copt-level=3
+//@ needs-deterministic-layouts
+
+// Currently Vec and &[T] have layouts that start with (pointer, len)
+// which makes the conversion branchless.
+// A nice-to-have property, not guaranteed.
+#![crate_type = "cdylib"]
+
+// CHECK-LABEL: @branchless_cow_slices
+#[no_mangle]
+pub fn branchless_cow_slices<'a>(cow: &'a std::borrow::Cow<'a, [u8]>) -> &'a [u8] {
+    // CHECK-NOT: br
+    // CHECK-NOT: select
+    // CHECK-NOT: icmp
+    // CHECK: ret { ptr, {{i32|i64}} }
+    &*cow
+}
diff --git a/tests/codegen-llvm/issues/elided-division-by-zero-check-74917.rs b/tests/codegen-llvm/issues/elided-division-by-zero-check-74917.rs
new file mode 100644
index 000000000000..9e890e148527
--- /dev/null
+++ b/tests/codegen-llvm/issues/elided-division-by-zero-check-74917.rs
@@ -0,0 +1,13 @@
+// Tests that there is no check for dividing by zero since the
+// denominator, `(x - y)`, will always be greater than 0 since `x > y`.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @issue_74917
+#[no_mangle]
+pub fn issue_74917(x: u16, y: u16) -> u16 {
+    // CHECK-NOT: panic
+    if x > y { 100 / (x - y) } else { 100 }
+}
diff --git a/tests/codegen-llvm/issues/for-loop-inner-assert-91109.rs b/tests/codegen-llvm/issues/for-loop-inner-assert-91109.rs
new file mode 100644
index 000000000000..bffbcd7d069e
--- /dev/null
+++ b/tests/codegen-llvm/issues/for-loop-inner-assert-91109.rs
@@ -0,0 +1,18 @@
+// Tests that there's no bounds check for the inner loop after the assert.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @zero
+#[no_mangle]
+pub fn zero(d: &mut [Vec]) {
+    // CHECK-NOT: panic_bounds_check
+    let n = d.len();
+    for i in 0..n {
+        assert!(d[i].len() == n);
+        for j in 0..n {
+            d[i][j] = 0;
+        }
+    }
+}
diff --git a/tests/codegen-llvm/issues/issue-101082.rs b/tests/codegen-llvm/issues/issue-101082.rs
index 8d15921ddb4e..0c1f90f951a7 100644
--- a/tests/codegen-llvm/issues/issue-101082.rs
+++ b/tests/codegen-llvm/issues/issue-101082.rs
@@ -1,6 +1,5 @@
 //@ compile-flags: -Copt-level=3
 //@ revisions: host x86-64 x86-64-v3
-//@ min-llvm-version: 20
 
 //@[host] ignore-x86_64
 
diff --git a/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs b/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs
index 848aa910b584..95e70bf56784 100644
--- a/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs
+++ b/tests/codegen-llvm/issues/issue-105386-ub-in-debuginfo.rs
@@ -19,6 +19,6 @@ pub fn outer_function(x: S, y: S) -> usize {
 // CHECK: [[spill:%.*]] = alloca
 // CHECK-NOT: [[ptr_tmp:%.*]] = getelementptr inbounds i8, ptr [[spill]]
 // CHECK-NOT: [[load:%.*]] = load ptr, ptr
-// CHECK: call void @llvm.lifetime.start{{.*}}({{.*}}, ptr [[spill]])
+// CHECK: call void @llvm.lifetime.start{{.*}}({{(.*, )?}}ptr [[spill]])
 // CHECK: [[inner:%.*]] = getelementptr inbounds i8, ptr [[spill]]
 // CHECK: call void @llvm.memcpy{{.*}}(ptr {{align .*}} [[inner]], ptr {{align .*}} %x
diff --git a/tests/codegen-llvm/issues/issue-113757-bounds-check-after-cmp-max.rs b/tests/codegen-llvm/issues/issue-113757-bounds-check-after-cmp-max.rs
index d495adf99804..0db8a5220ec5 100644
--- a/tests/codegen-llvm/issues/issue-113757-bounds-check-after-cmp-max.rs
+++ b/tests/codegen-llvm/issues/issue-113757-bounds-check-after-cmp-max.rs
@@ -5,7 +5,7 @@
 use std::cmp::max;
 
 // CHECK-LABEL: @foo
-// CHECK-NOT: slice_start_index_len_fail
+// CHECK-NOT: slice_index_fail
 // CHECK-NOT: unreachable
 #[no_mangle]
 pub fn foo(v: &mut Vec, size: usize) -> Option<&mut [u8]> {
diff --git a/tests/codegen-llvm/issues/issue-118306.rs b/tests/codegen-llvm/issues/issue-118306.rs
index f12dc7cdfe2c..934a7687b601 100644
--- a/tests/codegen-llvm/issues/issue-118306.rs
+++ b/tests/codegen-llvm/issues/issue-118306.rs
@@ -11,7 +11,7 @@ pub fn branchy(input: u64) -> u64 {
     // CHECK-LABEL: @branchy(
     // CHECK-NEXT:  start:
     // CHECK-NEXT:    [[_2:%.*]] = and i64 [[INPUT:%.*]], 3
-    // CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds{{( nuw)?}} [4 x i64], ptr @switch.table.branchy, i64 0, i64 [[_2]]
+    // CHECK-NEXT:    [[SWITCH_GEP:%.*]] = getelementptr inbounds{{( nuw)?}} {{\[4 x i64\]|i64}}, ptr @switch.table.branchy{{(, i64 0)?}}, i64 [[_2]]
     // CHECK-NEXT:    [[SWITCH_LOAD:%.*]] = load i64, ptr [[SWITCH_GEP]]
     // CHECK-NEXT:    ret i64 [[SWITCH_LOAD]]
     match input % 4 {
diff --git a/tests/codegen-llvm/issues/issue-122600-ptr-discriminant-update.rs b/tests/codegen-llvm/issues/issue-122600-ptr-discriminant-update.rs
index 853a1ff36b10..a0b453fac8e9 100644
--- a/tests/codegen-llvm/issues/issue-122600-ptr-discriminant-update.rs
+++ b/tests/codegen-llvm/issues/issue-122600-ptr-discriminant-update.rs
@@ -1,4 +1,7 @@
 //@ compile-flags: -Copt-level=3
+//@ revisions: new old
+//@ [old] max-llvm-major-version: 21
+//@ [new] min-llvm-version: 22
 
 #![crate_type = "lib"]
 
@@ -22,8 +25,8 @@ pub unsafe fn update(s: *mut State) {
     // CHECK-NOT: memcpy
     // CHECK-NOT: 75{{3|4}}
 
-    // CHECK: %[[TAG:.+]] = load i8, ptr %s, align 1
-    // CHECK-NEXT: trunc nuw i8 %[[TAG]] to i1
+    // old: %[[TAG:.+]] = load i8, ptr %s, align 1
+    // old-NEXT: trunc nuw i8 %[[TAG]] to i1
 
     // CHECK-NOT: load
     // CHECK-NOT: store
diff --git a/tests/codegen-llvm/issues/issue-122734-match-eq.rs b/tests/codegen-llvm/issues/issue-122734-match-eq.rs
new file mode 100644
index 000000000000..89858972677f
--- /dev/null
+++ b/tests/codegen-llvm/issues/issue-122734-match-eq.rs
@@ -0,0 +1,78 @@
+//@ min-llvm-version: 21
+//@ compile-flags: -Copt-level=3 -Zmerge-functions=disabled
+//! Tests that matching + eq on `Option` produces a simple compare with no branching
+
+#![crate_type = "lib"]
+
+#[derive(PartialEq)]
+pub enum TwoNum {
+    A,
+    B,
+}
+
+#[derive(PartialEq)]
+pub enum ThreeNum {
+    A,
+    B,
+    C,
+}
+
+// CHECK-LABEL: @match_two
+#[no_mangle]
+pub fn match_two(a: Option, b: Option) -> bool {
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: icmp eq i8
+    // CHECK-NEXT: ret
+    match (a, b) {
+        (Some(x), Some(y)) => x == y,
+        (Some(_), None) => false,
+        (None, Some(_)) => false,
+        (None, None) => true,
+    }
+}
+
+// CHECK-LABEL: @match_three
+#[no_mangle]
+pub fn match_three(a: Option, b: Option) -> bool {
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: icmp eq
+    // CHECK-NEXT: ret
+    match (a, b) {
+        (Some(x), Some(y)) => x == y,
+        (Some(_), None) => false,
+        (None, Some(_)) => false,
+        (None, None) => true,
+    }
+}
+
+// CHECK-LABEL: @match_two_ref
+#[no_mangle]
+pub fn match_two_ref(a: &Option, b: &Option) -> bool {
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: load i8
+    // CHECK-NEXT: load i8
+    // CHECK-NEXT: icmp eq i8
+    // CHECK-NEXT: ret
+    match (a, b) {
+        (Some(x), Some(y)) => x == y,
+        (Some(_), None) => false,
+        (None, Some(_)) => false,
+        (None, None) => true,
+    }
+}
+
+// CHECK-LABEL: @match_three_ref
+#[no_mangle]
+pub fn match_three_ref(a: &Option, b: &Option) -> bool {
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: load i8
+    // CHECK-NEXT: load i8
+    // CHECK-NEXT: icmp eq
+    // CHECK-NEXT: ret
+    match (a, b) {
+        (Some(x), Some(y)) => x == y,
+        (Some(_), None) => false,
+        (None, Some(_)) => false,
+        (None, None) => true,
+    }
+}
diff --git a/tests/codegen-llvm/issues/issue-129795.rs b/tests/codegen-llvm/issues/issue-129795.rs
index dc64ee35c97e..7a928389fab2 100644
--- a/tests/codegen-llvm/issues/issue-129795.rs
+++ b/tests/codegen-llvm/issues/issue-129795.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -Copt-level=3
-//@ min-llvm-version: 20
 #![crate_type = "lib"]
 
 // Ensure that a modulo operation with an operand that is known to be
diff --git a/tests/codegen-llvm/issues/issue-27130.rs b/tests/codegen-llvm/issues/issue-27130.rs
index 594e02af0977..3e53c5cffd66 100644
--- a/tests/codegen-llvm/issues/issue-27130.rs
+++ b/tests/codegen-llvm/issues/issue-27130.rs
@@ -6,7 +6,7 @@
 #[no_mangle]
 pub fn trim_in_place(a: &mut &[u8]) {
     while a.first() == Some(&42) {
-        // CHECK-NOT: slice_index_order_fail
+        // CHECK-NOT: slice_index_fail
         *a = &a[1..];
     }
 }
@@ -15,7 +15,7 @@ pub fn trim_in_place(a: &mut &[u8]) {
 #[no_mangle]
 pub fn trim_in_place2(a: &mut &[u8]) {
     while let Some(&42) = a.first() {
-        // CHECK-NOT: slice_index_order_fail
+        // CHECK-COUNT-1: slice_index_fail
         *a = &a[2..];
     }
 }
diff --git a/tests/codegen-llvm/issues/issue-69101-bounds-check.rs b/tests/codegen-llvm/issues/issue-69101-bounds-check.rs
index 953b79aa263e..f1857a9ce89d 100644
--- a/tests/codegen-llvm/issues/issue-69101-bounds-check.rs
+++ b/tests/codegen-llvm/issues/issue-69101-bounds-check.rs
@@ -10,7 +10,7 @@
 // CHECK-LABEL: @already_sliced_no_bounds_check
 #[no_mangle]
 pub fn already_sliced_no_bounds_check(a: &[u8], b: &[u8], c: &mut [u8]) {
-    // CHECK: slice_end_index_len_fail
+    // CHECK: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     let _ = (&a[..2048], &b[..2048], &mut c[..2048]);
     for i in 0..1024 {
@@ -21,7 +21,7 @@ pub fn already_sliced_no_bounds_check(a: &[u8], b: &[u8], c: &mut [u8]) {
 // CHECK-LABEL: @already_sliced_no_bounds_check_exact
 #[no_mangle]
 pub fn already_sliced_no_bounds_check_exact(a: &[u8], b: &[u8], c: &mut [u8]) {
-    // CHECK: slice_end_index_len_fail
+    // CHECK: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     let _ = (&a[..1024], &b[..1024], &mut c[..1024]);
     for i in 0..1024 {
@@ -33,7 +33,7 @@ pub fn already_sliced_no_bounds_check_exact(a: &[u8], b: &[u8], c: &mut [u8]) {
 // CHECK-LABEL: @already_sliced_bounds_check
 #[no_mangle]
 pub fn already_sliced_bounds_check(a: &[u8], b: &[u8], c: &mut [u8]) {
-    // CHECK: slice_end_index_len_fail
+    // CHECK: slice_index_fail
     // CHECK: panic_bounds_check
     let _ = (&a[..1023], &b[..2048], &mut c[..2048]);
     for i in 0..1024 {
diff --git a/tests/codegen-llvm/issues/issue-73396-bounds-check-after-position.rs b/tests/codegen-llvm/issues/issue-73396-bounds-check-after-position.rs
index 1e2c25babe0a..8a2200478aa0 100644
--- a/tests/codegen-llvm/issues/issue-73396-bounds-check-after-position.rs
+++ b/tests/codegen-llvm/issues/issue-73396-bounds-check-after-position.rs
@@ -8,8 +8,7 @@
 #[no_mangle]
 pub fn position_slice_to_no_bounds_check(s: &[u8]) -> &[u8] {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().position(|b| *b == b'\\') { &s[..idx] } else { s }
@@ -19,8 +18,7 @@ pub fn position_slice_to_no_bounds_check(s: &[u8]) -> &[u8] {
 #[no_mangle]
 pub fn position_slice_from_no_bounds_check(s: &[u8]) -> &[u8] {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().position(|b| *b == b'\\') { &s[idx..] } else { s }
@@ -30,8 +28,7 @@ pub fn position_slice_from_no_bounds_check(s: &[u8]) -> &[u8] {
 #[no_mangle]
 pub fn position_index_no_bounds_check(s: &[u8]) -> u8 {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().position(|b| *b == b'\\') { s[idx] } else { 42 }
@@ -40,8 +37,7 @@ pub fn position_index_no_bounds_check(s: &[u8]) -> u8 {
 #[no_mangle]
 pub fn rposition_slice_to_no_bounds_check(s: &[u8]) -> &[u8] {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().rposition(|b| *b == b'\\') { &s[..idx] } else { s }
@@ -51,8 +47,7 @@ pub fn rposition_slice_to_no_bounds_check(s: &[u8]) -> &[u8] {
 #[no_mangle]
 pub fn rposition_slice_from_no_bounds_check(s: &[u8]) -> &[u8] {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().rposition(|b| *b == b'\\') { &s[idx..] } else { s }
@@ -62,8 +57,7 @@ pub fn rposition_slice_from_no_bounds_check(s: &[u8]) -> &[u8] {
 #[no_mangle]
 pub fn rposition_index_no_bounds_check(s: &[u8]) -> u8 {
     // CHECK-NOT: panic
-    // CHECK-NOT: slice_start_index_len_fail
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: panic_bounds_check
     // CHECK-NOT: unreachable
     if let Some(idx) = s.iter().rposition(|b| *b == b'\\') { s[idx] } else { 42 }
diff --git a/tests/codegen-llvm/issues/iter-max-no-unwrap-failed-129583.rs b/tests/codegen-llvm/issues/iter-max-no-unwrap-failed-129583.rs
new file mode 100644
index 000000000000..4c4eebeabb5f
--- /dev/null
+++ b/tests/codegen-llvm/issues/iter-max-no-unwrap-failed-129583.rs
@@ -0,0 +1,30 @@
+// Tests that `unwrap` is optimized out when the slice has a known length.
+// The iterator may unroll for values smaller than a certain threshold so we
+// use a larger value to prevent unrolling.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @infallible_max_not_unrolled
+#[no_mangle]
+pub fn infallible_max_not_unrolled(x: &[u8; 1024]) -> u8 {
+    // CHECK-NOT: panic
+    // CHECK-NOT: unwrap_failed
+    *x.iter().max().unwrap()
+}
+
+// CHECK-LABEL: @infallible_max_unrolled
+#[no_mangle]
+pub fn infallible_max_unrolled(x: &[u8; 10]) -> u8 {
+    // CHECK-NOT: panic
+    // CHECK-NOT: unwrap_failed
+    *x.iter().max().unwrap()
+}
+
+// CHECK-LABEL: @may_panic_max
+#[no_mangle]
+pub fn may_panic_max(x: &[u8]) -> u8 {
+    // CHECK: unwrap_failed
+    *x.iter().max().unwrap()
+}
diff --git a/tests/codegen-llvm/issues/looping-over-ne-bytes-133528.rs b/tests/codegen-llvm/issues/looping-over-ne-bytes-133528.rs
index 35acf765d690..b686f8c4b3ac 100644
--- a/tests/codegen-llvm/issues/looping-over-ne-bytes-133528.rs
+++ b/tests/codegen-llvm/issues/looping-over-ne-bytes-133528.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -Copt-level=3
-//@ min-llvm-version: 20
 #![crate_type = "lib"]
 
 /// Ensure the function is properly optimized
diff --git a/tests/codegen-llvm/issues/matches-logical-or-141497.rs b/tests/codegen-llvm/issues/matches-logical-or-141497.rs
new file mode 100644
index 000000000000..348f62096a5f
--- /dev/null
+++ b/tests/codegen-llvm/issues/matches-logical-or-141497.rs
@@ -0,0 +1,25 @@
+// Tests that `matches!` optimizes the same as
+// `f == FrameType::Inter || f == FrameType::Switch`.
+
+//@ compile-flags: -Copt-level=3
+//@ min-llvm-version: 21
+
+#![crate_type = "lib"]
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum FrameType {
+    Key = 0,
+    Inter = 1,
+    Intra = 2,
+    Switch = 3,
+}
+
+// CHECK-LABEL: @is_inter_or_switch
+#[no_mangle]
+pub fn is_inter_or_switch(f: FrameType) -> bool {
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: and i8
+    // CHECK-NEXT: icmp
+    // CHECK-NEXT: ret
+    matches!(f, FrameType::Inter | FrameType::Switch)
+}
diff --git a/tests/codegen-llvm/issues/no-bounds-check-after-assert-110971.rs b/tests/codegen-llvm/issues/no-bounds-check-after-assert-110971.rs
new file mode 100644
index 000000000000..aa4002f176da
--- /dev/null
+++ b/tests/codegen-llvm/issues/no-bounds-check-after-assert-110971.rs
@@ -0,0 +1,14 @@
+// Tests that the slice access for `j` doesn't have a bounds check panic after
+// being asserted as less than half of the slice length.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @check_only_assert_panic
+#[no_mangle]
+pub fn check_only_assert_panic(arr: &[u32], j: usize) -> u32 {
+    // CHECK-NOT: panic_bounds_check
+    assert!(j < arr.len() / 2);
+    arr[j]
+}
diff --git a/tests/codegen-llvm/issues/no-panic-for-pop-after-assert-71257.rs b/tests/codegen-llvm/issues/no-panic-for-pop-after-assert-71257.rs
new file mode 100644
index 000000000000..68877c28d6ce
--- /dev/null
+++ b/tests/codegen-llvm/issues/no-panic-for-pop-after-assert-71257.rs
@@ -0,0 +1,19 @@
+// Tests that the `unwrap` branch is optimized out from the `pop` since the
+// length has already been validated.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+pub enum Foo {
+    First(usize),
+    Second(usize),
+}
+
+// CHECK-LABEL: @check_only_one_panic
+#[no_mangle]
+pub fn check_only_one_panic(v: &mut Vec) -> Foo {
+    // CHECK-COUNT-1: call{{.+}}panic
+    assert!(v.len() == 1);
+    v.pop().unwrap()
+}
diff --git a/tests/codegen-llvm/issues/num-is-digit-to-digit-59352.rs b/tests/codegen-llvm/issues/num-is-digit-to-digit-59352.rs
new file mode 100644
index 000000000000..1d23eb2cdf9e
--- /dev/null
+++ b/tests/codegen-llvm/issues/num-is-digit-to-digit-59352.rs
@@ -0,0 +1,14 @@
+// Tests that there's no panic on unwrapping `to_digit` call after checking
+// with `is_digit`.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @num_to_digit_slow
+#[no_mangle]
+pub fn num_to_digit_slow(num: char) -> u32 {
+    // CHECK-NOT: br
+    // CHECK-NOT: panic
+    if num.is_digit(8) { num.to_digit(8).unwrap() } else { 0 }
+}
diff --git a/tests/codegen-llvm/issues/saturating-sub-index-139759.rs b/tests/codegen-llvm/issues/saturating-sub-index-139759.rs
new file mode 100644
index 000000000000..eac2f4d306b9
--- /dev/null
+++ b/tests/codegen-llvm/issues/saturating-sub-index-139759.rs
@@ -0,0 +1,19 @@
+// Test that calculating an index with saturating subtraction from an in-bounds
+// index doesn't generate another bounds check.
+
+//@ compile-flags: -Copt-level=3
+//@ min-llvm-version: 21
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @bounds_check_is_elided
+#[no_mangle]
+pub fn bounds_check_is_elided(s: &[i32], index: usize) -> i32 {
+    // CHECK-NOT: panic_bounds_check
+    if index < s.len() {
+        let lower_bound = index.saturating_sub(1);
+        s[lower_bound]
+    } else {
+        -1
+    }
+}
diff --git a/tests/codegen-llvm/issues/slice-index-bounds-check-80075.rs b/tests/codegen-llvm/issues/slice-index-bounds-check-80075.rs
new file mode 100644
index 000000000000..ecb5eb787c70
--- /dev/null
+++ b/tests/codegen-llvm/issues/slice-index-bounds-check-80075.rs
@@ -0,0 +1,13 @@
+// Tests that no bounds check panic is generated for `j` since
+// `j <= i < data.len()`.
+
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+
+// CHECK-LABEL: @issue_80075
+#[no_mangle]
+pub fn issue_80075(data: &[u8], i: usize, j: usize) -> u8 {
+    // CHECK-NOT: panic_bounds_check
+    if i < data.len() && j <= i { data[j] } else { 0 }
+}
diff --git a/tests/codegen-llvm/iter-repeat-n-trivial-drop.rs b/tests/codegen-llvm/iter-repeat-n-trivial-drop.rs
index 281735303243..6f3409784280 100644
--- a/tests/codegen-llvm/iter-repeat-n-trivial-drop.rs
+++ b/tests/codegen-llvm/iter-repeat-n-trivial-drop.rs
@@ -4,7 +4,6 @@
 
 #![crate_type = "lib"]
 #![feature(iter_repeat_n)]
-#![feature(array_repeat)]
 
 #[derive(Clone)]
 pub struct NotCopy(u16);
diff --git a/tests/codegen-llvm/lib-optimizations/slice_fill.rs b/tests/codegen-llvm/lib-optimizations/slice_fill.rs
new file mode 100644
index 000000000000..2d924ebf726d
--- /dev/null
+++ b/tests/codegen-llvm/lib-optimizations/slice_fill.rs
@@ -0,0 +1,28 @@
+//@ compile-flags: -Copt-level=3
+#![crate_type = "lib"]
+
+use std::mem::MaybeUninit;
+
+// CHECK-LABEL: @slice_fill_pass_undef
+#[no_mangle]
+pub fn slice_fill_pass_undef(s: &mut [MaybeUninit], v: MaybeUninit) {
+    // CHECK: tail call void @llvm.memset.{{.*}}(ptr nonnull align 1 %s.0, i8 %v, {{.*}} %s.1, i1 false)
+    // CHECK: ret
+    s.fill(v);
+}
+
+// CHECK-LABEL: @slice_fill_uninit
+#[no_mangle]
+pub fn slice_fill_uninit(s: &mut [MaybeUninit]) {
+    // CHECK-NOT: call
+    // CHECK: ret void
+    s.fill(MaybeUninit::uninit());
+}
+
+// CHECK-LABEL: @slice_wide_memset
+#[no_mangle]
+pub fn slice_wide_memset(s: &mut [u16]) {
+    // CHECK: tail call void @llvm.memset.{{.*}}(ptr nonnull align 2 %s.0, i8 -1
+    // CHECK: ret
+    s.fill(0xFFFF);
+}
diff --git a/tests/codegen-llvm/lifetime_start_end.rs b/tests/codegen-llvm/lifetime_start_end.rs
index 0639e7640aa1..2df5acf54df5 100644
--- a/tests/codegen-llvm/lifetime_start_end.rs
+++ b/tests/codegen-llvm/lifetime_start_end.rs
@@ -8,27 +8,27 @@ pub fn test() {
     let a = 0u8;
     &a; // keep variable in an alloca
 
-    // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, ptr %a)
+    // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}ptr %a)
 
     {
         let b = &Some(a);
         &b; // keep variable in an alloca
 
-        // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, {{.*}})
+        // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}{{.*}})
 
-        // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, {{.*}})
+        // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}{{.*}})
 
-        // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, {{.*}})
+        // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}{{.*}})
 
-        // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, {{.*}})
+        // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}{{.*}})
     }
 
     let c = 1u8;
     &c; // keep variable in an alloca
 
-    // CHECK: call void @llvm.lifetime.start{{.*}}(i{{[0-9 ]+}}, ptr %c)
+    // CHECK: call void @llvm.lifetime.start{{.*}}({{(i[0-9 ]+, )?}}ptr %c)
 
-    // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, ptr %c)
+    // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}ptr %c)
 
-    // CHECK: call void @llvm.lifetime.end{{.*}}(i{{[0-9 ]+}}, ptr %a)
+    // CHECK: call void @llvm.lifetime.end{{.*}}({{(i[0-9 ]+, )?}}ptr %a)
 }
diff --git a/tests/codegen-llvm/loongarch-abi/cast-local-large-enough.rs b/tests/codegen-llvm/loongarch-abi/cast-local-large-enough.rs
new file mode 100644
index 000000000000..e5a0e4cd3a2c
--- /dev/null
+++ b/tests/codegen-llvm/loongarch-abi/cast-local-large-enough.rs
@@ -0,0 +1,44 @@
+//@ add-core-stubs
+//@ compile-flags: -Copt-level=0 -Cdebuginfo=0 --target loongarch64-unknown-linux-gnu
+//@ needs-llvm-components: loongarch
+
+#![feature(no_core, lang_items)]
+#![no_std]
+#![no_core]
+#![crate_type = "lib"]
+
+extern crate minicore;
+use minicore::*;
+
+#[repr(C, align(64))]
+struct Aligned(f64);
+
+#[repr(C, align(64))]
+struct AlignedPair(f32, f64);
+
+impl Copy for Aligned {}
+impl Copy for AlignedPair {}
+
+// CHECK-LABEL: define double @read_aligned
+#[unsafe(no_mangle)]
+pub extern "C" fn read_aligned(x: &Aligned) -> Aligned {
+    // CHECK: %[[TEMP:.*]] = alloca [64 x i8], align 64
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 64 %[[TEMP]], ptr align 64 %[[PTR:.*]], i64 64, i1 false)
+    // CHECK-NEXT: %[[RES:.*]] = load double, ptr %[[TEMP]], align 64
+    // CHECK-NEXT: ret double %[[RES]]
+    *x
+}
+
+// CHECK-LABEL: define { float, double } @read_aligned_pair
+#[unsafe(no_mangle)]
+pub extern "C" fn read_aligned_pair(x: &AlignedPair) -> AlignedPair {
+    // CHECK: %[[TEMP:.*]] = alloca [64 x i8], align 64
+    // CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 64 %[[TEMP]], ptr align 64 %[[PTR:.*]], i64 64, i1 false)
+    // CHECK-NEXT: %[[FIRST:.*]] = load float, ptr %[[TEMP]], align 64
+    // CHECK-NEXT: %[[SECOND_PTR:.*]] = getelementptr inbounds i8, ptr %[[TEMP]], i64 8
+    // CHECK-NEXT: %[[SECOND:.*]] = load double, ptr %[[SECOND_PTR]], align 8
+    // CHECK-NEXT: %[[RES1:.*]] = insertvalue { float, double } poison, float %[[FIRST]], 0
+    // CHECK-NEXT: %[[RES2:.*]] = insertvalue { float, double } %[[RES1]], double %[[SECOND]], 1
+    // CHECK-NEXT: ret { float, double } %[[RES2]]
+    *x
+}
diff --git a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs
index 93c8d60930b7..7ea08a1a8f72 100644
--- a/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs
+++ b/tests/codegen-llvm/loongarch-abi/loongarch64-lp64d-abi.rs
@@ -256,11 +256,11 @@ pub struct IntDoubleInt {
     c: i32,
 }
 
-// CHECK: define void @f_int_double_int_s_arg(ptr noalias{{( nocapture)?}} noundef align 8{{( captures\(none\))?}} dereferenceable(24) %a)
+// CHECK: define void @f_int_double_int_s_arg(ptr{{( dead_on_return)?}} noalias noundef align 8{{( captures\(address\))?}} dereferenceable(24) %a)
 #[no_mangle]
 pub extern "C" fn f_int_double_int_s_arg(a: IntDoubleInt) {}
 
-// CHECK: define void @f_ret_int_double_int_s(ptr{{( dead_on_unwind)?}} noalias{{( nocapture)?}} noundef{{( writable)?}} sret([24 x i8]) align 8{{( captures\(none\))?}} dereferenceable(24) %_0)
+// CHECK: define void @f_ret_int_double_int_s(ptr{{( dead_on_unwind)?}} noalias noundef{{( writable)?}} sret([24 x i8]) align 8{{( captures\(address\))?}} dereferenceable(24) %_0)
 #[no_mangle]
 pub extern "C" fn f_ret_int_double_int_s() -> IntDoubleInt {
     IntDoubleInt { a: 1, b: 2., c: 3 }
diff --git a/tests/codegen-llvm/naked-asan.rs b/tests/codegen-llvm/naked-asan.rs
index 46218cf79d6d..a57e55d1366c 100644
--- a/tests/codegen-llvm/naked-asan.rs
+++ b/tests/codegen-llvm/naked-asan.rs
@@ -18,10 +18,10 @@ pub fn caller() {
     unsafe { asm!("call {}", sym page_fault_handler) }
 }
 
-// CHECK: declare x86_intrcc void @page_fault_handler(){{.*}}#[[ATTRS:[0-9]+]]
+// CHECK: declare x86_intrcc void @page_fault_handler(ptr {{.*}}, i64{{.*}}){{.*}}#[[ATTRS:[0-9]+]]
 #[unsafe(naked)]
 #[no_mangle]
-pub extern "x86-interrupt" fn page_fault_handler() {
+pub extern "x86-interrupt" fn page_fault_handler(_: u64, _: u64) {
     naked_asm!("ud2")
 }
 
diff --git a/tests/codegen-llvm/option-niche-eq.rs b/tests/codegen-llvm/option-niche-eq.rs
index 3900cb79aa2a..e9c3fa2407e7 100644
--- a/tests/codegen-llvm/option-niche-eq.rs
+++ b/tests/codegen-llvm/option-niche-eq.rs
@@ -1,5 +1,4 @@
 //@ revisions: REGULAR LLVM21
-//@ min-llvm-version: 20
 //@ compile-flags: -Copt-level=3 -Zmerge-functions=disabled
 //@ [LLVM21] min-llvm-version: 21
 #![crate_type = "lib"]
diff --git a/tests/codegen-llvm/pattern_type_symbols.rs b/tests/codegen-llvm/pattern_type_symbols.rs
index e86a9ef27de1..a90262ff12d2 100644
--- a/tests/codegen-llvm/pattern_type_symbols.rs
+++ b/tests/codegen-llvm/pattern_type_symbols.rs
@@ -16,7 +16,7 @@ pub fn bar() {
     // CHECK: call pattern_type_symbols::foo::
     // CHECK: call void @_RINvC[[CRATE_IDENT:[a-zA-Z0-9]{12}]]_20pattern_type_symbols3foomEB2_
     foo::();
-    // CHECK: call pattern_type_symbols::foo::<(u32, [(); 0], [(); 999999999])>
-    // CHECK: call void @_RINvC[[CRATE_IDENT]]_20pattern_type_symbols3fooTmAum0_Aum3b9ac9ff_EEB2_
+    // CHECK: call pattern_type_symbols::foo::
+    // CHECK: call void @_RINvC[[CRATE_IDENT]]_20pattern_type_symbols3fooWmRm0_m3b9ac9ff_EB2_
     foo::();
 }
diff --git a/tests/codegen-llvm/range-attribute.rs b/tests/codegen-llvm/range-attribute.rs
index b81ff9ab3e2b..865d36d47474 100644
--- a/tests/codegen-llvm/range-attribute.rs
+++ b/tests/codegen-llvm/range-attribute.rs
@@ -67,7 +67,7 @@ pub fn enum2_value(x: Enum2) -> Enum2 {
     x
 }
 
-// CHECK: noundef [[USIZE]] @takes_slice(ptr noalias noundef nonnull readonly align 4 %x.0, [[USIZE]] noundef %x.1)
+// CHECK: noundef [[USIZE]] @takes_slice(ptr {{.*}} %x.0, [[USIZE]] noundef %x.1)
 #[no_mangle]
 pub fn takes_slice(x: &[i32]) -> usize {
     x.len()
diff --git a/tests/codegen-llvm/read-only-capture-opt.rs b/tests/codegen-llvm/read-only-capture-opt.rs
new file mode 100644
index 000000000000..78d56f8efc23
--- /dev/null
+++ b/tests/codegen-llvm/read-only-capture-opt.rs
@@ -0,0 +1,18 @@
+//@ compile-flags: -C opt-level=3 -Z mir-opt-level=0
+//@ min-llvm-version: 21
+
+#![crate_type = "lib"]
+
+unsafe extern "C" {
+    safe fn do_something(p: &i32);
+}
+
+#[unsafe(no_mangle)]
+pub fn test() -> i32 {
+    // CHECK-LABEL: @test(
+    // CHECK: ret i32 0
+    let i = 0;
+    do_something(&i);
+    do_something(&i);
+    i
+}
diff --git a/tests/codegen-llvm/repeat-operand-zero-len.rs b/tests/codegen-llvm/repeat-operand-zero-len.rs
index b4cec42a07c5..8d2a0e77d608 100644
--- a/tests/codegen-llvm/repeat-operand-zero-len.rs
+++ b/tests/codegen-llvm/repeat-operand-zero-len.rs
@@ -11,7 +11,7 @@
 #[repr(transparent)]
 pub struct Wrapper([T; N]);
 
-// CHECK-LABEL: define {{.+}}do_repeat{{.+}}(i32 noundef %x)
+// CHECK-LABEL: define {{.+}}do_repeat{{.+}}(i32 noundef{{( signext)?}} %x)
 // CHECK-NEXT: start:
 // CHECK-NOT: alloca
 // CHECK-NEXT: ret void
@@ -23,6 +23,6 @@ pub fn do_repeat(x: T) -> Wrapper {
 // CHECK-LABEL: @trigger_repeat_zero_len
 #[no_mangle]
 pub fn trigger_repeat_zero_len() -> Wrapper {
-    // CHECK: call void {{.+}}do_repeat{{.+}}(i32 noundef 4)
+    // CHECK: call void {{.+}}do_repeat{{.+}}(i32 noundef{{( signext)?}} 4)
     do_repeat(4)
 }
diff --git a/tests/codegen-llvm/rust-abi-arch-specific-adjustment.rs b/tests/codegen-llvm/rust-abi-arch-specific-adjustment.rs
index 561f081c700e..ffff4b359947 100644
--- a/tests/codegen-llvm/rust-abi-arch-specific-adjustment.rs
+++ b/tests/codegen-llvm/rust-abi-arch-specific-adjustment.rs
@@ -1,15 +1,19 @@
+//@ add-core-stubs
 //@ compile-flags: -Copt-level=3 -C no-prepopulate-passes
 //@ revisions: riscv64 loongarch64
 
-//@[riscv64] only-riscv64
 //@[riscv64] compile-flags: --target riscv64gc-unknown-linux-gnu
 //@[riscv64] needs-llvm-components: riscv
 
-//@[loongarch64] only-loongarch64
 //@[loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu
 //@[loongarch64] needs-llvm-components: loongarch
 
 #![crate_type = "lib"]
+#![feature(no_core)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
 
 #[no_mangle]
 // riscv64:     define noundef i8 @arg_attr_u8(i8 noundef zeroext %x)
diff --git a/tests/codegen-llvm/s390x-simd.rs b/tests/codegen-llvm/s390x-simd.rs
index ac39357519e4..464c1be11f1a 100644
--- a/tests/codegen-llvm/s390x-simd.rs
+++ b/tests/codegen-llvm/s390x-simd.rs
@@ -6,7 +6,7 @@
 
 #![crate_type = "rlib"]
 #![feature(no_core, asm_experimental_arch)]
-#![feature(s390x_target_feature, simd_ffi, link_llvm_intrinsics, repr_simd)]
+#![feature(s390x_target_feature, simd_ffi, intrinsics, repr_simd)]
 #![no_core]
 
 extern crate minicore;
@@ -30,16 +30,20 @@ struct f32x4([f32; 4]);
 #[repr(simd)]
 struct f64x2([f64; 2]);
 
-#[allow(improper_ctypes)]
-extern "C" {
-    #[link_name = "llvm.smax.v16i8"]
-    fn vmxb(a: i8x16, b: i8x16) -> i8x16;
-    #[link_name = "llvm.smax.v8i16"]
-    fn vmxh(a: i16x8, b: i16x8) -> i16x8;
-    #[link_name = "llvm.smax.v4i32"]
-    fn vmxf(a: i32x4, b: i32x4) -> i32x4;
-    #[link_name = "llvm.smax.v2i64"]
-    fn vmxg(a: i64x2, b: i64x2) -> i64x2;
+impl Copy for i8x16 {}
+impl Copy for i16x8 {}
+impl Copy for i32x4 {}
+impl Copy for i64x2 {}
+
+#[rustc_intrinsic]
+unsafe fn simd_ge(x: T, y: T) -> U;
+
+#[rustc_intrinsic]
+unsafe fn simd_select(mask: M, a: V, b: V) -> V;
+
+#[inline(always)]
+unsafe fn simd_max(a: T, b: T) -> T {
+    simd_select(simd_ge::(a, b), a, b)
 }
 
 // CHECK-LABEL: define <16 x i8> @max_i8x16
@@ -48,7 +52,7 @@ extern "C" {
 #[no_mangle]
 #[target_feature(enable = "vector")]
 pub unsafe extern "C" fn max_i8x16(a: i8x16, b: i8x16) -> i8x16 {
-    vmxb(a, b)
+    simd_max(a, b)
 }
 
 // CHECK-LABEL: define <8 x i16> @max_i16x8
@@ -57,7 +61,7 @@ pub unsafe extern "C" fn max_i8x16(a: i8x16, b: i8x16) -> i8x16 {
 #[no_mangle]
 #[target_feature(enable = "vector")]
 pub unsafe extern "C" fn max_i16x8(a: i16x8, b: i16x8) -> i16x8 {
-    vmxh(a, b)
+    simd_max(a, b)
 }
 
 // CHECK-LABEL: define <4 x i32> @max_i32x4
@@ -66,7 +70,7 @@ pub unsafe extern "C" fn max_i16x8(a: i16x8, b: i16x8) -> i16x8 {
 #[no_mangle]
 #[target_feature(enable = "vector")]
 pub unsafe extern "C" fn max_i32x4(a: i32x4, b: i32x4) -> i32x4 {
-    vmxf(a, b)
+    simd_max(a, b)
 }
 
 // CHECK-LABEL: define <2 x i64> @max_i64x2
@@ -75,7 +79,7 @@ pub unsafe extern "C" fn max_i32x4(a: i32x4, b: i32x4) -> i32x4 {
 #[no_mangle]
 #[target_feature(enable = "vector")]
 pub unsafe extern "C" fn max_i64x2(a: i64x2, b: i64x2) -> i64x2 {
-    vmxg(a, b)
+    simd_max(a, b)
 }
 
 // CHECK-LABEL: define <4 x float> @choose_f32x4
@@ -108,7 +112,7 @@ pub unsafe extern "C" fn max_wrapper_i8x16(a: Wrapper, b: Wrapper)
     // CHECK: call <16 x i8> @llvm.smax.v16i8
     // CHECK-SAME: <16 x i8>
     // CHECK-SAME: <16 x i8>
-    Wrapper(vmxb(a.0, b.0))
+    Wrapper(simd_max(a.0, b.0))
 }
 
 #[no_mangle]
@@ -122,7 +126,7 @@ pub unsafe extern "C" fn max_wrapper_i64x2(a: Wrapper, b: Wrapper)
     // CHECK: call <2 x i64> @llvm.smax.v2i64
     // CHECK-SAME: <2 x i64>
     // CHECK-SAME: <2 x i64>
-    Wrapper(vmxg(a.0, b.0))
+    Wrapper(simd_max(a.0, b.0))
 }
 
 #[no_mangle]
diff --git a/tests/codegen-llvm/sanitizer/address-sanitizer-globals-tracking.rs b/tests/codegen-llvm/sanitizer/address-sanitizer-globals-tracking.rs
index 642bf5e75762..ada525b6c803 100644
--- a/tests/codegen-llvm/sanitizer/address-sanitizer-globals-tracking.rs
+++ b/tests/codegen-llvm/sanitizer/address-sanitizer-globals-tracking.rs
@@ -19,7 +19,7 @@
 //@ only-linux
 //
 //@ revisions:ASAN ASAN-FAT-LTO
-//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static
+//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 // [ASAN] no extra compile-flags
 //@[ASAN-FAT-LTO] compile-flags: -Cprefer-dynamic=false -Clto=fat
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/add-canonical-jump-tables-flag.rs b/tests/codegen-llvm/sanitizer/cfi/add-canonical-jump-tables-flag.rs
index 22577e2a3c46..77857ca4ccb9 100644
--- a/tests/codegen-llvm/sanitizer/cfi/add-canonical-jump-tables-flag.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/add-canonical-jump-tables-flag.rs
@@ -1,7 +1,7 @@
 // Verifies that "CFI Canonical Jump Tables" module flag is added.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi
+//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/add-cfi-normalize-integers-flag.rs b/tests/codegen-llvm/sanitizer/cfi/add-cfi-normalize-integers-flag.rs
index a54a6d84a807..6cf9a72b7488 100644
--- a/tests/codegen-llvm/sanitizer/cfi/add-cfi-normalize-integers-flag.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/add-cfi-normalize-integers-flag.rs
@@ -1,7 +1,7 @@
 // Verifies that "cfi-normalize-integers" module flag is added.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers
+//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -C unsafe-allow-abi-mismatch=sanitizer,sanitizer-cfi-normalize-integers
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/add-enable-split-lto-unit-flag.rs b/tests/codegen-llvm/sanitizer/cfi/add-enable-split-lto-unit-flag.rs
index 283b8f261029..0bfdbfba5d2e 100644
--- a/tests/codegen-llvm/sanitizer/cfi/add-enable-split-lto-unit-flag.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/add-enable-split-lto-unit-flag.rs
@@ -1,7 +1,7 @@
 // Verifies that "EnableSplitLTOUnit" module flag is added.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi
+//@ compile-flags: -Clto -Ctarget-feature=-crt-static -Zsanitizer=cfi -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/dbg-location-on-cfi-blocks.rs b/tests/codegen-llvm/sanitizer/cfi/dbg-location-on-cfi-blocks.rs
index df65960dfe0b..2a18e30e2b0d 100644
--- a/tests/codegen-llvm/sanitizer/cfi/dbg-location-on-cfi-blocks.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/dbg-location-on-cfi-blocks.rs
@@ -1,7 +1,7 @@
 // Verifies that the parent block's debug information are assigned to the inserted cfi block.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -Cdebuginfo=1
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -Cdebuginfo=1 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-no-sanitize.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-no-sanitize.rs
deleted file mode 100644
index 71ccdc8ca624..000000000000
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-no-sanitize.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-// Verifies that pointer type membership tests for indirect calls are omitted.
-//
-//@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
-
-#![crate_type = "lib"]
-#![feature(no_sanitize)]
-
-#[no_sanitize(cfi)]
-pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 {
-    // CHECK-LABEL: emit_type_checks_attr_no_sanitize::foo
-    // CHECK:       Function Attrs: {{.*}}
-    // CHECK-LABEL: define{{.*}}foo{{.*}}!type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
-    // CHECK:       start:
-    // CHECK-NEXT:  {{%.+}} = call i32 %f(i32{{.*}} %arg)
-    // CHECK-NEXT:  ret i32 {{%.+}}
-    f(arg)
-}
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-sanitize-off.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-sanitize-off.rs
new file mode 100644
index 000000000000..c49438f43186
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-checks-attr-sanitize-off.rs
@@ -0,0 +1,18 @@
+// Verifies that pointer type membership tests for indirect calls are omitted.
+//
+//@ needs-sanitizer-cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
+
+#![crate_type = "lib"]
+#![feature(sanitize)]
+
+#[sanitize(cfi = "off")]
+pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 {
+    // CHECK-LABEL: emit_type_checks_attr_sanitize_off::foo
+    // CHECK:       Function Attrs: {{.*}}
+    // CHECK-LABEL: define{{.*}}foo{{.*}}!type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}} !type !{{[0-9]+}}
+    // CHECK:       start:
+    // CHECK-NEXT:  {{%.+}} = call i32 %f(i32{{.*}} %arg)
+    // CHECK-NEXT:  ret i32 {{%.+}}
+    f(arg)
+}
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-checks.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-checks.rs
index ebc66a015df6..9cad88f65182 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-checks.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-checks.rs
@@ -1,7 +1,7 @@
 // Verifies that pointer type membership tests for indirect calls are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs
index 9bc2e42db0f6..cd9088f58af4 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-attr-cfi-encoding.rs
@@ -1,7 +1,7 @@
 // Verifies that user-defined CFI encoding for types are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 #![feature(cfi_encoding, extern_types)]
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
index 9048c6a1f183..cf26c17af1ed 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-const-generics.rs
@@ -2,7 +2,7 @@
 // for const generics.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 #![feature(type_alias_impl_trait)]
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-drop-in-place.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-drop-in-place.rs
index 8fec275fd064..279350d20c5f 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-drop-in-place.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-drop-in-place.rs
@@ -5,7 +5,7 @@
 // future.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs
index 7e60aafff680..047b532e994e 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-function-types.rs
@@ -2,7 +2,7 @@
 // for function types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
index 36d2e8c9f25a..92b2ab32ea03 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-lifetimes.rs
@@ -2,7 +2,7 @@
 // for lifetimes/regions.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 #![feature(type_alias_impl_trait)]
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-method-secondary-typeid.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-method-secondary-typeid.rs
index 9d611777ff0b..5de39dc85c17 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-method-secondary-typeid.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-method-secondary-typeid.rs
@@ -2,7 +2,7 @@
 // self so they can be used as function pointers.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs
index a8ba8db1be3b..4ce9c57070a7 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-paths.rs
@@ -2,7 +2,7 @@
 // for paths.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 #![feature(type_alias_impl_trait)]
@@ -78,9 +78,9 @@ pub fn foo12(_: &Type4, _: &Type4, _: &Type4) {}
 // CHECK: ![[TYPE4]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooEE"}
 // CHECK: ![[TYPE5]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooES0_E"}
 // CHECK: ![[TYPE6]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNCNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo11{{[{}][{}]}}closure{{[}][}]}}3FooES0_S0_E"}
-// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooEE"}
-// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooES0_E"}
-// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNkNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooES0_S0_E"}
+// CHECK: ![[TYPE7]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNKNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooEE"}
+// CHECK: ![[TYPE8]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNKNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooES0_E"}
+// CHECK: ![[TYPE9]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NtNKNvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo12{{[{}][{}]}}constant{{[}][}]}}3FooES0_S0_E"}
 // CHECK: ![[TYPE10]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barEE"}
 // CHECK: ![[TYPE11]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barES0_E"}
 // CHECK: ![[TYPE12]] = !{i64 0, !"_ZTSFvu3refIu{{[0-9]+}}NvNINvC{{[[:print:]]+}}_{{[[:print:]]+}}3foo8{{[{}][{}]}}impl{{[}][}]}}3barES0_S0_E"}
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
index d37bb740f550..ad4fe11d0872 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-pointer-types.rs
@@ -2,7 +2,7 @@
 // for pointer types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
index 7d9e4d058727..93845d051954 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-primitive-types.rs
@@ -2,7 +2,7 @@
 // for primitive types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
index 0f97c70f3f92..025aa902658e 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-repr-transparent-types.rs
@@ -2,7 +2,7 @@
 // for repr transparent types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
index bdee3f47a837..76c8150b7785 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-sequence-types.rs
@@ -2,7 +2,7 @@
 // for sequence types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
index 55e816178f8f..4fafdd2f040f 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-trait-types.rs
@@ -2,7 +2,7 @@
 // for trait types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
index c1f3ca61afeb..91351096ca20 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-id-itanium-cxx-abi-user-defined-types.rs
@@ -2,7 +2,7 @@
 // for user-defined types.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 #![feature(extern_types)]
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs
index 32637b64b3ea..22d518cca744 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-generalized.rs
@@ -1,7 +1,7 @@
 // Verifies that generalized type metadata for functions are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-generalize-pointers
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-generalize-pointers -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
index 51121b0aef1a..7639ce7b1044 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized-generalized.rs
@@ -1,7 +1,7 @@
 // Verifies that normalized and generalized type metadata for functions are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -Zsanitizer-cfi-generalize-pointers
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -Zsanitizer-cfi-generalize-pointers -C unsafe-allow-abi-mismatch=sanitizer,sanitizer-cfi-normalize-integers
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs
index 1cfdd23006e3..acd72b0ca3cf 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi-normalized.rs
@@ -1,7 +1,7 @@
 // Verifies that normalized type metadata for functions are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -C unsafe-allow-abi-mismatch=sanitizer,sanitizer-cfi-normalize-integers
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs
index 56ab1ce4b358..fa5cd471466e 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-itanium-cxx-abi.rs
@@ -1,7 +1,7 @@
 // Verifies that type metadata for functions are emitted.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-trait-objects.rs b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-trait-objects.rs
index 0e57ce322d12..82873e935b29 100644
--- a/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-trait-objects.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/emit-type-metadata-trait-objects.rs
@@ -1,7 +1,7 @@
 // Verifies that type metadata identifiers for trait objects are emitted correctly.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Ctarget-feature=-crt-static -Zsanitizer=cfi
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Copt-level=0 -Ctarget-feature=-crt-static -Zsanitizer=cfi -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/external_weak_symbols.rs b/tests/codegen-llvm/sanitizer/cfi/external_weak_symbols.rs
index 00e9b5029af8..893b01676936 100644
--- a/tests/codegen-llvm/sanitizer/cfi/external_weak_symbols.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/external_weak_symbols.rs
@@ -2,7 +2,7 @@
 // emitted correctly.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clinker-plugin-lto -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static
+//@ compile-flags: -Clinker-plugin-lto -Copt-level=0 -Zsanitizer=cfi -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 #![crate_type = "bin"]
 #![feature(linkage)]
 
@@ -10,7 +10,7 @@ unsafe extern "C" {
     #[linkage = "extern_weak"]
     static FOO: Option ()>;
 }
-// CHECK: @_rust_extern_with_linkage_FOO = internal global ptr @FOO
+// CHECK: @_rust_extern_with_linkage_{{.*}}_FOO = internal global ptr @FOO
 
 fn main() {
     unsafe {
diff --git a/tests/codegen-llvm/sanitizer/cfi/generalize-pointers.rs b/tests/codegen-llvm/sanitizer/cfi/generalize-pointers.rs
index 57004da6f8e0..caa2f258f8f2 100644
--- a/tests/codegen-llvm/sanitizer/cfi/generalize-pointers.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/generalize-pointers.rs
@@ -1,7 +1,7 @@
 // Verifies that pointer types are generalized.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-generalize-pointers -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-generalize-pointers -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/cfi/normalize-integers.rs b/tests/codegen-llvm/sanitizer/cfi/normalize-integers.rs
index 770ee4e64e08..16f76adafb82 100644
--- a/tests/codegen-llvm/sanitizer/cfi/normalize-integers.rs
+++ b/tests/codegen-llvm/sanitizer/cfi/normalize-integers.rs
@@ -1,7 +1,7 @@
 // Verifies that integer types are normalized.
 //
 //@ needs-sanitizer-cfi
-//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -Copt-level=0
+//@ compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer,sanitizer-cfi-normalize-integers
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/dataflow-instrument-functions.rs b/tests/codegen-llvm/sanitizer/dataflow-instrument-functions.rs
index a2d0d63cc175..cd8c2c48ea4f 100644
--- a/tests/codegen-llvm/sanitizer/dataflow-instrument-functions.rs
+++ b/tests/codegen-llvm/sanitizer/dataflow-instrument-functions.rs
@@ -1,7 +1,7 @@
 // Verifies that functions are instrumented.
 //
 //@ needs-sanitizer-dataflow
-//@ compile-flags: -Copt-level=0 -Zsanitizer=dataflow
+//@ compile-flags: -Copt-level=0 -Zsanitizer=dataflow -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/kasan-emits-instrumentation.rs b/tests/codegen-llvm/sanitizer/kasan-emits-instrumentation.rs
index 774c9ab53f1e..c70aae1703ef 100644
--- a/tests/codegen-llvm/sanitizer/kasan-emits-instrumentation.rs
+++ b/tests/codegen-llvm/sanitizer/kasan-emits-instrumentation.rs
@@ -13,7 +13,7 @@
 //@[x86_64] needs-llvm-components: x86
 
 #![crate_type = "rlib"]
-#![feature(no_core, no_sanitize, lang_items)]
+#![feature(no_core, sanitize, lang_items)]
 #![no_core]
 
 extern crate minicore;
@@ -25,7 +25,7 @@ use minicore::*;
 // CHECK:       start:
 // CHECK-NOT:   call void @__asan_report_load
 // CHECK:       }
-#[no_sanitize(address)]
+#[sanitize(address = "off")]
 pub fn unsanitized(b: &mut u8) -> u8 {
     *b
 }
diff --git a/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs b/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs
deleted file mode 100644
index 02c31fb8e9b0..000000000000
--- a/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-no-sanitize.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-// Verifies that KCFI operand bundles are omitted.
-//
-//@ add-core-stubs
-//@ revisions: aarch64 x86_64
-//@ [aarch64] compile-flags: --target aarch64-unknown-none
-//@ [aarch64] needs-llvm-components: aarch64
-//@ [x86_64] compile-flags: --target x86_64-unknown-none
-//@ [x86_64] needs-llvm-components: x86
-//@ compile-flags: -Cno-prepopulate-passes -Zsanitizer=kcfi -Copt-level=0
-
-#![crate_type = "lib"]
-#![feature(no_core, no_sanitize, lang_items)]
-#![no_core]
-
-extern crate minicore;
-use minicore::*;
-
-#[no_sanitize(kcfi)]
-pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 {
-    // CHECK-LABEL: emit_kcfi_operand_bundle_attr_no_sanitize::foo
-    // CHECK:       Function Attrs: {{.*}}
-    // CHECK-LABEL: define{{.*}}foo{{.*}}!{{|kcfi_type}} !{{[0-9]+}}
-    // CHECK:       start:
-    // CHECK-NOT:   {{%.+}} = call {{(noundef )*}}i32 %f(i32 {{(noundef )*}}%arg){{.*}}[ "kcfi"(i32 {{[-0-9]+}}) ]
-    // CHECK:       ret i32 {{%.+}}
-    f(arg)
-}
diff --git a/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-sanitize-off.rs b/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-sanitize-off.rs
new file mode 100644
index 000000000000..2581784ce3ee
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/kcfi/emit-kcfi-operand-bundle-attr-sanitize-off.rs
@@ -0,0 +1,27 @@
+// Verifies that KCFI operand bundles are omitted.
+//
+//@ add-core-stubs
+//@ revisions: aarch64 x86_64
+//@ [aarch64] compile-flags: --target aarch64-unknown-none
+//@ [aarch64] needs-llvm-components: aarch64
+//@ [x86_64] compile-flags: --target x86_64-unknown-none
+//@ [x86_64] needs-llvm-components: x86
+//@ compile-flags: -Cno-prepopulate-passes -Zsanitizer=kcfi -Copt-level=0
+
+#![crate_type = "lib"]
+#![feature(no_core, sanitize, lang_items)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+#[sanitize(kcfi = "off")]
+pub fn foo(f: fn(i32) -> i32, arg: i32) -> i32 {
+    // CHECK-LABEL: emit_kcfi_operand_bundle_attr_sanitize_off::foo
+    // CHECK:       Function Attrs: {{.*}}
+    // CHECK-LABEL: define{{.*}}foo{{.*}}!{{|kcfi_type}} !{{[0-9]+}}
+    // CHECK:       start:
+    // CHECK-NOT:   {{%.+}} = call {{(noundef )*}}i32 %f(i32 {{(noundef )*}}%arg){{.*}}[ "kcfi"(i32 {{[-0-9]+}}) ]
+    // CHECK:       ret i32 {{%.+}}
+    f(arg)
+}
diff --git a/tests/codegen-llvm/sanitizer/kcfi/fn-ptr-reify-shim.rs b/tests/codegen-llvm/sanitizer/kcfi/fn-ptr-reify-shim.rs
new file mode 100644
index 000000000000..604b4c8c2f8a
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/kcfi/fn-ptr-reify-shim.rs
@@ -0,0 +1,74 @@
+//@ add-core-stubs
+//@ revisions: aarch64 x86_64
+//@ [aarch64] compile-flags: --target aarch64-unknown-none
+//@ [aarch64] needs-llvm-components: aarch64
+//@ [x86_64] compile-flags: --target x86_64-unknown-none
+//@ [x86_64] needs-llvm-components: x86
+//@ compile-flags: -Ctarget-feature=-crt-static -Zsanitizer=kcfi -Cno-prepopulate-passes -Copt-level=0
+
+#![feature(no_core, lang_items)]
+#![crate_type = "lib"]
+#![no_core]
+
+// A `ReifyShim` should only be created when the trait is dyn-compatible.
+
+extern crate minicore;
+use minicore::*;
+
+trait DynCompatible {
+    fn dyn_name(&self) -> &'static str;
+
+    fn dyn_name_default(&self) -> &'static str {
+        let _ = self;
+        "dyn_default"
+    }
+}
+
+// Not dyn-compatible because the `Self: Sized` bound is missing.
+trait NotDynCompatible {
+    fn not_dyn_name() -> &'static str;
+
+    fn not_dyn_name_default() -> &'static str {
+        "not_dyn_default"
+    }
+}
+
+struct S;
+
+impl DynCompatible for S {
+    fn dyn_name(&self) -> &'static str {
+        "dyn_compatible"
+    }
+}
+
+impl NotDynCompatible for S {
+    fn not_dyn_name() -> &'static str {
+        "not_dyn_compatible"
+    }
+}
+
+#[no_mangle]
+pub fn main() {
+    let s = S;
+
+    // `DynCompatible` is indeed dyn-compatible.
+    let _: &dyn DynCompatible = &s;
+
+    // CHECK: call ::dyn_name{{.*}}reify.shim.fnptr
+    let dyn_name = S::dyn_name as fn(&S) -> &str;
+    let _unused = dyn_name(&s);
+
+    // CHECK: call fn_ptr_reify_shim::DynCompatible::dyn_name_default{{.*}}reify.shim.fnptr
+    let dyn_name_default = S::dyn_name_default as fn(&S) -> &str;
+    let _unused = dyn_name_default(&s);
+
+    // Check using $ (end-of-line) that these calls do not contain `reify.shim.fnptr`.
+
+    // CHECK: call ::not_dyn_name{{$}}
+    let not_dyn_name = S::not_dyn_name as fn() -> &'static str;
+    let _unused = not_dyn_name();
+
+    // CHECK: call fn_ptr_reify_shim::NotDynCompatible::not_dyn_name_default{{$}}
+    let not_dyn_name_default = S::not_dyn_name_default as fn() -> &'static str;
+    let _unused = not_dyn_name_default();
+}
diff --git a/tests/codegen-llvm/sanitizer/kcfi/naked-function.rs b/tests/codegen-llvm/sanitizer/kcfi/naked-function.rs
index 2c8cdc919b85..31f59ee01dec 100644
--- a/tests/codegen-llvm/sanitizer/kcfi/naked-function.rs
+++ b/tests/codegen-llvm/sanitizer/kcfi/naked-function.rs
@@ -15,8 +15,9 @@ use minicore::*;
 
 struct Thing;
 trait MyTrait {
+    // NOTE: this test assumes that this trait is dyn-compatible.
     #[unsafe(naked)]
-    extern "C" fn my_naked_function() {
+    extern "C" fn my_naked_function(&self) {
         // the real function is defined
         // CHECK: .globl
         // CHECK-SAME: my_naked_function
@@ -34,13 +35,13 @@ impl MyTrait for Thing {}
 #[unsafe(no_mangle)]
 pub fn main() {
     // Trick the compiler into generating an indirect call.
-    const F: extern "C" fn() = Thing::my_naked_function;
+    const F: extern "C" fn(&Thing) = Thing::my_naked_function;
 
     // main calls the shim function
     // CHECK: call void
     // CHECK-SAME: my_naked_function
     // CHECK-SAME: reify.shim.fnptr
-    (F)();
+    (F)(&Thing);
 }
 
 // CHECK: declare !kcfi_type
diff --git a/tests/codegen-llvm/sanitizer/memory-track-origins.rs b/tests/codegen-llvm/sanitizer/memory-track-origins.rs
index 5eb5b356b051..a72e523c4e19 100644
--- a/tests/codegen-llvm/sanitizer/memory-track-origins.rs
+++ b/tests/codegen-llvm/sanitizer/memory-track-origins.rs
@@ -4,7 +4,7 @@
 //@ needs-sanitizer-memory
 //@ revisions:MSAN-0 MSAN-1 MSAN-2 MSAN-1-LTO MSAN-2-LTO
 //
-//@ compile-flags: -Zsanitizer=memory -Ctarget-feature=-crt-static
+//@ compile-flags: -Zsanitizer=memory -Ctarget-feature=-crt-static -C unsafe-allow-abi-mismatch=sanitizer
 // [MSAN-0] no extra compile-flags
 //@[MSAN-1] compile-flags: -Zsanitizer-memory-track-origins=1
 //@[MSAN-2] compile-flags: -Zsanitizer-memory-track-origins
diff --git a/tests/codegen-llvm/sanitizer/memtag-attr-check.rs b/tests/codegen-llvm/sanitizer/memtag-attr-check.rs
index ffe3a2322a20..fc430f3a5700 100644
--- a/tests/codegen-llvm/sanitizer/memtag-attr-check.rs
+++ b/tests/codegen-llvm/sanitizer/memtag-attr-check.rs
@@ -2,7 +2,7 @@
 // applied when enabling the memtag sanitizer.
 //
 //@ needs-sanitizer-memtag
-//@ compile-flags: -Zsanitizer=memtag -Ctarget-feature=+mte -Copt-level=0
+//@ compile-flags: -C unsafe-allow-abi-mismatch=sanitizer -Zsanitizer=memtag -Ctarget-feature=+mte -Copt-level=0
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/no-sanitize-inlining.rs b/tests/codegen-llvm/sanitizer/no-sanitize-inlining.rs
deleted file mode 100644
index 4bd832d2ab19..000000000000
--- a/tests/codegen-llvm/sanitizer/no-sanitize-inlining.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-// Verifies that no_sanitize attribute prevents inlining when
-// given sanitizer is enabled, but has no effect on inlining otherwise.
-//
-//@ needs-sanitizer-address
-//@ needs-sanitizer-leak
-//@ revisions: ASAN LSAN
-//@       compile-flags: -Copt-level=3 -Zmir-opt-level=4 -Ctarget-feature=-crt-static
-//@[ASAN] compile-flags: -Zsanitizer=address
-//@[LSAN] compile-flags: -Zsanitizer=leak
-
-#![crate_type = "lib"]
-#![feature(no_sanitize)]
-
-// ASAN-LABEL: define void @test
-// ASAN:         call {{.*}} @random_inline
-// ASAN:       }
-//
-// LSAN-LABEL: define void @test
-// LSAN-NOT:     call
-// LSAN:       }
-#[no_mangle]
-pub fn test(n: &mut u32) {
-    random_inline(n);
-}
-
-#[no_sanitize(address)]
-#[inline]
-#[no_mangle]
-pub fn random_inline(n: &mut u32) {
-    *n = 42;
-}
diff --git a/tests/codegen-llvm/sanitizer/no-sanitize.rs b/tests/codegen-llvm/sanitizer/no-sanitize.rs
deleted file mode 100644
index 2a309f6b9c69..000000000000
--- a/tests/codegen-llvm/sanitizer/no-sanitize.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-// Verifies that no_sanitize attribute can be used to
-// selectively disable sanitizer instrumentation.
-//
-//@ needs-sanitizer-address
-//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static -Copt-level=0
-
-#![crate_type = "lib"]
-#![feature(no_sanitize)]
-
-// CHECK:     @UNSANITIZED = constant{{.*}} no_sanitize_address
-// CHECK-NOT: @__asan_global_UNSANITIZED
-#[no_mangle]
-#[no_sanitize(address)]
-pub static UNSANITIZED: u32 = 0;
-
-// CHECK: @__asan_global_SANITIZED
-#[no_mangle]
-pub static SANITIZED: u32 = 0;
-
-// CHECK-LABEL: ; no_sanitize::unsanitized
-// CHECK-NEXT:  ; Function Attrs:
-// CHECK-NOT:   sanitize_address
-// CHECK:       start:
-// CHECK-NOT:   call void @__asan_report_load
-// CHECK:       }
-#[no_sanitize(address)]
-pub fn unsanitized(b: &mut u8) -> u8 {
-    *b
-}
-
-// CHECK-LABEL: ; no_sanitize::sanitized
-// CHECK-NEXT:  ; Function Attrs:
-// CHECK:       sanitize_address
-// CHECK:       start:
-// CHECK:       call void @__asan_report_load
-// CHECK:       }
-pub fn sanitized(b: &mut u8) -> u8 {
-    *b
-}
diff --git a/tests/codegen-llvm/sanitizer/safestack-attr-check.rs b/tests/codegen-llvm/sanitizer/safestack-attr-check.rs
index 050a60333afa..414dd89a5807 100644
--- a/tests/codegen-llvm/sanitizer/safestack-attr-check.rs
+++ b/tests/codegen-llvm/sanitizer/safestack-attr-check.rs
@@ -1,7 +1,7 @@
 // This tests that the safestack attribute is applied when enabling the safe-stack sanitizer.
 //
 //@ needs-sanitizer-safestack
-//@ compile-flags: -Zsanitizer=safestack -Copt-level=0
+//@ compile-flags: -Zsanitizer=safestack -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
 
 #![crate_type = "lib"]
 
diff --git a/tests/codegen-llvm/sanitizer/sanitize-off-asan-kasan.rs b/tests/codegen-llvm/sanitizer/sanitize-off-asan-kasan.rs
new file mode 100644
index 000000000000..37549aba4477
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/sanitize-off-asan-kasan.rs
@@ -0,0 +1,42 @@
+// Verifies that the `#[sanitize(address = "off")]` attribute also turns off
+// the kernel address sanitizer.
+//
+//@ add-core-stubs
+//@ compile-flags: -Zsanitizer=kernel-address -Ctarget-feature=-crt-static -Copt-level=0
+//@ revisions: aarch64 riscv64imac riscv64gc x86_64
+//@[aarch64] compile-flags: --target aarch64-unknown-none
+//@[aarch64] needs-llvm-components: aarch64
+//@[riscv64imac] compile-flags: --target riscv64imac-unknown-none-elf
+//@[riscv64imac] needs-llvm-components: riscv
+//@[riscv64gc] compile-flags: --target riscv64gc-unknown-none-elf
+//@[riscv64gc] needs-llvm-components: riscv
+//@[x86_64] compile-flags: --target x86_64-unknown-none
+//@[x86_64] needs-llvm-components: x86
+
+#![crate_type = "rlib"]
+#![feature(no_core, sanitize, lang_items)]
+#![no_core]
+
+extern crate minicore;
+use minicore::*;
+
+// CHECK-LABEL: ; sanitize_off_asan_kasan::unsanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK-NOT:   sanitize_address
+// CHECK:       start:
+// CHECK-NOT:   call void @__asan_report_load
+// CHECK:       }
+#[sanitize(address = "off")]
+pub fn unsanitized(b: &mut u8) -> u8 {
+    *b
+}
+
+// CHECK-LABEL: ; sanitize_off_asan_kasan::sanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK:       sanitize_address
+// CHECK:       start:
+// CHECK:       call void @__asan_report_load
+// CHECK:       }
+pub fn sanitized(b: &mut u8) -> u8 {
+    *b
+}
diff --git a/tests/codegen-llvm/sanitizer/sanitize-off-inlining.rs b/tests/codegen-llvm/sanitizer/sanitize-off-inlining.rs
new file mode 100644
index 000000000000..0f43e6b8393d
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/sanitize-off-inlining.rs
@@ -0,0 +1,31 @@
+// Verifies that sanitize(xyz = "off") attribute prevents inlining when
+// given sanitizer is enabled, but has no effect on inlining otherwise.
+//@ needs-sanitizer-address
+//@ needs-sanitizer-leak
+//@ revisions: ASAN LSAN
+//@       compile-flags: -C unsafe-allow-abi-mismatch=sanitizer
+//@       compile-flags: -Copt-level=3 -Zmir-opt-level=4 -Ctarget-feature=-crt-static
+//@[ASAN] compile-flags: -Zsanitizer=address
+//@[LSAN] compile-flags: -Zsanitizer=leak
+
+#![crate_type = "lib"]
+#![feature(sanitize)]
+
+// ASAN-LABEL: define void @test
+// ASAN:         call {{.*}} @random_inline
+// ASAN:       }
+//
+// LSAN-LABEL: define void @test
+// LSAN-NOT:     call
+// LSAN:       }
+#[no_mangle]
+pub fn test(n: &mut u32) {
+    random_inline(n);
+}
+
+#[sanitize(address = "off")]
+#[inline]
+#[no_mangle]
+pub fn random_inline(n: &mut u32) {
+    *n = 42;
+}
diff --git a/tests/codegen-llvm/sanitizer/sanitize-off-kasan-asan.rs b/tests/codegen-llvm/sanitizer/sanitize-off-kasan-asan.rs
new file mode 100644
index 000000000000..61ad0ba7d90d
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/sanitize-off-kasan-asan.rs
@@ -0,0 +1,29 @@
+// Verifies that the `#[sanitize(kernel_address = "off")]` attribute also turns off
+// the address sanitizer.
+//
+//@ needs-sanitizer-address
+//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static -Copt-level=0 -C unsafe-allow-abi-mismatch=sanitizer
+
+#![crate_type = "lib"]
+#![feature(sanitize)]
+
+// CHECK-LABEL: ; sanitize_off_kasan_asan::unsanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK-NOT:   sanitize_address
+// CHECK:       start:
+// CHECK-NOT:   call void @__asan_report_load
+// CHECK:       }
+#[sanitize(kernel_address = "off")]
+pub fn unsanitized(b: &mut u8) -> u8 {
+    *b
+}
+
+// CHECK-LABEL: ; sanitize_off_kasan_asan::sanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK:       sanitize_address
+// CHECK:       start:
+// CHECK:       call void @__asan_report_load
+// CHECK:       }
+pub fn sanitized(b: &mut u8) -> u8 {
+    *b
+}
diff --git a/tests/codegen-llvm/sanitizer/sanitize-off.rs b/tests/codegen-llvm/sanitizer/sanitize-off.rs
new file mode 100644
index 000000000000..9f3f7cd9df78
--- /dev/null
+++ b/tests/codegen-llvm/sanitizer/sanitize-off.rs
@@ -0,0 +1,138 @@
+// Verifies that the `#[sanitize(address = "off")]` attribute can be used to
+// selectively disable sanitizer instrumentation.
+//
+//@ needs-sanitizer-address
+//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static -Copt-level=0
+
+#![crate_type = "lib"]
+#![feature(sanitize)]
+
+// CHECK:     @UNSANITIZED = constant{{.*}} no_sanitize_address
+// CHECK-NOT: @__asan_global_SANITIZED
+#[no_mangle]
+#[sanitize(address = "off")]
+pub static UNSANITIZED: u32 = 0;
+
+// CHECK: @__asan_global_SANITIZED
+#[no_mangle]
+pub static SANITIZED: u32 = 0;
+
+// CHECK-LABEL: ; sanitize_off::unsanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK-NOT:   sanitize_address
+// CHECK:       start:
+// CHECK-NOT:   call void @__asan_report_load
+// CHECK:       }
+#[sanitize(address = "off")]
+pub fn unsanitized(b: &mut u8) -> u8 {
+    *b
+}
+
+// CHECK-LABEL: ; sanitize_off::sanitized
+// CHECK-NEXT:  ; Function Attrs:
+// CHECK:       sanitize_address
+// CHECK:       start:
+// CHECK:       call void @__asan_report_load
+// CHECK:       }
+pub fn sanitized(b: &mut u8) -> u8 {
+    *b
+}
+
+#[sanitize(address = "off")]
+pub mod foo {
+    // CHECK-LABEL: ; sanitize_off::foo::unsanitized
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK-NOT:   sanitize_address
+    // CHECK:       start:
+    // CHECK-NOT:   call void @__asan_report_load
+    // CHECK:       }
+    pub fn unsanitized(b: &mut u8) -> u8 {
+        *b
+    }
+
+    // CHECK-LABEL: ; sanitize_off::foo::sanitized
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK:       sanitize_address
+    // CHECK:       start:
+    // CHECK:       call void @__asan_report_load
+    // CHECK:       }
+    #[sanitize(address = "on")]
+    pub fn sanitized(b: &mut u8) -> u8 {
+        *b
+    }
+}
+
+pub trait MyTrait {
+    fn unsanitized(&self, b: &mut u8) -> u8;
+    fn sanitized(&self, b: &mut u8) -> u8;
+
+    // CHECK-LABEL: ; sanitize_off::MyTrait::unsanitized_default
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK-NOT:   sanitize_address
+    // CHECK:       start:
+    // CHECK-NOT:   call void @__asan_report_load
+    // CHECK:       }
+    #[sanitize(address = "off")]
+    fn unsanitized_default(&self, b: &mut u8) -> u8 {
+        *b
+    }
+
+    // CHECK-LABEL: ; sanitize_off::MyTrait::sanitized_default
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK:       sanitize_address
+    // CHECK:       start:
+    // CHECK:       call void @__asan_report_load
+    // CHECK:       }
+    fn sanitized_default(&self, b: &mut u8) -> u8 {
+        *b
+    }
+}
+
+#[sanitize(address = "off")]
+impl MyTrait for () {
+    // CHECK-LABEL: ; <() as sanitize_off::MyTrait>::unsanitized
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK-NOT:   sanitize_address
+    // CHECK:       start:
+    // CHECK-NOT:   call void @__asan_report_load
+    // CHECK:       }
+    fn unsanitized(&self, b: &mut u8) -> u8 {
+        *b
+    }
+
+    // CHECK-LABEL: ; <() as sanitize_off::MyTrait>::sanitized
+    // CHECK-NEXT:  ; Function Attrs:
+    // CHECK:       sanitize_address
+    // CHECK:       start:
+    // CHECK:       call void @__asan_report_load
+    // CHECK:       }
+    #[sanitize(address = "on")]
+    fn sanitized(&self, b: &mut u8) -> u8 {
+        *b
+    }
+}
+
+pub fn expose_trait(b: &mut u8) -> u8 {
+    <() as MyTrait>::unsanitized_default(&(), b);
+    <() as MyTrait>::sanitized_default(&(), b)
+}
+
+#[sanitize(address = "off")]
+pub mod outer {
+    #[sanitize(thread = "off")]
+    pub mod inner {
+        // CHECK-LABEL: ; sanitize_off::outer::inner::unsanitized
+        // CHECK-NEXT:  ; Function Attrs:
+        // CHECK-NOT:   sanitize_address
+        // CHECK:       start:
+        // CHECK-NOT:   call void @__asan_report_load
+        // CHECK:       }
+        pub fn unsanitized() {
+            let xs = [0, 1, 2, 3];
+            // Avoid optimizing everything out.
+            let xs = std::hint::black_box(xs.as_ptr());
+            let code = unsafe { *xs.offset(4) };
+            std::process::exit(code);
+        }
+    }
+}
diff --git a/tests/codegen-llvm/sanitizer/sanitizer-recover.rs b/tests/codegen-llvm/sanitizer/sanitizer-recover.rs
index 6b6593204818..b8a24e31c30b 100644
--- a/tests/codegen-llvm/sanitizer/sanitizer-recover.rs
+++ b/tests/codegen-llvm/sanitizer/sanitizer-recover.rs
@@ -5,7 +5,7 @@
 //@ needs-sanitizer-memory
 //@ revisions:ASAN ASAN-RECOVER MSAN MSAN-RECOVER MSAN-RECOVER-LTO
 //@ no-prefer-dynamic
-//
+//@                   compile-flags: -C unsafe-allow-abi-mismatch=sanitizer
 //@                   compile-flags: -Ctarget-feature=-crt-static
 //@[ASAN]             compile-flags: -Zsanitizer=address -Copt-level=0
 //@[ASAN-RECOVER]     compile-flags: -Zsanitizer=address -Zsanitizer-recover=address -Copt-level=0
diff --git a/tests/codegen-llvm/sanitizer/scs-attr-check.rs b/tests/codegen-llvm/sanitizer/scs-attr-check.rs
index 6f4cbc2c0a6b..f726503503c9 100644
--- a/tests/codegen-llvm/sanitizer/scs-attr-check.rs
+++ b/tests/codegen-llvm/sanitizer/scs-attr-check.rs
@@ -5,7 +5,7 @@
 //@ compile-flags: -Zsanitizer=shadow-call-stack
 
 #![crate_type = "lib"]
-#![feature(no_sanitize)]
+#![feature(sanitize)]
 
 // CHECK: ; sanitizer_scs_attr_check::scs
 // CHECK-NEXT: ; Function Attrs:{{.*}}shadowcallstack
@@ -13,5 +13,5 @@ pub fn scs() {}
 
 // CHECK: ; sanitizer_scs_attr_check::no_scs
 // CHECK-NOT: ; Function Attrs:{{.*}}shadowcallstack
-#[no_sanitize(shadow_call_stack)]
+#[sanitize(shadow_call_stack = "off")]
 pub fn no_scs() {}
diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-gather.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-gather.rs
index 690bfb432f9b..d79a9cf96ada 100644
--- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-gather.rs
+++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-gather.rs
@@ -1,6 +1,10 @@
 //
 
 //@ compile-flags: -C no-prepopulate-passes
+//@ revisions: LLVM21 LLVM22
+//@ [LLVM22] min-llvm-version: 22
+//@ [LLVM21] max-llvm-major-version: 21
+// ignore-tidy-linelength
 
 #![crate_type = "lib"]
 #![feature(repr_simd, core_intrinsics)]
@@ -24,7 +28,8 @@ pub unsafe fn gather_f32x2(
 ) -> Vec2 {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM21: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM22: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
     simd_gather(values, pointers, mask)
 }
 
@@ -37,7 +42,8 @@ pub unsafe fn gather_f32x2_unsigned(
 ) -> Vec2 {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM21: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM22: call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
     simd_gather(values, pointers, mask)
 }
 
@@ -50,6 +56,7 @@ pub unsafe fn gather_pf32x2(
 ) -> Vec2<*const f32> {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
+    // LLVM21: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
+    // LLVM22: call <2 x ptr> @llvm.masked.gather.v2p0.v2p0(<2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]], <2 x ptr> {{.*}})
     simd_gather(values, pointers, mask)
 }
diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
index fda315dc66ca..7df73cb9b4d7 100644
--- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
+++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-load.rs
@@ -1,4 +1,8 @@
 //@ compile-flags: -C no-prepopulate-passes
+//@ revisions: LLVM21 LLVM22
+//@ [LLVM22] min-llvm-version: 22
+//@ [LLVM21] max-llvm-major-version: 21
+// ignore-tidy-linelength
 
 #![crate_type = "lib"]
 #![feature(repr_simd, core_intrinsics)]
@@ -18,7 +22,8 @@ pub type Vec4 = Simd;
 pub unsafe fn load_f32x2(mask: Vec2, pointer: *const f32, values: Vec2) -> Vec2 {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
     simd_masked_load(mask, pointer, values)
 }
 
@@ -31,7 +36,8 @@ pub unsafe fn load_f32x2_unsigned(
 ) -> Vec2 {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM21: call <2 x float> @llvm.masked.load.v2f32.p0(ptr {{.*}}, i32 4, <2 x i1> [[B]], <2 x float> {{.*}})
+    // LLVM22: call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 4 {{.*}}, <2 x i1> [[B]], <2 x float> {{.*}})
     simd_masked_load(mask, pointer, values)
 }
 
@@ -44,6 +50,7 @@ pub unsafe fn load_pf32x4(
 ) -> Vec4<*const f32> {
     // CHECK: [[A:%[0-9]+]] = lshr <4 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
-    // CHECK: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
+    // LLVM21: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
+    // LLVM22: call <4 x ptr> @llvm.masked.load.v4p0.p0(ptr align {{.*}} {{.*}}, <4 x i1> [[B]], <4 x ptr> {{.*}})
     simd_masked_load(mask, pointer, values)
 }
diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
index 6ca7388d464b..03119d89bacd 100644
--- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
+++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-masked-store.rs
@@ -1,4 +1,8 @@
 //@ compile-flags: -C no-prepopulate-passes
+//@ revisions: LLVM21 LLVM22
+//@ [LLVM22] min-llvm-version: 22
+//@ [LLVM21] max-llvm-major-version: 21
+// ignore-tidy-linelength
 
 #![crate_type = "lib"]
 #![feature(repr_simd, core_intrinsics)]
@@ -18,7 +22,8 @@ pub type Vec4 = Simd;
 pub unsafe fn store_f32x2(mask: Vec2, pointer: *mut f32, values: Vec2) {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
+    // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
+    // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
     simd_masked_store(mask, pointer, values)
 }
 
@@ -27,7 +32,8 @@ pub unsafe fn store_f32x2(mask: Vec2, pointer: *mut f32, values: Vec2)
 pub unsafe fn store_f32x2_unsigned(mask: Vec2, pointer: *mut f32, values: Vec2) {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
+    // LLVM21: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr {{.*}}, i32 4, <2 x i1> [[B]])
+    // LLVM22: call void @llvm.masked.store.v2f32.p0(<2 x float> {{.*}}, ptr align 4 {{.*}}, <2 x i1> [[B]])
     simd_masked_store(mask, pointer, values)
 }
 
@@ -36,6 +42,7 @@ pub unsafe fn store_f32x2_unsigned(mask: Vec2, pointer: *mut f32, values: V
 pub unsafe fn store_pf32x4(mask: Vec4, pointer: *mut *const f32, values: Vec4<*const f32>) {
     // CHECK: [[A:%[0-9]+]] = lshr <4 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <4 x i32> [[A]] to <4 x i1>
-    // CHECK: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]])
+    // LLVM21: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr {{.*}}, i32 {{.*}}, <4 x i1> [[B]])
+    // LLVM22: call void @llvm.masked.store.v4p0.p0(<4 x ptr> {{.*}}, ptr align {{.*}} {{.*}}, <4 x i1> [[B]])
     simd_masked_store(mask, pointer, values)
 }
diff --git a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-scatter.rs b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-scatter.rs
index 743652966e16..533fa429ae70 100644
--- a/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-scatter.rs
+++ b/tests/codegen-llvm/simd-intrinsic/simd-intrinsic-generic-scatter.rs
@@ -1,6 +1,10 @@
 //
 
 //@ compile-flags: -C no-prepopulate-passes
+//@ revisions: LLVM21 LLVM22
+//@ [LLVM22] min-llvm-version: 22
+//@ [LLVM21] max-llvm-major-version: 21
+// ignore-tidy-linelength
 
 #![crate_type = "lib"]
 #![feature(repr_simd, core_intrinsics)]
@@ -20,7 +24,8 @@ pub type Vec4 = Simd;
 pub unsafe fn scatter_f32x2(pointers: Vec2<*mut f32>, mask: Vec2, values: Vec2) {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM21: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM22: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
     simd_scatter(values, pointers, mask)
 }
 
@@ -29,7 +34,8 @@ pub unsafe fn scatter_f32x2(pointers: Vec2<*mut f32>, mask: Vec2, values: V
 pub unsafe fn scatter_f32x2_unsigned(pointers: Vec2<*mut f32>, mask: Vec2, values: Vec2) {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM21: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM22: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
     simd_scatter(values, pointers, mask)
 }
 
@@ -42,6 +48,7 @@ pub unsafe fn scatter_pf32x2(
 ) {
     // CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{|splat \(i32 31\)}}
     // CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
-    // CHECK: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM21: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
+    // LLVM22: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> align {{.*}} {{.*}}, <2 x i1> [[B]]
     simd_scatter(values, pointers, mask)
 }
diff --git a/tests/codegen-llvm/simd/extract-insert-dyn.rs b/tests/codegen-llvm/simd/extract-insert-dyn.rs
index 729f0145314a..9c17b82e5535 100644
--- a/tests/codegen-llvm/simd/extract-insert-dyn.rs
+++ b/tests/codegen-llvm/simd/extract-insert-dyn.rs
@@ -5,7 +5,8 @@
     repr_simd,
     arm_target_feature,
     mips_target_feature,
-    s390x_target_feature
+    s390x_target_feature,
+    riscv_target_feature
 )]
 #![no_std]
 #![crate_type = "lib"]
@@ -25,97 +26,105 @@ pub struct u32x16([u32; 16]);
 pub struct i8x16([i8; 16]);
 
 // CHECK-LABEL: dyn_simd_extract
-// CHECK: extractelement <16 x i8> %x, i32 %idx
+// CHECK: extractelement <16 x i8> %[[TEMP:.+]], i32 %idx
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn dyn_simd_extract(x: i8x16, idx: u32) -> i8 {
     simd_extract_dyn(x, idx)
 }
 
 // CHECK-LABEL: literal_dyn_simd_extract
-// CHECK: extractelement <16 x i8> %x, i32 7
+// CHECK: extractelement <16 x i8> %[[TEMP:.+]], i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn literal_dyn_simd_extract(x: i8x16) -> i8 {
     simd_extract_dyn(x, 7)
 }
 
 // CHECK-LABEL: const_dyn_simd_extract
-// CHECK: extractelement <16 x i8> %x, i32 7
+// CHECK: extractelement <16 x i8> %[[TEMP:.+]], i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn const_dyn_simd_extract(x: i8x16) -> i8 {
     simd_extract_dyn(x, const { 3 + 4 })
 }
 
 // CHECK-LABEL: const_simd_extract
-// CHECK: extractelement <16 x i8> %x, i32 7
+// CHECK: extractelement <16 x i8> %[[TEMP:.+]], i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn const_simd_extract(x: i8x16) -> i8 {
     simd_extract(x, const { 3 + 4 })
 }
 
 // CHECK-LABEL: dyn_simd_insert
-// CHECK: insertelement <16 x i8> %x, i8 %e, i32 %idx
+// CHECK: insertelement <16 x i8> %[[TEMP:.+]], i8 %e, i32 %idx
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn dyn_simd_insert(x: i8x16, e: i8, idx: u32) -> i8x16 {
     simd_insert_dyn(x, idx, e)
 }
 
 // CHECK-LABEL: literal_dyn_simd_insert
-// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+// CHECK: insertelement <16 x i8> %[[TEMP:.+]], i8 %e, i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn literal_dyn_simd_insert(x: i8x16, e: i8) -> i8x16 {
     simd_insert_dyn(x, 7, e)
 }
 
 // CHECK-LABEL: const_dyn_simd_insert
-// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+// CHECK: insertelement <16 x i8> %[[TEMP:.+]], i8 %e, i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn const_dyn_simd_insert(x: i8x16, e: i8) -> i8x16 {
     simd_insert_dyn(x, const { 3 + 4 }, e)
 }
 
 // CHECK-LABEL: const_simd_insert
-// CHECK: insertelement <16 x i8> %x, i8 %e, i32 7
+// CHECK: insertelement <16 x i8> %[[TEMP:.+]], i8 %e, i32 7
 #[no_mangle]
 #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
 #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
 #[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
 #[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
+#[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))]
 unsafe extern "C" fn const_simd_insert(x: i8x16, e: i8) -> i8x16 {
     simd_insert(x, const { 3 + 4 }, e)
 }
diff --git a/tests/codegen-llvm/slice-last-elements-optimization.rs b/tests/codegen-llvm/slice-last-elements-optimization.rs
index b90f91d7b17b..77fc1d21cd90 100644
--- a/tests/codegen-llvm/slice-last-elements-optimization.rs
+++ b/tests/codegen-llvm/slice-last-elements-optimization.rs
@@ -1,19 +1,17 @@
 //@ compile-flags: -Copt-level=3
-//@ only-x86_64
-//@ min-llvm-version: 20
 #![crate_type = "lib"]
 
 // This test verifies that LLVM 20 properly optimizes the bounds check
 // when accessing the last few elements of a slice with proper conditions.
 // Previously, this would generate an unreachable branch to
-// slice_start_index_len_fail even when the bounds check was provably safe.
+// slice_index_fail even when the bounds check was provably safe.
 
 // CHECK-LABEL: @last_four_initial(
 #[no_mangle]
 pub fn last_four_initial(s: &[u8]) -> &[u8] {
-    // Previously this would generate a branch to slice_start_index_len_fail
+    // Previously this would generate a branch to slice_index_fail
     // that is unreachable. The LLVM 20 fix should eliminate this branch.
-    // CHECK-NOT: slice_start_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: unreachable
     let start = if s.len() <= 4 { 0 } else { s.len() - 4 };
     &s[start..]
@@ -23,7 +21,7 @@ pub fn last_four_initial(s: &[u8]) -> &[u8] {
 #[no_mangle]
 pub fn last_four_optimized(s: &[u8]) -> &[u8] {
     // This version was already correctly optimized before the fix in LLVM 20.
-    // CHECK-NOT: slice_start_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK-NOT: unreachable
     if s.len() <= 4 { &s[0..] } else { &s[s.len() - 4..] }
 }
@@ -32,6 +30,6 @@ pub fn last_four_optimized(s: &[u8]) -> &[u8] {
 // CHECK-LABEL: @test_bounds_check_happens(
 #[no_mangle]
 pub fn test_bounds_check_happens(s: &[u8], i: usize) -> &[u8] {
-    // CHECK: slice_start_index_len_fail
+    // CHECK: slice_index_fail
     &s[i..]
 }
diff --git a/tests/codegen-llvm/slice-reverse.rs b/tests/codegen-llvm/slice-reverse.rs
index e58d1c1d9d8e..c31cff5010b4 100644
--- a/tests/codegen-llvm/slice-reverse.rs
+++ b/tests/codegen-llvm/slice-reverse.rs
@@ -8,10 +8,10 @@
 #[no_mangle]
 pub fn slice_reverse_u8(slice: &mut [u8]) {
     // CHECK-NOT: panic_bounds_check
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK: shufflevector <{{[0-9]+}} x i8>
     // CHECK-NOT: panic_bounds_check
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     slice.reverse();
 }
 
@@ -19,9 +19,9 @@ pub fn slice_reverse_u8(slice: &mut [u8]) {
 #[no_mangle]
 pub fn slice_reverse_i32(slice: &mut [i32]) {
     // CHECK-NOT: panic_bounds_check
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     // CHECK: shufflevector <{{[0-9]+}} x i32>
     // CHECK-NOT: panic_bounds_check
-    // CHECK-NOT: slice_end_index_len_fail
+    // CHECK-NOT: slice_index_fail
     slice.reverse();
 }
diff --git a/tests/codegen-llvm/swap-small-types.rs b/tests/codegen-llvm/swap-small-types.rs
index 7aa613ae9c22..0799ff76331d 100644
--- a/tests/codegen-llvm/swap-small-types.rs
+++ b/tests/codegen-llvm/swap-small-types.rs
@@ -1,6 +1,5 @@
 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
 //@ only-x86_64
-//@ min-llvm-version: 20
 //@ ignore-std-debug-assertions (`ptr::swap_nonoverlapping` has one which blocks some optimizations)
 
 #![crate_type = "lib"]
diff --git a/tests/codegen-llvm/transmute-scalar.rs b/tests/codegen-llvm/transmute-scalar.rs
index ce1b0558b2ee..e1ce8e506066 100644
--- a/tests/codegen-llvm/transmute-scalar.rs
+++ b/tests/codegen-llvm/transmute-scalar.rs
@@ -1,8 +1,9 @@
 //@ add-core-stubs
-//@ compile-flags: -C opt-level=0 -C no-prepopulate-passes
+//@ compile-flags: -C opt-level=0 -C no-prepopulate-passes --target=x86_64-unknown-linux-gnu
+//@ needs-llvm-components: x86
 
 #![crate_type = "lib"]
-#![feature(no_core, repr_simd, arm_target_feature, mips_target_feature, s390x_target_feature)]
+#![feature(no_core, repr_simd)]
 #![no_core]
 extern crate minicore;
 
@@ -55,7 +56,7 @@ pub fn ptr_to_int(p: *mut u16) -> usize {
 }
 
 // CHECK: define{{.*}}ptr @int_to_ptr([[USIZE]] %i)
-// CHECK: %_0 = getelementptr i8, ptr null, [[USIZE]] %i
+// CHECK: %_0 = inttoptr [[USIZE]] %i to ptr
 // CHECK-NEXT: ret ptr %_0
 #[no_mangle]
 pub fn int_to_ptr(i: usize) -> *mut u16 {
@@ -117,11 +118,7 @@ struct S([i64; 1]);
 // CHECK-NEXT: %[[TEMP:.+]] = load i64, ptr %[[RET]]
 // CHECK-NEXT: ret i64 %[[TEMP]]
 #[no_mangle]
-#[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
-#[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
-#[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
-#[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
 pub extern "C" fn single_element_simd_to_scalar(b: S) -> i64 {
     unsafe { mem::transmute(b) }
 }
@@ -133,11 +130,7 @@ pub extern "C" fn single_element_simd_to_scalar(b: S) -> i64 {
 // CHECK-NEXT: %[[TEMP:.+]] = load <1 x i64>, ptr %[[RET]]
 // CHECK-NEXT: ret <1 x i64> %[[TEMP]]
 #[no_mangle]
-#[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))]
-#[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))]
 #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))]
-#[cfg_attr(target_arch = "mips", target_feature(enable = "msa"))]
-#[cfg_attr(target_arch = "s390x", target_feature(enable = "vector"))]
 pub extern "C" fn scalar_to_single_element_simd(b: i64) -> S {
     unsafe { mem::transmute(b) }
 }
diff --git a/tests/codegen-llvm/try_question_mark_nop.rs b/tests/codegen-llvm/try_question_mark_nop.rs
index 398c9a580bc3..a09fa0a49019 100644
--- a/tests/codegen-llvm/try_question_mark_nop.rs
+++ b/tests/codegen-llvm/try_question_mark_nop.rs
@@ -1,9 +1,6 @@
 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
 //@ edition: 2021
 //@ only-x86_64
-//@ revisions: NINETEEN TWENTY
-//@[NINETEEN] exact-llvm-major-version: 19
-//@[TWENTY] min-llvm-version: 20
 
 #![crate_type = "lib"]
 #![feature(try_blocks)]
@@ -17,13 +14,9 @@ pub fn option_nop_match_32(x: Option) -> Option {
     // CHECK: start:
     // CHECK-NEXT: [[TRUNC:%.*]] = trunc nuw i32 %0 to i1
 
-    // NINETEEN-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i32 %0, i32 0
-    // NINETEEN-NEXT: [[REG2:%.*]] = insertvalue { i32, i32 } poison, i32 [[SELECT]], 0
-    // NINETEEN-NEXT: [[REG3:%.*]] = insertvalue { i32, i32 } [[REG2]], i32 %1, 1
-
-    // TWENTY-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i32 %1, i32 undef
-    // TWENTY-NEXT: [[REG2:%.*]] = insertvalue { i32, i32 } poison, i32 %0, 0
-    // TWENTY-NEXT: [[REG3:%.*]] = insertvalue { i32, i32 } [[REG2]], i32 [[SELECT]], 1
+    // CHECK-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i32 %1, i32 undef
+    // CHECK-NEXT: [[REG2:%.*]] = insertvalue { i32, i32 } poison, i32 %0, 0
+    // CHECK-NEXT: [[REG3:%.*]] = insertvalue { i32, i32 } [[REG2]], i32 [[SELECT]], 1
 
     // CHECK-NEXT: ret { i32, i32 } [[REG3]]
     match x {
@@ -36,8 +29,8 @@ pub fn option_nop_match_32(x: Option) -> Option {
 #[no_mangle]
 pub fn option_nop_traits_32(x: Option) -> Option {
     // CHECK: start:
-    // TWENTY-NEXT: %[[IS_SOME:.+]] = trunc nuw i32 %0 to i1
-    // TWENTY-NEXT: select i1 %[[IS_SOME]], i32 %1, i32 undef
+    // CHECK-NEXT: %[[IS_SOME:.+]] = trunc nuw i32 %0 to i1
+    // CHECK-NEXT: select i1 %[[IS_SOME]], i32 %1, i32 undef
     // CHECK-NEXT: insertvalue { i32, i32 }
     // CHECK-NEXT: insertvalue { i32, i32 }
     // CHECK-NEXT: ret { i32, i32 }
@@ -96,13 +89,9 @@ pub fn option_nop_match_64(x: Option) -> Option {
     // CHECK: start:
     // CHECK-NEXT: [[TRUNC:%.*]] = trunc nuw i64 %0 to i1
 
-    // NINETEEN-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i64 %0, i64 0
-    // NINETEEN-NEXT: [[REG2:%.*]] = insertvalue { i64, i64 } poison, i64 [[SELECT]], 0
-    // NINETEEN-NEXT: [[REG3:%.*]] = insertvalue { i64, i64 } [[REG2]], i64 %1, 1
-
-    // TWENTY-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i64 %1, i64 undef
-    // TWENTY-NEXT: [[REG2:%.*]] = insertvalue { i64, i64 } poison, i64 %0, 0
-    // TWENTY-NEXT: [[REG3:%.*]] = insertvalue { i64, i64 } [[REG2]], i64 [[SELECT]], 1
+    // CHECK-NEXT: [[SELECT:%.*]] = select i1 [[TRUNC]], i64 %1, i64 undef
+    // CHECK-NEXT: [[REG2:%.*]] = insertvalue { i64, i64 } poison, i64 %0, 0
+    // CHECK-NEXT: [[REG3:%.*]] = insertvalue { i64, i64 } [[REG2]], i64 [[SELECT]], 1
 
     // CHECK-NEXT: ret { i64, i64 } [[REG3]]
     match x {
@@ -115,8 +104,8 @@ pub fn option_nop_match_64(x: Option) -> Option {
 #[no_mangle]
 pub fn option_nop_traits_64(x: Option) -> Option {
     // CHECK: start:
-    // TWENTY-NEXT: %[[TRUNC:[0-9]+]] = trunc nuw i64 %0 to i1
-    // TWENTY-NEXT: %[[SEL:\.[0-9]+]] = select i1 %[[TRUNC]], i64 %1, i64 undef
+    // CHECK-NEXT: %[[TRUNC:[0-9]+]] = trunc nuw i64 %0 to i1
+    // CHECK-NEXT: %[[SEL:\.[0-9]+]] = select i1 %[[TRUNC]], i64 %1, i64 undef
     // CHECK-NEXT: insertvalue { i64, i64 }
     // CHECK-NEXT: insertvalue { i64, i64 }
     // CHECK-NEXT: ret { i64, i64 }
diff --git a/tests/codegen-llvm/uninhabited-transparent-return-abi.rs b/tests/codegen-llvm/uninhabited-transparent-return-abi.rs
index face1577c3f6..507cd7ae2a67 100644
--- a/tests/codegen-llvm/uninhabited-transparent-return-abi.rs
+++ b/tests/codegen-llvm/uninhabited-transparent-return-abi.rs
@@ -23,7 +23,7 @@ extern "Rust" {
 #[no_mangle]
 pub fn test_uninhabited_ret_by_ref() {
     // CHECK: %_1 = alloca [24 x i8], align {{8|4}}
-    // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %_1)
+    // CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 24, )?}}ptr nonnull %_1)
     // CHECK-NEXT: call void @opaque({{.*}} sret([24 x i8]) {{.*}} %_1) #2
     // CHECK-NEXT: unreachable
     unsafe {
@@ -35,8 +35,8 @@ pub fn test_uninhabited_ret_by_ref() {
 #[no_mangle]
 pub fn test_uninhabited_ret_by_ref_with_arg(rsi: u32) {
     // CHECK: %_2 = alloca [24 x i8], align {{8|4}}
-    // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %_2)
-    // CHECK-NEXT: call void @opaque_with_arg({{.*}} sret([24 x i8]) {{.*}} %_2, i32 noundef %rsi) #2
+    // CHECK-NEXT: call void @llvm.lifetime.start.p0({{(i64 24, )?}}ptr nonnull %_2)
+    // CHECK-NEXT: call void @opaque_with_arg({{.*}} sret([24 x i8]) {{.*}} %_2, i32 noundef{{( signext)?}} %rsi) #2
     // CHECK-NEXT: unreachable
     unsafe {
         opaque_with_arg(rsi);
diff --git a/tests/codegen-llvm/union-aggregate.rs b/tests/codegen-llvm/union-aggregate.rs
index aac66c5dcdd9..7faa66804fea 100644
--- a/tests/codegen-llvm/union-aggregate.rs
+++ b/tests/codegen-llvm/union-aggregate.rs
@@ -1,5 +1,4 @@
 //@ compile-flags: -Copt-level=0 -Cno-prepopulate-passes
-//@ min-llvm-version: 19
 //@ only-64bit
 
 #![crate_type = "lib"]
diff --git a/tests/codegen-llvm/unwind-abis/c-unwind-abi-panic-abort.rs b/tests/codegen-llvm/unwind-abis/c-unwind-abi-panic-abort.rs
index 8d2745ba2f7a..2ce1d1b2e00e 100644
--- a/tests/codegen-llvm/unwind-abis/c-unwind-abi-panic-abort.rs
+++ b/tests/codegen-llvm/unwind-abis/c-unwind-abi-panic-abort.rs
@@ -1,4 +1,9 @@
 //@ compile-flags: -C panic=abort
+//@ revisions: NONWASM WASM WASMEXN
+//@ [NONWASM] ignore-wasm32
+//@ [WASM] only-wasm32
+//@ [WASMEXN] only-wasm32
+//@ [WASMEXN] compile-flags: -Ctarget-feature=+exception-handling
 
 // Test that `nounwind` attributes are also applied to extern `C-unwind` Rust functions
 // when the code is compiled with `panic=abort`.
@@ -9,7 +14,9 @@
 #[no_mangle]
 pub unsafe extern "C-unwind" fn rust_item_that_can_unwind() {
     // Handle both legacy and v0 symbol mangling.
-    // CHECK: call void @{{.*core9panicking19panic_cannot_unwind}}
+    // NONWASM: call void @{{.*core9panicking19panic_cannot_unwind}}
+    // WASMEXN: call void @{{.*core9panicking19panic_cannot_unwind}}
+    // WASM-NOT: call void @{{.*core9panicking19panic_cannot_unwind}}
     may_unwind();
 }
 
diff --git a/tests/codegen-llvm/unwind-and-panic-abort.rs b/tests/codegen-llvm/unwind-and-panic-abort.rs
index 8efa140058ad..c2838be00afa 100644
--- a/tests/codegen-llvm/unwind-and-panic-abort.rs
+++ b/tests/codegen-llvm/unwind-and-panic-abort.rs
@@ -1,4 +1,9 @@
 //@ compile-flags: -C panic=abort
+//@ revisions: NONWASM WASM WASMEXN
+//@ [NONWASM] ignore-wasm32
+//@ [WASM] only-wasm32
+//@ [WASMEXN] only-wasm32
+//@ [WASMEXN] compile-flags: -Ctarget-feature=+exception-handling
 
 #![crate_type = "lib"]
 
@@ -9,7 +14,9 @@ extern "C-unwind" {
 // CHECK: Function Attrs:{{.*}}nounwind
 // CHECK-NEXT: define{{.*}}void @foo
 // Handle both legacy and v0 symbol mangling.
-// CHECK: call void @{{.*core9panicking19panic_cannot_unwind}}
+// NONWASM: call void @{{.*core9panicking19panic_cannot_unwind}}
+// WASMEXN: call void @{{.*core9panicking19panic_cannot_unwind}}
+// WASM-NOT: call void @{{.*core9panicking19panic_cannot_unwind}}
 #[no_mangle]
 pub unsafe extern "C" fn foo() {
     bar();
diff --git a/tests/codegen-llvm/vec-calloc.rs b/tests/codegen-llvm/vec-calloc.rs
index d1c320ead012..15971bbfa003 100644
--- a/tests/codegen-llvm/vec-calloc.rs
+++ b/tests/codegen-llvm/vec-calloc.rs
@@ -1,4 +1,6 @@
+//@ revisions: normal llvm21
 //@ compile-flags: -Copt-level=3 -Z merge-functions=disabled
+//@ [llvm21] min-llvm-version: 21
 //@ only-x86_64
 
 #![crate_type = "lib"]
@@ -176,6 +178,24 @@ pub fn vec_option_i32(n: usize) -> Vec> {
     vec![None; n]
 }
 
+// LLVM21-LABEL: @vec_array
+#[cfg(llvm21)]
+#[no_mangle]
+pub fn vec_array(n: usize) -> Vec<[u32; 1_000_000]> {
+    // LLVM21-NOT: call {{.*}}alloc::vec::from_elem
+    // LLVM21-NOT: call {{.*}}reserve
+    // LLVM21-NOT: call {{.*}}__rust_alloc(
+
+    // LLVM21: call {{.*}}__rust_alloc_zeroed(
+
+    // LLVM21-NOT: call {{.*}}alloc::vec::from_elem
+    // LLVM21-NOT: call {{.*}}reserve
+    // LLVM21-NOT: call {{.*}}__rust_alloc(
+
+    // LLVM21: ret void
+    vec![[0; 1_000_000]; 3]
+}
+
 // Ensure that __rust_alloc_zeroed gets the right attributes for LLVM to optimize it away.
 // CHECK: declare noalias noundef ptr @{{.*}}__rust_alloc_zeroed(i64 noundef, i64 allocalign noundef) unnamed_addr [[RUST_ALLOC_ZEROED_ATTRS:#[0-9]+]]
 
diff --git a/tests/codegen-llvm/vec_pop_push_noop.rs b/tests/codegen-llvm/vec_pop_push_noop.rs
index 3e375219fe01..977c220b3bae 100644
--- a/tests/codegen-llvm/vec_pop_push_noop.rs
+++ b/tests/codegen-llvm/vec_pop_push_noop.rs
@@ -1,4 +1,7 @@
 //@ compile-flags: -Copt-level=3
+//@ revisions: new old
+//@ [old] max-llvm-major-version: 21
+//@ [new] min-llvm-version: 22
 
 #![crate_type = "lib"]
 
@@ -7,7 +10,7 @@
 pub fn noop(v: &mut Vec) {
     // CHECK-NOT: grow_one
     // CHECK-NOT: call
-    // CHECK: tail call void @llvm.assume
+    // old: tail call void @llvm.assume
     // CHECK-NOT: grow_one
     // CHECK-NOT: call
     // CHECK: {{ret|[}]}}
diff --git a/tests/codegen-llvm/vecdeque_pop_push.rs b/tests/codegen-llvm/vecdeque_pop_push.rs
index 5afa1b2248b0..6f9ad6674d6c 100644
--- a/tests/codegen-llvm/vecdeque_pop_push.rs
+++ b/tests/codegen-llvm/vecdeque_pop_push.rs
@@ -1,4 +1,7 @@
 //@ compile-flags: -Copt-level=3
+//@ revisions: new old
+//@ [old] max-llvm-major-version: 21
+//@ [new] min-llvm-version: 22
 
 #![crate_type = "lib"]
 
@@ -8,7 +11,7 @@ use std::collections::VecDeque;
 // CHECK-LABEL: @noop_back(
 pub fn noop_back(v: &mut VecDeque) {
     // CHECK-NOT: grow
-    // CHECK: tail call void @llvm.assume
+    // old: tail call void @llvm.assume
     // CHECK-NOT: grow
     // CHECK: ret
     if let Some(x) = v.pop_back() {
diff --git a/tests/codegen-llvm/wasm_exceptions.rs b/tests/codegen-llvm/wasm_exceptions.rs
index 796b69b722b5..7f38b669a6f8 100644
--- a/tests/codegen-llvm/wasm_exceptions.rs
+++ b/tests/codegen-llvm/wasm_exceptions.rs
@@ -1,8 +1,9 @@
 //@ only-wasm32
-//@ compile-flags: -C panic=unwind -Z emscripten-wasm-eh
+//@ revisions: WASM WASMEXN
+//@ [WASMEXN] compile-flags: -C panic=unwind -Z emscripten-wasm-eh
 
 #![crate_type = "lib"]
-#![feature(core_intrinsics, wasm_exception_handling_intrinsics)]
+#![feature(core_intrinsics, link_llvm_intrinsics)]
 
 extern "C-unwind" {
     fn may_panic();
@@ -22,7 +23,8 @@ impl Drop for LogOnDrop {
     }
 }
 
-// CHECK-LABEL: @test_cleanup() {{.*}} @__gxx_wasm_personality_v0
+// WASM-LABEL: @test_cleanup() {{.*}}
+// WASMEXN-LABEL: @test_cleanup() {{.*}} @__gxx_wasm_personality_v0
 #[no_mangle]
 pub fn test_cleanup() {
     let _log_on_drop = LogOnDrop;
@@ -30,12 +32,16 @@ pub fn test_cleanup() {
         may_panic();
     }
 
-    // CHECK-NOT: call
-    // CHECK: invoke void @may_panic()
-    // CHECK: %cleanuppad = cleanuppad within none []
+    // WASMEXN-NOT: call
+    // WASMEXN: invoke void @may_panic()
+    // WASMEXN: %cleanuppad = cleanuppad within none []
+    //
+    // WASM: call void @may_panic()
+    // WASM-NOT: invoke void @may_panic()
 }
 
-// CHECK-LABEL: @test_rtry() {{.*}} @__gxx_wasm_personality_v0
+// WASM-LABEL: @test_rtry() {{.*}}
+// WASMEXN-LABEL: @test_rtry() {{.*}} @__gxx_wasm_personality_v0
 #[no_mangle]
 pub fn test_rtry() {
     unsafe {
@@ -51,23 +57,40 @@ pub fn test_rtry() {
         );
     }
 
-    // CHECK-NOT: call
-    // CHECK: invoke void @may_panic()
-    // CHECK: {{.*}} = catchswitch within none [label {{.*}}] unwind to caller
-    // CHECK: {{.*}} = catchpad within {{.*}} [ptr null]
-    // CHECK: catchret
+    // WASMEXN-NOT: call
+    // WASMEXN: invoke void @may_panic()
+    // WASMEXN: {{.*}} = catchswitch within none [label {{.*}}] unwind to caller
+    // WASMEXN: {{.*}} = catchpad within {{.*}} [ptr null]
+    // WASMEXN: catchret
+
+    // WASM: call void @may_panic()
+    // WASM-NOT: invoke void @may_panic()
+    // WASM-NOT: catchswitch
+    // WASM-NOT: catchpad
+    // WASM-NOT: catchret
 }
 
 // Make sure the intrinsic is not inferred as nounwind. This is a regression test for #132416.
-// CHECK-LABEL: @test_intrinsic() {{.*}} @__gxx_wasm_personality_v0
+//
+// Note that this test uses the raw `wasm_throw` intrinsic because the one from
+// libstd was built with `-Cpanic=abort` and it's technically not valid to use
+// when this crate is compiled with `-Cpanic=unwind`.
+//
+// WASMEXN-LABEL: @test_intrinsic() {{.*}} @__gxx_wasm_personality_v0
 #[no_mangle]
+#[cfg(wasmexn)]
 pub fn test_intrinsic() {
     let _log_on_drop = LogOnDrop;
+
+    unsafe extern "C-unwind" {
+        #[link_name = "llvm.wasm.throw"]
+        fn wasm_throw(tag: i32, ptr: *mut u8) -> !;
+    }
     unsafe {
-        core::arch::wasm32::throw::<0>(core::ptr::null_mut());
+        wasm_throw(0, core::ptr::null_mut());
     }
 
-    // CHECK-NOT: call
-    // CHECK: invoke void @llvm.wasm.throw(i32 noundef 0, ptr noundef null)
-    // CHECK: %cleanuppad = cleanuppad within none []
+    // WASMEXN-NOT: call
+    // WASMEXN: invoke void @llvm.wasm.throw(i32 noundef 0, ptr noundef null)
+    // WASMEXN: %cleanuppad = cleanuppad within none []
 }
diff --git a/tests/codegen-units/item-collection/async-fn-impl.rs b/tests/codegen-units/item-collection/async-fn-impl.rs
new file mode 100644
index 000000000000..540fa0b32281
--- /dev/null
+++ b/tests/codegen-units/item-collection/async-fn-impl.rs
@@ -0,0 +1,10 @@
+//@ edition: 2024
+// When pub async fn is monomorphized, its implementation coroutine is also monomorphized
+//@ compile-flags: --crate-type=lib
+
+//~ MONO_ITEM fn async_fn @@
+//~ MONO_ITEM fn async_fn::{closure#0} @@
+#[unsafe(no_mangle)]
+pub async fn async_fn(x: u64) -> bool {
+    true
+}
diff --git a/tests/codegen-units/item-collection/opaque-return-impls.rs b/tests/codegen-units/item-collection/opaque-return-impls.rs
new file mode 100644
index 000000000000..7d5f4f5b6698
--- /dev/null
+++ b/tests/codegen-units/item-collection/opaque-return-impls.rs
@@ -0,0 +1,89 @@
+//@ only-x86_64-unknown-linux-gnu
+//@ compile-flags: -C panic=abort -Zinline-mir=no -Copt-level=0 -Zcross-crate-inline-threshold=never -Zmir-opt-level=0 -Cno-prepopulate-passes
+//@ no-prefer-dynamic
+//@ edition:2024
+#![crate_type = "lib"]
+
+trait TestTrait {
+    fn test_func(&self);
+}
+
+struct TestStruct {}
+
+impl TestTrait for TestStruct {
+    fn test_func(&self) {
+        println!("TestStruct::test_func");
+    }
+}
+
+#[inline(never)]
+pub fn foo() -> impl TestTrait {
+    TestStruct {}
+}
+
+//~ MONO_ITEM fn foo
+//~ MONO_ITEM fn ::test_func
+
+trait TestTrait2 {
+    fn test_func2(&self);
+}
+
+struct TestStruct2 {}
+
+impl TestTrait2 for TestStruct2 {
+    fn test_func2(&self) {
+        println!("TestStruct2::test_func2");
+    }
+}
+
+#[inline(never)]
+pub fn foo2() -> Box {
+    Box::new(TestStruct2 {})
+}
+
+//~ MONO_ITEM fn ::test_func2
+//~ MONO_ITEM fn alloc::alloc::exchange_malloc
+//~ MONO_ITEM fn foo2
+//~ MONO_ITEM fn std::alloc::Global::alloc_impl
+//~ MONO_ITEM fn std::boxed::Box::::new
+//~ MONO_ITEM fn std::alloc::Layout::from_size_align_unchecked::precondition_check
+//~ MONO_ITEM fn std::ptr::NonNull::::new_unchecked::precondition_check
+
+struct Counter {
+    count: usize,
+}
+
+impl Counter {
+    fn new() -> Counter {
+        Counter { count: 0 }
+    }
+}
+
+impl Iterator for Counter {
+    type Item = usize;
+
+    fn next(&mut self) -> Option {
+        self.count += 1;
+        if self.count < 6 { Some(self.count) } else { None }
+    }
+}
+
+#[inline(never)]
+pub fn foo3() -> Box> {
+    Box::new(Counter::new())
+}
+
+//~ MONO_ITEM fn ::spec_advance_by
+//~ MONO_ITEM fn ::spec_advance_by::{closure#0}
+//~ MONO_ITEM fn ::advance_by
+//~ MONO_ITEM fn ::next
+//~ MONO_ITEM fn ::nth
+//~ MONO_ITEM fn ::size_hint
+//~ MONO_ITEM fn ::try_fold::, {closure@::spec_advance_by::{closure#0}}, std::option::Option>>
+//~ MONO_ITEM fn > as std::ops::FromResidual>>::from_residual
+//~ MONO_ITEM fn > as std::ops::Try>::branch
+//~ MONO_ITEM fn > as std::ops::Try>::from_output
+//~ MONO_ITEM fn foo3
+//~ MONO_ITEM fn std::boxed::Box::::new
+//~ MONO_ITEM fn Counter::new
+//~ MONO_ITEM fn core::fmt::rt::>::new_const::<1>
diff --git a/tests/coverage/async.cov-map b/tests/coverage/async.cov-map
index c528ad525b5f..d75c4b5981a0 100644
--- a/tests/coverage/async.cov-map
+++ b/tests/coverage/async.cov-map
@@ -103,21 +103,21 @@ Number of file 0 mappings: 3
 Highest counter ID seen: (none)
 
 Function name: async::g
-Raw bytes (9): 0x[01, 01, 00, 01, 01, 1b, 01, 00, 16]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 1b, 01, 00, 12]
 Number of files: 1
 - file 0 => $DIR/async.rs
 Number of expressions: 0
 Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 27, 1) to (start + 0, 22)
+- Code(Counter(0)) at (prev + 27, 1) to (start + 0, 18)
 Highest counter ID seen: c0
 
 Function name: async::g::{closure#0} (unused)
-Raw bytes (64): 0x[01, 01, 00, 0c, 00, 1b, 17, 00, 18, 00, 01, 0b, 00, 0c, 00, 01, 09, 00, 0a, 00, 00, 0e, 00, 17, 00, 00, 1b, 00, 1c, 00, 00, 20, 00, 22, 00, 01, 09, 00, 0a, 00, 00, 0e, 00, 17, 00, 00, 1b, 00, 1c, 00, 00, 20, 00, 22, 00, 01, 0e, 00, 10, 00, 02, 01, 00, 02]
+Raw bytes (64): 0x[01, 01, 00, 0c, 00, 1b, 13, 00, 14, 00, 01, 0b, 00, 0c, 00, 01, 09, 00, 0a, 00, 00, 0e, 00, 17, 00, 00, 1b, 00, 1c, 00, 00, 20, 00, 22, 00, 01, 09, 00, 0a, 00, 00, 0e, 00, 17, 00, 00, 1b, 00, 1c, 00, 00, 20, 00, 22, 00, 01, 0e, 00, 10, 00, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => $DIR/async.rs
 Number of expressions: 0
 Number of file 0 mappings: 12
-- Code(Zero) at (prev + 27, 23) to (start + 0, 24)
+- Code(Zero) at (prev + 27, 19) to (start + 0, 20)
 - Code(Zero) at (prev + 1, 11) to (start + 0, 12)
 - Code(Zero) at (prev + 1, 9) to (start + 0, 10)
 - Code(Zero) at (prev + 0, 14) to (start + 0, 23)
diff --git a/tests/coverage/async.coverage b/tests/coverage/async.coverage
index 9fca1b6997d0..9409e6b1deb8 100644
--- a/tests/coverage/async.coverage
+++ b/tests/coverage/async.coverage
@@ -24,8 +24,8 @@
    LL|       |
    LL|      0|async fn foo() -> [bool; 10] { [false; 10] } // unused function; executor does not block on `h()`
    LL|       |
-   LL|      1|pub async fn g(x: u8) {
-                                    ^0
+   LL|      1|async fn g(x: u8) {
+                                ^0
    LL|      0|    match x {
    LL|      0|        y if e().await == y => (),
    LL|      0|        y if f().await == y => (),
diff --git a/tests/coverage/async.rs b/tests/coverage/async.rs
index da0a1c0b6f09..777ad7ce7c0c 100644
--- a/tests/coverage/async.rs
+++ b/tests/coverage/async.rs
@@ -24,7 +24,7 @@ async fn f() -> u8 { 1 }
 
 async fn foo() -> [bool; 10] { [false; 10] } // unused function; executor does not block on `h()`
 
-pub async fn g(x: u8) {
+async fn g(x: u8) {
     match x {
         y if e().await == y => (),
         y if f().await == y => (),
diff --git a/tests/coverage/async_closure.cov-map b/tests/coverage/async_closure.cov-map
index 9f8dc8d6cbba..53128dd7a48b 100644
--- a/tests/coverage/async_closure.cov-map
+++ b/tests/coverage/async_closure.cov-map
@@ -37,32 +37,29 @@ Number of file 0 mappings: 8
 Highest counter ID seen: c0
 
 Function name: async_closure::main::{closure#0}
-Raw bytes (14): 0x[01, 01, 00, 02, 01, 0b, 22, 00, 23, 01, 00, 23, 00, 24]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 0b, 22, 00, 24]
 Number of files: 1
 - file 0 => $DIR/async_closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 2
-- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 35)
-- Code(Counter(0)) at (prev + 0, 35) to (start + 0, 36)
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 36)
 Highest counter ID seen: c0
 
 Function name: async_closure::main::{closure#0}
-Raw bytes (14): 0x[01, 01, 00, 02, 01, 0b, 22, 00, 23, 01, 00, 23, 00, 24]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 0b, 22, 00, 24]
 Number of files: 1
 - file 0 => $DIR/async_closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 2
-- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 35)
-- Code(Counter(0)) at (prev + 0, 35) to (start + 0, 36)
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 36)
 Highest counter ID seen: c0
 
 Function name: async_closure::main::{closure#0}::{closure#0}::
-Raw bytes (14): 0x[01, 01, 00, 02, 01, 0b, 22, 00, 23, 01, 00, 23, 00, 24]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 0b, 22, 00, 24]
 Number of files: 1
 - file 0 => $DIR/async_closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 2
-- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 35)
-- Code(Counter(0)) at (prev + 0, 35) to (start + 0, 36)
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 11, 34) to (start + 0, 36)
 Highest counter ID seen: c0
 
diff --git a/tests/coverage/branch/no-mir-spans.cov-map b/tests/coverage/branch/no-mir-spans.cov-map
deleted file mode 100644
index 4f893cba1f81..000000000000
--- a/tests/coverage/branch/no-mir-spans.cov-map
+++ /dev/null
@@ -1,63 +0,0 @@
-Function name: no_mir_spans::while_cond
-Raw bytes (18): 0x[01, 01, 01, 05, 01, 02, 01, 10, 01, 00, 10, 20, 02, 01, 04, 0b, 00, 10]
-Number of files: 1
-- file 0 => $DIR/no-mir-spans.rs
-Number of expressions: 1
-- expression 0 operands: lhs = Counter(1), rhs = Counter(0)
-Number of file 0 mappings: 2
-- Code(Counter(0)) at (prev + 16, 1) to (start + 0, 16)
-- Branch { true: Expression(0, Sub), false: Counter(0) } at (prev + 4, 11) to (start + 0, 16)
-    true  = (c1 - c0)
-    false = c0
-Highest counter ID seen: c0
-
-Function name: no_mir_spans::while_cond_not
-Raw bytes (18): 0x[01, 01, 01, 05, 01, 02, 01, 19, 01, 00, 14, 20, 02, 01, 04, 0b, 00, 14]
-Number of files: 1
-- file 0 => $DIR/no-mir-spans.rs
-Number of expressions: 1
-- expression 0 operands: lhs = Counter(1), rhs = Counter(0)
-Number of file 0 mappings: 2
-- Code(Counter(0)) at (prev + 25, 1) to (start + 0, 20)
-- Branch { true: Expression(0, Sub), false: Counter(0) } at (prev + 4, 11) to (start + 0, 20)
-    true  = (c1 - c0)
-    false = c0
-Highest counter ID seen: c0
-
-Function name: no_mir_spans::while_op_and
-Raw bytes (31): 0x[01, 01, 04, 09, 05, 09, 01, 0f, 09, 01, 05, 03, 01, 22, 01, 00, 12, 20, 05, 02, 05, 0b, 00, 10, 20, 06, 0a, 00, 14, 00, 19]
-Number of files: 1
-- file 0 => $DIR/no-mir-spans.rs
-Number of expressions: 4
-- expression 0 operands: lhs = Counter(2), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(2), rhs = Counter(0)
-- expression 2 operands: lhs = Expression(3, Add), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(0), rhs = Counter(1)
-Number of file 0 mappings: 3
-- Code(Counter(0)) at (prev + 34, 1) to (start + 0, 18)
-- Branch { true: Counter(1), false: Expression(0, Sub) } at (prev + 5, 11) to (start + 0, 16)
-    true  = c1
-    false = (c2 - c1)
-- Branch { true: Expression(1, Sub), false: Expression(2, Sub) } at (prev + 0, 20) to (start + 0, 25)
-    true  = (c2 - c0)
-    false = ((c0 + c1) - c2)
-Highest counter ID seen: c1
-
-Function name: no_mir_spans::while_op_or
-Raw bytes (29): 0x[01, 01, 03, 09, 05, 09, 0b, 01, 05, 03, 01, 2d, 01, 00, 11, 20, 05, 02, 05, 0b, 00, 10, 20, 06, 01, 00, 14, 00, 19]
-Number of files: 1
-- file 0 => $DIR/no-mir-spans.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(2), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(2), rhs = Expression(2, Add)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(1)
-Number of file 0 mappings: 3
-- Code(Counter(0)) at (prev + 45, 1) to (start + 0, 17)
-- Branch { true: Counter(1), false: Expression(0, Sub) } at (prev + 5, 11) to (start + 0, 16)
-    true  = c1
-    false = (c2 - c1)
-- Branch { true: Expression(1, Sub), false: Counter(0) } at (prev + 0, 20) to (start + 0, 25)
-    true  = (c2 - (c0 + c1))
-    false = c0
-Highest counter ID seen: c1
-
diff --git a/tests/coverage/branch/no-mir-spans.coverage b/tests/coverage/branch/no-mir-spans.coverage
deleted file mode 100644
index 2cae98ed3ff4..000000000000
--- a/tests/coverage/branch/no-mir-spans.coverage
+++ /dev/null
@@ -1,77 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=branch,no-mir-spans
-   LL|       |//@ llvm-cov-flags: --show-branches=count
-   LL|       |
-   LL|       |// Tests the behaviour of the `-Zcoverage-options=no-mir-spans` debugging flag.
-   LL|       |// The actual code below is just some non-trivial code copied from another test
-   LL|       |// (`while.rs`), and has no particular significance.
-   LL|       |
-   LL|       |macro_rules! no_merge {
-   LL|       |    () => {
-   LL|       |        for _ in 0..1 {}
-   LL|       |    };
-   LL|       |}
-   LL|       |
-   LL|      1|fn while_cond() {
-   LL|       |    no_merge!();
-   LL|       |
-   LL|       |    let mut a = 8;
-   LL|       |    while a > 0 {
-  ------------------
-  |  Branch (LL:11): [True: 8, False: 1]
-  ------------------
-   LL|       |        a -= 1;
-   LL|       |    }
-   LL|       |}
-   LL|       |
-   LL|      1|fn while_cond_not() {
-   LL|       |    no_merge!();
-   LL|       |
-   LL|       |    let mut a = 8;
-   LL|       |    while !(a == 0) {
-  ------------------
-  |  Branch (LL:11): [True: 8, False: 1]
-  ------------------
-   LL|       |        a -= 1;
-   LL|       |    }
-   LL|       |}
-   LL|       |
-   LL|      1|fn while_op_and() {
-   LL|       |    no_merge!();
-   LL|       |
-   LL|       |    let mut a = 8;
-   LL|       |    let mut b = 4;
-   LL|       |    while a > 0 && b > 0 {
-  ------------------
-  |  Branch (LL:11): [True: 5, False: 0]
-  |  Branch (LL:20): [True: 4, False: 1]
-  ------------------
-   LL|       |        a -= 1;
-   LL|       |        b -= 1;
-   LL|       |    }
-   LL|       |}
-   LL|       |
-   LL|      1|fn while_op_or() {
-   LL|       |    no_merge!();
-   LL|       |
-   LL|       |    let mut a = 4;
-   LL|       |    let mut b = 8;
-   LL|       |    while a > 0 || b > 0 {
-  ------------------
-  |  Branch (LL:11): [True: 4, False: 5]
-  |  Branch (LL:20): [True: 4, False: 1]
-  ------------------
-   LL|       |        a -= 1;
-   LL|       |        b -= 1;
-   LL|       |    }
-   LL|       |}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    while_cond();
-   LL|       |    while_cond_not();
-   LL|       |    while_op_and();
-   LL|       |    while_op_or();
-   LL|       |}
-
diff --git a/tests/coverage/branch/no-mir-spans.rs b/tests/coverage/branch/no-mir-spans.rs
deleted file mode 100644
index acb268f2d455..000000000000
--- a/tests/coverage/branch/no-mir-spans.rs
+++ /dev/null
@@ -1,62 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=branch,no-mir-spans
-//@ llvm-cov-flags: --show-branches=count
-
-// Tests the behaviour of the `-Zcoverage-options=no-mir-spans` debugging flag.
-// The actual code below is just some non-trivial code copied from another test
-// (`while.rs`), and has no particular significance.
-
-macro_rules! no_merge {
-    () => {
-        for _ in 0..1 {}
-    };
-}
-
-fn while_cond() {
-    no_merge!();
-
-    let mut a = 8;
-    while a > 0 {
-        a -= 1;
-    }
-}
-
-fn while_cond_not() {
-    no_merge!();
-
-    let mut a = 8;
-    while !(a == 0) {
-        a -= 1;
-    }
-}
-
-fn while_op_and() {
-    no_merge!();
-
-    let mut a = 8;
-    let mut b = 4;
-    while a > 0 && b > 0 {
-        a -= 1;
-        b -= 1;
-    }
-}
-
-fn while_op_or() {
-    no_merge!();
-
-    let mut a = 4;
-    let mut b = 8;
-    while a > 0 || b > 0 {
-        a -= 1;
-        b -= 1;
-    }
-}
-
-#[coverage(off)]
-fn main() {
-    while_cond();
-    while_cond_not();
-    while_op_and();
-    while_op_or();
-}
diff --git a/tests/coverage/closure.cov-map b/tests/coverage/closure.cov-map
index 5f7e4ce58e97..ee8934f0a846 100644
--- a/tests/coverage/closure.cov-map
+++ b/tests/coverage/closure.cov-map
@@ -118,21 +118,23 @@ Number of file 0 mappings: 4
 Highest counter ID seen: (none)
 
 Function name: closure::main::{closure#12} (unused)
-Raw bytes (10): 0x[01, 01, 00, 01, 00, a7, 01, 0a, 00, 16]
+Raw bytes (15): 0x[01, 01, 00, 02, 00, a7, 01, 01, 00, 09, 00, 00, 0a, 00, 16]
 Number of files: 1
 - file 0 => $DIR/closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 167, 10) to (start + 0, 22)
+Number of file 0 mappings: 2
+- Code(Zero) at (prev + 167, 1) to (start + 0, 9)
+- Code(Zero) at (prev + 0, 10) to (start + 0, 22)
 Highest counter ID seen: (none)
 
 Function name: closure::main::{closure#13} (unused)
-Raw bytes (10): 0x[01, 01, 00, 01, 00, ad, 01, 11, 00, 1d]
+Raw bytes (15): 0x[01, 01, 00, 02, 00, ac, 01, 0d, 00, 15, 00, 01, 11, 00, 1d]
 Number of files: 1
 - file 0 => $DIR/closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 173, 17) to (start + 0, 29)
+Number of file 0 mappings: 2
+- Code(Zero) at (prev + 172, 13) to (start + 0, 21)
+- Code(Zero) at (prev + 1, 17) to (start + 0, 29)
 Highest counter ID seen: (none)
 
 Function name: closure::main::{closure#14}
@@ -289,30 +291,33 @@ Number of file 0 mappings: 7
 Highest counter ID seen: (none)
 
 Function name: closure::main::{closure#5}
-Raw bytes (10): 0x[01, 01, 00, 01, 01, 8c, 01, 46, 00, 4e]
+Raw bytes (15): 0x[01, 01, 00, 02, 01, 8c, 01, 3d, 00, 45, 01, 00, 46, 00, 4e]
 Number of files: 1
 - file 0 => $DIR/closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 140, 70) to (start + 0, 78)
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 140, 61) to (start + 0, 69)
+- Code(Counter(0)) at (prev + 0, 70) to (start + 0, 78)
 Highest counter ID seen: c0
 
 Function name: closure::main::{closure#6}
-Raw bytes (10): 0x[01, 01, 00, 01, 01, 8d, 01, 4a, 00, 56]
+Raw bytes (15): 0x[01, 01, 00, 02, 01, 8d, 01, 41, 00, 49, 01, 00, 4a, 00, 56]
 Number of files: 1
 - file 0 => $DIR/closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 141, 74) to (start + 0, 86)
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 141, 65) to (start + 0, 73)
+- Code(Counter(0)) at (prev + 0, 74) to (start + 0, 86)
 Highest counter ID seen: c0
 
 Function name: closure::main::{closure#7} (unused)
-Raw bytes (10): 0x[01, 01, 00, 01, 00, 8e, 01, 44, 00, 50]
+Raw bytes (15): 0x[01, 01, 00, 02, 00, 8e, 01, 3b, 00, 43, 00, 00, 44, 00, 50]
 Number of files: 1
 - file 0 => $DIR/closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 142, 68) to (start + 0, 80)
+Number of file 0 mappings: 2
+- Code(Zero) at (prev + 142, 59) to (start + 0, 67)
+- Code(Zero) at (prev + 0, 68) to (start + 0, 80)
 Highest counter ID seen: (none)
 
 Function name: closure::main::{closure#8} (unused)
diff --git a/tests/coverage/closure.coverage b/tests/coverage/closure.coverage
index 142852692289..d44ecf5a69e2 100644
--- a/tests/coverage/closure.coverage
+++ b/tests/coverage/closure.coverage
@@ -139,9 +139,9 @@
    LL|       |
    LL|      1|    let short_used_covered_closure_macro = | used_arg: u8 | println!("called");
    LL|      1|    let short_used_not_covered_closure_macro = | used_arg: u8 | println!("not called");
-                                                                                       ^0
+                                                                              ^0       ^0
    LL|      1|    let _short_unused_closure_macro = | _unused_arg: u8 | println!("not called");
-                                                                                 ^0
+                                                                        ^0       ^0
    LL|       |
    LL|       |
    LL|       |
@@ -173,7 +173,7 @@
    LL|       |
    LL|      1|    let _short_unused_closure_line_break_no_block2 =
    LL|       |        | _unused_arg: u8 |
-   LL|       |            println!(
+   LL|      0|            println!(
    LL|      0|                "not called"
    LL|       |            )
    LL|       |    ;
diff --git a/tests/coverage/issue-83601.cov-map b/tests/coverage/issue-83601.cov-map
index d1d751ff24b8..e42b5591c0fb 100644
--- a/tests/coverage/issue-83601.cov-map
+++ b/tests/coverage/issue-83601.cov-map
@@ -1,30 +1,22 @@
 Function name: issue_83601::main
-Raw bytes (76): 0x[01, 01, 01, 05, 09, 0e, 01, 06, 01, 00, 0a, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 05, 01, 09, 00, 0c, 05, 00, 0f, 00, 15, 05, 01, 05, 00, 0f, 02, 01, 05, 00, 0d, 02, 00, 0e, 00, 14, 02, 01, 05, 00, 0d, 02, 00, 0e, 00, 14, 02, 01, 05, 00, 0d, 02, 00, 0e, 00, 14, 02, 01, 01, 00, 02]
+Raw bytes (74): 0x[01, 01, 00, 0e, 01, 06, 01, 00, 0a, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => $DIR/issue-83601.rs
-Number of expressions: 1
-- expression 0 operands: lhs = Counter(1), rhs = Counter(2)
+Number of expressions: 0
 Number of file 0 mappings: 14
 - Code(Counter(0)) at (prev + 6, 1) to (start + 0, 10)
 - Code(Counter(0)) at (prev + 1, 9) to (start + 0, 12)
 - Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
 - Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
-- Code(Counter(1)) at (prev + 0, 15) to (start + 0, 21)
-- Code(Counter(1)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 13)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 0, 14) to (start + 0, 20)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 13)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 0, 14) to (start + 0, 20)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 13)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 0, 14) to (start + 0, 20)
-    = (c1 - c2)
-- Code(Expression(0, Sub)) at (prev + 1, 1) to (start + 0, 2)
-    = (c1 - c2)
-Highest counter ID seen: c1
+- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 12)
+- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
+Highest counter ID seen: c0
 
diff --git a/tests/coverage/issue-84561.cov-map b/tests/coverage/issue-84561.cov-map
index 2b643ea599ed..e5bb1afdcc26 100644
--- a/tests/coverage/issue-84561.cov-map
+++ b/tests/coverage/issue-84561.cov-map
@@ -73,20 +73,20 @@ Number of file 0 mappings: 4
 Highest counter ID seen: c0
 
 Function name: issue_84561::test3
-Raw bytes (409): 0x[01, 01, 0a, 0d, 11, 0d, 15, 0d, 19, 1d, 21, 29, 2d, 25, 29, 25, 29, 25, 29, 27, 31, 29, 2d, 4d, 01, 08, 01, 00, 0b, 01, 01, 09, 00, 10, 01, 00, 13, 00, 2e, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 05, 01, 09, 00, 0c, 05, 00, 0f, 00, 15, 05, 01, 05, 00, 0f, 09, 01, 05, 00, 0d, 09, 00, 0e, 00, 14, 09, 01, 05, 00, 0d, 09, 00, 0e, 00, 14, 09, 01, 05, 00, 0d, 09, 00, 0e, 00, 14, 09, 02, 05, 00, 0f, 09, 01, 05, 00, 0f, 09, 01, 05, 00, 0f, 09, 01, 09, 00, 0c, 09, 00, 0f, 00, 15, 09, 01, 05, 00, 0f, 0d, 01, 05, 00, 0f, 0d, 01, 05, 00, 0f, 00, 00, 20, 00, 30, 0d, 01, 05, 00, 0d, 0d, 00, 0e, 00, 14, 0d, 01, 05, 00, 0d, 0d, 00, 0e, 00, 14, 0d, 02, 05, 00, 0f, 00, 00, 20, 00, 24, 00, 00, 29, 00, 30, 00, 00, 33, 00, 41, 00, 00, 4b, 00, 5a, 0d, 01, 05, 00, 0f, 00, 05, 09, 00, 0d, 00, 03, 09, 00, 10, 00, 02, 0d, 00, 1b, 00, 02, 0d, 00, 1c, 0d, 04, 09, 00, 10, 0d, 00, 13, 00, 2e, 0d, 02, 05, 00, 0f, 0d, 04, 05, 00, 0f, 0d, 04, 05, 00, 0f, 0d, 04, 09, 00, 0c, 0d, 00, 0f, 00, 15, 0d, 01, 05, 00, 0f, 0d, 04, 08, 00, 0f, 11, 01, 09, 00, 13, 02, 05, 09, 00, 13, 0d, 05, 08, 00, 0f, 15, 01, 09, 00, 13, 00, 03, 0d, 00, 1d, 06, 03, 09, 00, 13, 00, 03, 0d, 00, 1d, 0d, 03, 05, 00, 0f, 0d, 01, 0c, 00, 13, 19, 01, 0d, 00, 13, 0a, 02, 0d, 00, 13, 1d, 04, 05, 00, 0f, 1d, 02, 0c, 00, 13, 21, 01, 0d, 00, 13, 0e, 02, 0d, 00, 13, 27, 03, 05, 00, 0f, 25, 01, 0c, 00, 13, 29, 01, 0d, 00, 17, 29, 04, 0d, 00, 13, 1e, 02, 0d, 00, 17, 1e, 01, 14, 00, 1b, 00, 01, 15, 00, 1b, 1e, 02, 15, 00, 1b, 2d, 04, 0d, 00, 13, 22, 03, 09, 00, 19, 31, 02, 05, 00, 0f, 31, 03, 09, 00, 22, 00, 02, 05, 00, 0f, 00, 03, 09, 00, 2c, 00, 02, 01, 00, 02]
+Raw bytes (409): 0x[01, 01, 0a, 01, 05, 01, 09, 01, 0d, 11, 15, 1d, 21, 19, 1d, 19, 1d, 19, 1d, 27, 25, 1d, 21, 4d, 01, 08, 01, 00, 0b, 01, 01, 09, 00, 10, 01, 00, 13, 00, 2e, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 02, 05, 00, 0f, 01, 01, 05, 00, 0f, 01, 01, 05, 00, 0f, 01, 01, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 01, 05, 00, 0f, 01, 01, 05, 00, 0f, 00, 00, 20, 00, 30, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 01, 05, 00, 0d, 01, 00, 0e, 00, 14, 01, 02, 05, 00, 0f, 00, 00, 20, 00, 24, 00, 00, 29, 00, 30, 00, 00, 33, 00, 41, 00, 00, 4b, 00, 5a, 01, 01, 05, 00, 0f, 00, 05, 09, 00, 0d, 00, 03, 09, 00, 10, 00, 02, 0d, 00, 1b, 00, 02, 0d, 00, 1c, 01, 04, 09, 00, 10, 01, 00, 13, 00, 2e, 01, 02, 05, 00, 0f, 01, 04, 05, 00, 0f, 01, 04, 05, 00, 0f, 01, 04, 09, 00, 0c, 01, 00, 0f, 00, 15, 01, 01, 05, 00, 0f, 01, 04, 08, 00, 0f, 05, 01, 09, 00, 13, 02, 05, 09, 00, 13, 01, 05, 08, 00, 0f, 09, 01, 09, 00, 13, 00, 03, 0d, 00, 1d, 06, 03, 09, 00, 13, 00, 03, 0d, 00, 1d, 01, 03, 05, 00, 0f, 01, 01, 0c, 00, 13, 0d, 01, 0d, 00, 13, 0a, 02, 0d, 00, 13, 11, 04, 05, 00, 0f, 11, 02, 0c, 00, 13, 15, 01, 0d, 00, 13, 0e, 02, 0d, 00, 13, 27, 03, 05, 00, 0f, 19, 01, 0c, 00, 13, 1d, 01, 0d, 00, 17, 1d, 04, 0d, 00, 13, 1e, 02, 0d, 00, 17, 1e, 01, 14, 00, 1b, 00, 01, 15, 00, 1b, 1e, 02, 15, 00, 1b, 21, 04, 0d, 00, 13, 22, 03, 09, 00, 19, 25, 02, 05, 00, 0f, 25, 03, 09, 00, 22, 00, 02, 05, 00, 0f, 00, 03, 09, 00, 2c, 00, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => $DIR/issue-84561.rs
 Number of expressions: 10
-- expression 0 operands: lhs = Counter(3), rhs = Counter(4)
-- expression 1 operands: lhs = Counter(3), rhs = Counter(5)
-- expression 2 operands: lhs = Counter(3), rhs = Counter(6)
-- expression 3 operands: lhs = Counter(7), rhs = Counter(8)
-- expression 4 operands: lhs = Counter(10), rhs = Counter(11)
-- expression 5 operands: lhs = Counter(9), rhs = Counter(10)
-- expression 6 operands: lhs = Counter(9), rhs = Counter(10)
-- expression 7 operands: lhs = Counter(9), rhs = Counter(10)
-- expression 8 operands: lhs = Expression(9, Add), rhs = Counter(12)
-- expression 9 operands: lhs = Counter(10), rhs = Counter(11)
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 1 operands: lhs = Counter(0), rhs = Counter(2)
+- expression 2 operands: lhs = Counter(0), rhs = Counter(3)
+- expression 3 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 4 operands: lhs = Counter(7), rhs = Counter(8)
+- expression 5 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 6 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 7 operands: lhs = Counter(6), rhs = Counter(7)
+- expression 8 operands: lhs = Expression(9, Add), rhs = Counter(9)
+- expression 9 operands: lhs = Counter(7), rhs = Counter(8)
 Number of file 0 mappings: 77
 - Code(Counter(0)) at (prev + 8, 1) to (start + 0, 11)
 - Code(Counter(0)) at (prev + 1, 9) to (start + 0, 16)
@@ -94,85 +94,85 @@ Number of file 0 mappings: 77
 - Code(Counter(0)) at (prev + 1, 9) to (start + 0, 12)
 - Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
 - Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
-- Code(Counter(1)) at (prev + 0, 15) to (start + 0, 21)
-- Code(Counter(1)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 13)
-- Code(Counter(2)) at (prev + 0, 14) to (start + 0, 20)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 13)
-- Code(Counter(2)) at (prev + 0, 14) to (start + 0, 20)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 13)
-- Code(Counter(2)) at (prev + 0, 14) to (start + 0, 20)
-- Code(Counter(2)) at (prev + 2, 5) to (start + 0, 15)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(2)) at (prev + 1, 9) to (start + 0, 12)
-- Code(Counter(2)) at (prev + 0, 15) to (start + 0, 21)
-- Code(Counter(2)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 12)
+- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 2, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 12)
+- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
 - Code(Zero) at (prev + 0, 32) to (start + 0, 48)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 13)
-- Code(Counter(3)) at (prev + 0, 14) to (start + 0, 20)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 13)
-- Code(Counter(3)) at (prev + 0, 14) to (start + 0, 20)
-- Code(Counter(3)) at (prev + 2, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 0, 14) to (start + 0, 20)
+- Code(Counter(0)) at (prev + 2, 5) to (start + 0, 15)
 - Code(Zero) at (prev + 0, 32) to (start + 0, 36)
 - Code(Zero) at (prev + 0, 41) to (start + 0, 48)
 - Code(Zero) at (prev + 0, 51) to (start + 0, 65)
 - Code(Zero) at (prev + 0, 75) to (start + 0, 90)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
 - Code(Zero) at (prev + 5, 9) to (start + 0, 13)
 - Code(Zero) at (prev + 3, 9) to (start + 0, 16)
 - Code(Zero) at (prev + 2, 13) to (start + 0, 27)
 - Code(Zero) at (prev + 2, 13) to (start + 0, 28)
-- Code(Counter(3)) at (prev + 4, 9) to (start + 0, 16)
-- Code(Counter(3)) at (prev + 0, 19) to (start + 0, 46)
-- Code(Counter(3)) at (prev + 2, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 4, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 4, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 4, 9) to (start + 0, 12)
-- Code(Counter(3)) at (prev + 0, 15) to (start + 0, 21)
-- Code(Counter(3)) at (prev + 1, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 4, 8) to (start + 0, 15)
-- Code(Counter(4)) at (prev + 1, 9) to (start + 0, 19)
+- Code(Counter(0)) at (prev + 4, 9) to (start + 0, 16)
+- Code(Counter(0)) at (prev + 0, 19) to (start + 0, 46)
+- Code(Counter(0)) at (prev + 2, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 4, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 4, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 4, 9) to (start + 0, 12)
+- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 21)
+- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 4, 8) to (start + 0, 15)
+- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 19)
 - Code(Expression(0, Sub)) at (prev + 5, 9) to (start + 0, 19)
-    = (c3 - c4)
-- Code(Counter(3)) at (prev + 5, 8) to (start + 0, 15)
-- Code(Counter(5)) at (prev + 1, 9) to (start + 0, 19)
+    = (c0 - c1)
+- Code(Counter(0)) at (prev + 5, 8) to (start + 0, 15)
+- Code(Counter(2)) at (prev + 1, 9) to (start + 0, 19)
 - Code(Zero) at (prev + 3, 13) to (start + 0, 29)
 - Code(Expression(1, Sub)) at (prev + 3, 9) to (start + 0, 19)
-    = (c3 - c5)
+    = (c0 - c2)
 - Code(Zero) at (prev + 3, 13) to (start + 0, 29)
-- Code(Counter(3)) at (prev + 3, 5) to (start + 0, 15)
-- Code(Counter(3)) at (prev + 1, 12) to (start + 0, 19)
-- Code(Counter(6)) at (prev + 1, 13) to (start + 0, 19)
+- Code(Counter(0)) at (prev + 3, 5) to (start + 0, 15)
+- Code(Counter(0)) at (prev + 1, 12) to (start + 0, 19)
+- Code(Counter(3)) at (prev + 1, 13) to (start + 0, 19)
 - Code(Expression(2, Sub)) at (prev + 2, 13) to (start + 0, 19)
-    = (c3 - c6)
-- Code(Counter(7)) at (prev + 4, 5) to (start + 0, 15)
-- Code(Counter(7)) at (prev + 2, 12) to (start + 0, 19)
-- Code(Counter(8)) at (prev + 1, 13) to (start + 0, 19)
+    = (c0 - c3)
+- Code(Counter(4)) at (prev + 4, 5) to (start + 0, 15)
+- Code(Counter(4)) at (prev + 2, 12) to (start + 0, 19)
+- Code(Counter(5)) at (prev + 1, 13) to (start + 0, 19)
 - Code(Expression(3, Sub)) at (prev + 2, 13) to (start + 0, 19)
-    = (c7 - c8)
+    = (c4 - c5)
 - Code(Expression(9, Add)) at (prev + 3, 5) to (start + 0, 15)
-    = (c10 + c11)
-- Code(Counter(9)) at (prev + 1, 12) to (start + 0, 19)
-- Code(Counter(10)) at (prev + 1, 13) to (start + 0, 23)
-- Code(Counter(10)) at (prev + 4, 13) to (start + 0, 19)
+    = (c7 + c8)
+- Code(Counter(6)) at (prev + 1, 12) to (start + 0, 19)
+- Code(Counter(7)) at (prev + 1, 13) to (start + 0, 23)
+- Code(Counter(7)) at (prev + 4, 13) to (start + 0, 19)
 - Code(Expression(7, Sub)) at (prev + 2, 13) to (start + 0, 23)
-    = (c9 - c10)
+    = (c6 - c7)
 - Code(Expression(7, Sub)) at (prev + 1, 20) to (start + 0, 27)
-    = (c9 - c10)
+    = (c6 - c7)
 - Code(Zero) at (prev + 1, 21) to (start + 0, 27)
 - Code(Expression(7, Sub)) at (prev + 2, 21) to (start + 0, 27)
-    = (c9 - c10)
-- Code(Counter(11)) at (prev + 4, 13) to (start + 0, 19)
+    = (c6 - c7)
+- Code(Counter(8)) at (prev + 4, 13) to (start + 0, 19)
 - Code(Expression(8, Sub)) at (prev + 3, 9) to (start + 0, 25)
-    = ((c10 + c11) - c12)
-- Code(Counter(12)) at (prev + 2, 5) to (start + 0, 15)
-- Code(Counter(12)) at (prev + 3, 9) to (start + 0, 34)
+    = ((c7 + c8) - c9)
+- Code(Counter(9)) at (prev + 2, 5) to (start + 0, 15)
+- Code(Counter(9)) at (prev + 3, 9) to (start + 0, 34)
 - Code(Zero) at (prev + 2, 5) to (start + 0, 15)
 - Code(Zero) at (prev + 3, 9) to (start + 0, 44)
 - Code(Zero) at (prev + 2, 1) to (start + 0, 2)
-Highest counter ID seen: c12
+Highest counter ID seen: c9
 
diff --git a/tests/coverage/macro_in_closure.cov-map b/tests/coverage/macro_in_closure.cov-map
index 4544aa50143e..3529d0c4c321 100644
--- a/tests/coverage/macro_in_closure.cov-map
+++ b/tests/coverage/macro_in_closure.cov-map
@@ -1,10 +1,11 @@
 Function name: macro_in_closure::NO_BLOCK::{closure#0}
-Raw bytes (9): 0x[01, 01, 00, 01, 01, 07, 25, 00, 2c]
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 07, 1c, 00, 24, 01, 00, 25, 00, 2c]
 Number of files: 1
 - file 0 => $DIR/macro_in_closure.rs
 Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 7, 37) to (start + 0, 44)
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 7, 28) to (start + 0, 36)
+- Code(Counter(0)) at (prev + 0, 37) to (start + 0, 44)
 Highest counter ID seen: c0
 
 Function name: macro_in_closure::WITH_BLOCK::{closure#0}
diff --git a/tests/coverage/mcdc/condition-limit.cov-map b/tests/coverage/mcdc/condition-limit.cov-map
deleted file mode 100644
index ffee97cfbc50..000000000000
--- a/tests/coverage/mcdc/condition-limit.cov-map
+++ /dev/null
@@ -1,58 +0,0 @@
-Function name: condition_limit::accept_7_conditions
-Raw bytes (192): 0x[01, 01, 08, 01, 05, 05, 09, 09, 0d, 0d, 11, 11, 15, 15, 19, 19, 1d, 01, 1d, 1b, 01, 06, 01, 00, 2c, 01, 01, 0a, 00, 0b, 01, 00, 0d, 00, 0e, 01, 00, 10, 00, 11, 01, 00, 13, 00, 14, 01, 00, 16, 00, 17, 01, 00, 19, 00, 1a, 01, 00, 1c, 00, 1d, 01, 00, 21, 00, 29, 01, 01, 08, 00, 09, 28, 08, 07, 00, 08, 00, 27, 30, 05, 02, 01, 07, 00, 00, 08, 00, 09, 05, 00, 0d, 00, 0e, 30, 09, 06, 07, 06, 00, 00, 0d, 00, 0e, 09, 00, 12, 00, 13, 30, 0d, 0a, 06, 05, 00, 00, 12, 00, 13, 0d, 00, 17, 00, 18, 30, 11, 0e, 05, 04, 00, 00, 17, 00, 18, 11, 00, 1c, 00, 1d, 30, 15, 12, 04, 03, 00, 00, 1c, 00, 1d, 15, 00, 21, 00, 22, 30, 19, 16, 03, 02, 00, 00, 21, 00, 22, 19, 00, 26, 00, 27, 30, 1d, 1a, 02, 00, 00, 00, 26, 00, 27, 1d, 00, 28, 02, 06, 1e, 02, 05, 00, 06, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/condition-limit.rs
-Number of expressions: 8
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 3 operands: lhs = Counter(3), rhs = Counter(4)
-- expression 4 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 5 operands: lhs = Counter(5), rhs = Counter(6)
-- expression 6 operands: lhs = Counter(6), rhs = Counter(7)
-- expression 7 operands: lhs = Counter(0), rhs = Counter(7)
-Number of file 0 mappings: 27
-- Code(Counter(0)) at (prev + 6, 1) to (start + 0, 44)
-- Code(Counter(0)) at (prev + 1, 10) to (start + 0, 11)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 16) to (start + 0, 17)
-- Code(Counter(0)) at (prev + 0, 19) to (start + 0, 20)
-- Code(Counter(0)) at (prev + 0, 22) to (start + 0, 23)
-- Code(Counter(0)) at (prev + 0, 25) to (start + 0, 26)
-- Code(Counter(0)) at (prev + 0, 28) to (start + 0, 29)
-- Code(Counter(0)) at (prev + 0, 33) to (start + 0, 41)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 8, conditions_num: 7 } at (prev + 0, 8) to (start + 0, 39)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 7, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 7, true_next_id: 6, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 18) to (start + 0, 19)
-- MCDCBranch { true: Counter(3), false: Expression(2, Sub), condition_id: 6, true_next_id: 5, false_next_id: 0 } at (prev + 0, 18) to (start + 0, 19)
-    true  = c3
-    false = (c2 - c3)
-- Code(Counter(3)) at (prev + 0, 23) to (start + 0, 24)
-- MCDCBranch { true: Counter(4), false: Expression(3, Sub), condition_id: 5, true_next_id: 4, false_next_id: 0 } at (prev + 0, 23) to (start + 0, 24)
-    true  = c4
-    false = (c3 - c4)
-- Code(Counter(4)) at (prev + 0, 28) to (start + 0, 29)
-- MCDCBranch { true: Counter(5), false: Expression(4, Sub), condition_id: 4, true_next_id: 3, false_next_id: 0 } at (prev + 0, 28) to (start + 0, 29)
-    true  = c5
-    false = (c4 - c5)
-- Code(Counter(5)) at (prev + 0, 33) to (start + 0, 34)
-- MCDCBranch { true: Counter(6), false: Expression(5, Sub), condition_id: 3, true_next_id: 2, false_next_id: 0 } at (prev + 0, 33) to (start + 0, 34)
-    true  = c6
-    false = (c5 - c6)
-- Code(Counter(6)) at (prev + 0, 38) to (start + 0, 39)
-- MCDCBranch { true: Counter(7), false: Expression(6, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 38) to (start + 0, 39)
-    true  = c7
-    false = (c6 - c7)
-- Code(Counter(7)) at (prev + 0, 40) to (start + 2, 6)
-- Code(Expression(7, Sub)) at (prev + 2, 5) to (start + 0, 6)
-    = (c0 - c7)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c7
-
diff --git a/tests/coverage/mcdc/condition-limit.coverage b/tests/coverage/mcdc/condition-limit.coverage
deleted file mode 100644
index 04ccd6497c3d..000000000000
--- a/tests/coverage/mcdc/condition-limit.coverage
+++ /dev/null
@@ -1,55 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=mcdc
-   LL|       |//@ llvm-cov-flags: --show-branches=count --show-mcdc
-   LL|       |
-   LL|      2|fn accept_7_conditions(bool_arr: [bool; 7]) {
-   LL|      2|    let [a, b, c, d, e, f, g] = bool_arr;
-   LL|      2|    if a && b && c && d && e && f && g {
-                          ^1   ^1   ^1   ^1   ^1   ^1
-  ------------------
-  |  Branch (LL:8): [True: 1, False: 1]
-  |  Branch (LL:13): [True: 1, False: 0]
-  |  Branch (LL:18): [True: 1, False: 0]
-  |  Branch (LL:23): [True: 1, False: 0]
-  |  Branch (LL:28): [True: 1, False: 0]
-  |  Branch (LL:33): [True: 1, False: 0]
-  |  Branch (LL:38): [True: 1, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:39)
-  |
-  |  Number of Conditions: 7
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |     Condition C3 --> (LL:18)
-  |     Condition C4 --> (LL:23)
-  |     Condition C5 --> (LL:28)
-  |     Condition C6 --> (LL:33)
-  |     Condition C7 --> (LL:38)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3, C4, C5, C6, C7    Result
-  |  1 { F,  -,  -,  -,  -,  -,  -  = F      }
-  |  2 { T,  T,  T,  T,  T,  T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,2)
-  |  C2-Pair: not covered
-  |  C3-Pair: not covered
-  |  C4-Pair: not covered
-  |  C5-Pair: not covered
-  |  C6-Pair: not covered
-  |  C7-Pair: not covered
-  |  MC/DC Coverage for Decision: 14.29%
-  |
-  ------------------
-   LL|      1|        core::hint::black_box("hello");
-   LL|      1|    }
-   LL|      2|}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    accept_7_conditions([false; 7]);
-   LL|       |    accept_7_conditions([true; 7]);
-   LL|       |}
-
diff --git a/tests/coverage/mcdc/condition-limit.rs b/tests/coverage/mcdc/condition-limit.rs
deleted file mode 100644
index 867636cdaed2..000000000000
--- a/tests/coverage/mcdc/condition-limit.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=mcdc
-//@ llvm-cov-flags: --show-branches=count --show-mcdc
-
-fn accept_7_conditions(bool_arr: [bool; 7]) {
-    let [a, b, c, d, e, f, g] = bool_arr;
-    if a && b && c && d && e && f && g {
-        core::hint::black_box("hello");
-    }
-}
-
-#[coverage(off)]
-fn main() {
-    accept_7_conditions([false; 7]);
-    accept_7_conditions([true; 7]);
-}
diff --git a/tests/coverage/mcdc/if.cov-map b/tests/coverage/mcdc/if.cov-map
deleted file mode 100644
index dac1eb4c94bd..000000000000
--- a/tests/coverage/mcdc/if.cov-map
+++ /dev/null
@@ -1,223 +0,0 @@
-Function name: if::mcdc_check_a
-Raw bytes (67): 0x[01, 01, 03, 01, 05, 05, 09, 01, 09, 09, 01, 0e, 01, 00, 22, 01, 01, 08, 00, 09, 28, 03, 02, 00, 08, 00, 0e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 05, 00, 0d, 00, 0e, 30, 09, 06, 02, 00, 00, 00, 0d, 00, 0e, 09, 00, 0f, 02, 06, 0a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(2)
-Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 34)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 14)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 15) to (start + 2, 6)
-- Code(Expression(2, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c2)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: if::mcdc_check_b
-Raw bytes (67): 0x[01, 01, 03, 01, 05, 05, 09, 01, 09, 09, 01, 16, 01, 00, 22, 01, 01, 08, 00, 09, 28, 03, 02, 00, 08, 00, 0e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 05, 00, 0d, 00, 0e, 30, 09, 06, 02, 00, 00, 00, 0d, 00, 0e, 09, 00, 0f, 02, 06, 0a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(2)
-Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 22, 1) to (start + 0, 34)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 14)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 15) to (start + 2, 6)
-- Code(Expression(2, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c2)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: if::mcdc_check_both
-Raw bytes (67): 0x[01, 01, 03, 01, 05, 05, 09, 01, 09, 09, 01, 1e, 01, 00, 25, 01, 01, 08, 00, 09, 28, 03, 02, 00, 08, 00, 0e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 05, 00, 0d, 00, 0e, 30, 09, 06, 02, 00, 00, 00, 0d, 00, 0e, 09, 00, 0f, 02, 06, 0a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(2)
-Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 30, 1) to (start + 0, 37)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 14)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 15) to (start + 2, 6)
-- Code(Expression(2, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c2)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: if::mcdc_check_neither
-Raw bytes (67): 0x[01, 01, 03, 01, 05, 05, 09, 01, 09, 09, 01, 06, 01, 00, 28, 01, 01, 08, 00, 09, 28, 03, 02, 00, 08, 00, 0e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 05, 00, 0d, 00, 0e, 30, 09, 06, 02, 00, 00, 00, 0d, 00, 0e, 09, 00, 0f, 02, 06, 0a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(2)
-Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 6, 1) to (start + 0, 40)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 14)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 15) to (start + 2, 6)
-- Code(Expression(2, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c2)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: if::mcdc_check_not_tree_decision
-Raw bytes (90): 0x[01, 01, 07, 01, 05, 01, 17, 05, 09, 05, 09, 17, 0d, 05, 09, 01, 0d, 0b, 01, 30, 01, 00, 3b, 28, 05, 03, 03, 08, 00, 15, 01, 00, 09, 00, 0a, 30, 05, 02, 01, 02, 03, 00, 09, 00, 0a, 02, 00, 0e, 00, 0f, 30, 09, 06, 03, 02, 00, 00, 0e, 00, 0f, 17, 00, 14, 00, 15, 30, 0d, 12, 02, 00, 00, 00, 14, 00, 15, 0d, 00, 16, 02, 06, 1a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 7
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(0), rhs = Expression(5, Add)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 4 operands: lhs = Expression(5, Add), rhs = Counter(3)
-- expression 5 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 6 operands: lhs = Counter(0), rhs = Counter(3)
-Number of file 0 mappings: 11
-- Code(Counter(0)) at (prev + 48, 1) to (start + 0, 59)
-- MCDCDecision { bitmap_idx: 5, conditions_num: 3 } at (prev + 3, 8) to (start + 0, 21)
-- Code(Counter(0)) at (prev + 0, 9) to (start + 0, 10)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 3 } at (prev + 0, 9) to (start + 0, 10)
-    true  = c1
-    false = (c0 - c1)
-- Code(Expression(0, Sub)) at (prev + 0, 14) to (start + 0, 15)
-    = (c0 - c1)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 3, true_next_id: 2, false_next_id: 0 } at (prev + 0, 14) to (start + 0, 15)
-    true  = c2
-    false = (c0 - (c1 + c2))
-- Code(Expression(5, Add)) at (prev + 0, 20) to (start + 0, 21)
-    = (c1 + c2)
-- MCDCBranch { true: Counter(3), false: Expression(4, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 20) to (start + 0, 21)
-    true  = c3
-    false = ((c1 + c2) - c3)
-- Code(Counter(3)) at (prev + 0, 22) to (start + 2, 6)
-- Code(Expression(6, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c3)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c3
-
-Function name: if::mcdc_check_tree_decision
-Raw bytes (92): 0x[01, 01, 08, 01, 05, 05, 09, 05, 09, 05, 1f, 09, 0d, 09, 0d, 01, 1f, 09, 0d, 0b, 01, 26, 01, 00, 37, 01, 03, 08, 00, 09, 28, 04, 03, 00, 08, 00, 15, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 05, 00, 0e, 00, 0f, 30, 09, 0a, 02, 00, 03, 00, 0e, 00, 0f, 0a, 00, 13, 00, 14, 30, 0d, 0e, 03, 00, 00, 00, 13, 00, 14, 1f, 00, 16, 02, 06, 1a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 8
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Expression(7, Add)
-- expression 4 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 5 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 6 operands: lhs = Counter(0), rhs = Expression(7, Add)
-- expression 7 operands: lhs = Counter(2), rhs = Counter(3)
-Number of file 0 mappings: 11
-- Code(Counter(0)) at (prev + 38, 1) to (start + 0, 55)
-- Code(Counter(0)) at (prev + 3, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 4, conditions_num: 3 } at (prev + 0, 8) to (start + 0, 21)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 14) to (start + 0, 15)
-- MCDCBranch { true: Counter(2), false: Expression(2, Sub), condition_id: 2, true_next_id: 0, false_next_id: 3 } at (prev + 0, 14) to (start + 0, 15)
-    true  = c2
-    false = (c1 - c2)
-- Code(Expression(2, Sub)) at (prev + 0, 19) to (start + 0, 20)
-    = (c1 - c2)
-- MCDCBranch { true: Counter(3), false: Expression(3, Sub), condition_id: 3, true_next_id: 0, false_next_id: 0 } at (prev + 0, 19) to (start + 0, 20)
-    true  = c3
-    false = (c1 - (c2 + c3))
-- Code(Expression(7, Add)) at (prev + 0, 22) to (start + 2, 6)
-    = (c2 + c3)
-- Code(Expression(6, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - (c2 + c3))
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c3
-
-Function name: if::mcdc_nested_if
-Raw bytes (139): 0x[01, 01, 0d, 01, 05, 01, 33, 05, 09, 05, 09, 05, 09, 05, 09, 33, 0d, 05, 09, 0d, 11, 33, 11, 05, 09, 01, 33, 05, 09, 11, 01, 3a, 01, 00, 2d, 01, 01, 08, 00, 09, 28, 03, 02, 00, 08, 00, 0e, 30, 05, 02, 01, 00, 02, 00, 08, 00, 09, 02, 00, 0d, 00, 0e, 30, 09, 2e, 02, 00, 00, 00, 0d, 00, 0e, 33, 01, 09, 00, 0c, 33, 00, 0d, 00, 15, 33, 01, 0c, 00, 0d, 28, 06, 02, 00, 0c, 00, 12, 30, 0d, 1a, 01, 02, 00, 00, 0c, 00, 0d, 0d, 00, 11, 00, 12, 30, 11, 22, 02, 00, 00, 00, 11, 00, 12, 11, 00, 13, 02, 0a, 26, 02, 09, 00, 0a, 2e, 01, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/if.rs
-Number of expressions: 13
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(0), rhs = Expression(12, Add)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 4 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 5 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 6 operands: lhs = Expression(12, Add), rhs = Counter(3)
-- expression 7 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 8 operands: lhs = Counter(3), rhs = Counter(4)
-- expression 9 operands: lhs = Expression(12, Add), rhs = Counter(4)
-- expression 10 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 11 operands: lhs = Counter(0), rhs = Expression(12, Add)
-- expression 12 operands: lhs = Counter(1), rhs = Counter(2)
-Number of file 0 mappings: 17
-- Code(Counter(0)) at (prev + 58, 1) to (start + 0, 45)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 14)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- Code(Expression(0, Sub)) at (prev + 0, 13) to (start + 0, 14)
-    = (c0 - c1)
-- MCDCBranch { true: Counter(2), false: Expression(11, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c2
-    false = (c0 - (c1 + c2))
-- Code(Expression(12, Add)) at (prev + 1, 9) to (start + 0, 12)
-    = (c1 + c2)
-- Code(Expression(12, Add)) at (prev + 0, 13) to (start + 0, 21)
-    = (c1 + c2)
-- Code(Expression(12, Add)) at (prev + 1, 12) to (start + 0, 13)
-    = (c1 + c2)
-- MCDCDecision { bitmap_idx: 6, conditions_num: 2 } at (prev + 0, 12) to (start + 0, 18)
-- MCDCBranch { true: Counter(3), false: Expression(6, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 12) to (start + 0, 13)
-    true  = c3
-    false = ((c1 + c2) - c3)
-- Code(Counter(3)) at (prev + 0, 17) to (start + 0, 18)
-- MCDCBranch { true: Counter(4), false: Expression(8, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 17) to (start + 0, 18)
-    true  = c4
-    false = (c3 - c4)
-- Code(Counter(4)) at (prev + 0, 19) to (start + 2, 10)
-- Code(Expression(9, Sub)) at (prev + 2, 9) to (start + 0, 10)
-    = ((c1 + c2) - c4)
-- Code(Expression(11, Sub)) at (prev + 1, 12) to (start + 2, 6)
-    = (c0 - (c1 + c2))
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c4
-
diff --git a/tests/coverage/mcdc/if.coverage b/tests/coverage/mcdc/if.coverage
deleted file mode 100644
index fda5525c472c..000000000000
--- a/tests/coverage/mcdc/if.coverage
+++ /dev/null
@@ -1,287 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=mcdc
-   LL|       |//@ llvm-cov-flags: --show-branches=count --show-mcdc
-   LL|       |
-   LL|      2|fn mcdc_check_neither(a: bool, b: bool) {
-   LL|      2|    if a && b {
-                          ^0
-  ------------------
-  |  Branch (LL:8): [True: 0, False: 2]
-  |  Branch (LL:13): [True: 0, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:14)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |
-  |  C1-Pair: not covered
-  |  C2-Pair: not covered
-  |  MC/DC Coverage for Decision: 0.00%
-  |
-  ------------------
-   LL|      0|        say("a and b");
-   LL|      2|    } else {
-   LL|      2|        say("not both");
-   LL|      2|    }
-   LL|      2|}
-   LL|       |
-   LL|      2|fn mcdc_check_a(a: bool, b: bool) {
-   LL|      2|    if a && b {
-                          ^1
-  ------------------
-  |  Branch (LL:8): [True: 1, False: 1]
-  |  Branch (LL:13): [True: 1, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:14)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,2)
-  |  C2-Pair: not covered
-  |  MC/DC Coverage for Decision: 50.00%
-  |
-  ------------------
-   LL|      1|        say("a and b");
-   LL|      1|    } else {
-   LL|      1|        say("not both");
-   LL|      1|    }
-   LL|      2|}
-   LL|       |
-   LL|      2|fn mcdc_check_b(a: bool, b: bool) {
-   LL|      2|    if a && b {
-  ------------------
-  |  Branch (LL:8): [True: 2, False: 0]
-  |  Branch (LL:13): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:14)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { T,  F  = F      }
-  |  2 { T,  T  = T      }
-  |
-  |  C1-Pair: not covered
-  |  C2-Pair: covered: (1,2)
-  |  MC/DC Coverage for Decision: 50.00%
-  |
-  ------------------
-   LL|      1|        say("a and b");
-   LL|      1|    } else {
-   LL|      1|        say("not both");
-   LL|      1|    }
-   LL|      2|}
-   LL|       |
-   LL|      3|fn mcdc_check_both(a: bool, b: bool) {
-   LL|      3|    if a && b {
-                          ^2
-  ------------------
-  |  Branch (LL:8): [True: 2, False: 1]
-  |  Branch (LL:13): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:14)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      1|        say("a and b");
-   LL|      2|    } else {
-   LL|      2|        say("not both");
-   LL|      2|    }
-   LL|      3|}
-   LL|       |
-   LL|      4|fn mcdc_check_tree_decision(a: bool, b: bool, c: bool) {
-   LL|       |    // This expression is intentionally written in a way
-   LL|       |    // where 100% branch coverage indicates 100% mcdc coverage.
-   LL|      4|    if a && (b || c) {
-                           ^3   ^2
-  ------------------
-  |  Branch (LL:8): [True: 3, False: 1]
-  |  Branch (LL:14): [True: 1, False: 2]
-  |  Branch (LL:19): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:21)
-  |
-  |  Number of Conditions: 3
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:14)
-  |     Condition C3 --> (LL:19)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3    Result
-  |  1 { F,  -,  -  = F      }
-  |  2 { T,  F,  F  = F      }
-  |  3 { T,  F,  T  = T      }
-  |  4 { T,  T,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,4)
-  |  C3-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      2|        say("pass");
-   LL|      2|    } else {
-   LL|      2|        say("reject");
-   LL|      2|    }
-   LL|      4|}
-   LL|       |
-   LL|      4|fn mcdc_check_not_tree_decision(a: bool, b: bool, c: bool) {
-   LL|       |    // Contradict to `mcdc_check_tree_decision`,
-   LL|       |    // 100% branch coverage of this expression does not indicate 100% mcdc coverage.
-   LL|      4|    if (a || b) && c {
-                           ^1
-  ------------------
-  |  Branch (LL:9): [True: 3, False: 1]
-  |  Branch (LL:14): [True: 1, False: 0]
-  |  Branch (LL:20): [True: 2, False: 2]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:21)
-  |
-  |  Number of Conditions: 3
-  |     Condition C1 --> (LL:9)
-  |     Condition C2 --> (LL:14)
-  |     Condition C3 --> (LL:20)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3    Result
-  |  1 { T,  -,  F  = F      }
-  |  2 { F,  T,  T  = T      }
-  |  3 { T,  -,  T  = T      }
-  |
-  |  C1-Pair: not covered
-  |  C2-Pair: not covered
-  |  C3-Pair: covered: (1,3)
-  |  MC/DC Coverage for Decision: 33.33%
-  |
-  ------------------
-   LL|      2|        say("pass");
-   LL|      2|    } else {
-   LL|      2|        say("reject");
-   LL|      2|    }
-   LL|      4|}
-   LL|       |
-   LL|      3|fn mcdc_nested_if(a: bool, b: bool, c: bool) {
-   LL|      3|    if a || b {
-                          ^0
-  ------------------
-  |  Branch (LL:8): [True: 3, False: 0]
-  |  Branch (LL:13): [True: 0, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:14)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { T,  -  = T      }
-  |
-  |  C1-Pair: not covered
-  |  C2-Pair: not covered
-  |  MC/DC Coverage for Decision: 0.00%
-  |
-  ------------------
-   LL|      3|        say("a or b");
-   LL|      3|        if b && c {
-                              ^2
-  ------------------
-  |  Branch (LL:12): [True: 2, False: 1]
-  |  Branch (LL:17): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:12) to (LL:18)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:12)
-  |     Condition C2 --> (LL:17)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      1|            say("b and c");
-   LL|      2|        }
-   LL|      0|    } else {
-   LL|      0|        say("neither a nor b");
-   LL|      0|    }
-   LL|      3|}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    mcdc_check_neither(false, false);
-   LL|       |    mcdc_check_neither(false, true);
-   LL|       |
-   LL|       |    mcdc_check_a(true, true);
-   LL|       |    mcdc_check_a(false, true);
-   LL|       |
-   LL|       |    mcdc_check_b(true, true);
-   LL|       |    mcdc_check_b(true, false);
-   LL|       |
-   LL|       |    mcdc_check_both(false, true);
-   LL|       |    mcdc_check_both(true, true);
-   LL|       |    mcdc_check_both(true, false);
-   LL|       |
-   LL|       |    mcdc_check_tree_decision(false, true, true);
-   LL|       |    mcdc_check_tree_decision(true, true, false);
-   LL|       |    mcdc_check_tree_decision(true, false, false);
-   LL|       |    mcdc_check_tree_decision(true, false, true);
-   LL|       |
-   LL|       |    mcdc_check_not_tree_decision(false, true, true);
-   LL|       |    mcdc_check_not_tree_decision(true, true, false);
-   LL|       |    mcdc_check_not_tree_decision(true, false, false);
-   LL|       |    mcdc_check_not_tree_decision(true, false, true);
-   LL|       |
-   LL|       |    mcdc_nested_if(true, false, true);
-   LL|       |    mcdc_nested_if(true, true, true);
-   LL|       |    mcdc_nested_if(true, true, false);
-   LL|       |}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn say(message: &str) {
-   LL|       |    core::hint::black_box(message);
-   LL|       |}
-
diff --git a/tests/coverage/mcdc/if.rs b/tests/coverage/mcdc/if.rs
deleted file mode 100644
index c4675f5d1672..000000000000
--- a/tests/coverage/mcdc/if.rs
+++ /dev/null
@@ -1,102 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=mcdc
-//@ llvm-cov-flags: --show-branches=count --show-mcdc
-
-fn mcdc_check_neither(a: bool, b: bool) {
-    if a && b {
-        say("a and b");
-    } else {
-        say("not both");
-    }
-}
-
-fn mcdc_check_a(a: bool, b: bool) {
-    if a && b {
-        say("a and b");
-    } else {
-        say("not both");
-    }
-}
-
-fn mcdc_check_b(a: bool, b: bool) {
-    if a && b {
-        say("a and b");
-    } else {
-        say("not both");
-    }
-}
-
-fn mcdc_check_both(a: bool, b: bool) {
-    if a && b {
-        say("a and b");
-    } else {
-        say("not both");
-    }
-}
-
-fn mcdc_check_tree_decision(a: bool, b: bool, c: bool) {
-    // This expression is intentionally written in a way
-    // where 100% branch coverage indicates 100% mcdc coverage.
-    if a && (b || c) {
-        say("pass");
-    } else {
-        say("reject");
-    }
-}
-
-fn mcdc_check_not_tree_decision(a: bool, b: bool, c: bool) {
-    // Contradict to `mcdc_check_tree_decision`,
-    // 100% branch coverage of this expression does not indicate 100% mcdc coverage.
-    if (a || b) && c {
-        say("pass");
-    } else {
-        say("reject");
-    }
-}
-
-fn mcdc_nested_if(a: bool, b: bool, c: bool) {
-    if a || b {
-        say("a or b");
-        if b && c {
-            say("b and c");
-        }
-    } else {
-        say("neither a nor b");
-    }
-}
-
-#[coverage(off)]
-fn main() {
-    mcdc_check_neither(false, false);
-    mcdc_check_neither(false, true);
-
-    mcdc_check_a(true, true);
-    mcdc_check_a(false, true);
-
-    mcdc_check_b(true, true);
-    mcdc_check_b(true, false);
-
-    mcdc_check_both(false, true);
-    mcdc_check_both(true, true);
-    mcdc_check_both(true, false);
-
-    mcdc_check_tree_decision(false, true, true);
-    mcdc_check_tree_decision(true, true, false);
-    mcdc_check_tree_decision(true, false, false);
-    mcdc_check_tree_decision(true, false, true);
-
-    mcdc_check_not_tree_decision(false, true, true);
-    mcdc_check_not_tree_decision(true, true, false);
-    mcdc_check_not_tree_decision(true, false, false);
-    mcdc_check_not_tree_decision(true, false, true);
-
-    mcdc_nested_if(true, false, true);
-    mcdc_nested_if(true, true, true);
-    mcdc_nested_if(true, true, false);
-}
-
-#[coverage(off)]
-fn say(message: &str) {
-    core::hint::black_box(message);
-}
diff --git a/tests/coverage/mcdc/inlined_expressions.cov-map b/tests/coverage/mcdc/inlined_expressions.cov-map
deleted file mode 100644
index d05ef368ba4a..000000000000
--- a/tests/coverage/mcdc/inlined_expressions.cov-map
+++ /dev/null
@@ -1,21 +0,0 @@
-Function name: inlined_expressions::inlined_instance
-Raw bytes (55): 0x[01, 01, 02, 01, 05, 05, 09, 07, 01, 07, 01, 00, 2e, 01, 01, 05, 00, 06, 28, 03, 02, 00, 05, 00, 0b, 30, 05, 02, 01, 02, 00, 00, 05, 00, 06, 05, 00, 0a, 00, 0b, 30, 09, 06, 02, 00, 00, 00, 0a, 00, 0b, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/inlined_expressions.rs
-Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-Number of file 0 mappings: 7
-- Code(Counter(0)) at (prev + 7, 1) to (start + 0, 46)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 6)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 5) to (start + 0, 11)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 5) to (start + 0, 6)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 10) to (start + 0, 11)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 10) to (start + 0, 11)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
diff --git a/tests/coverage/mcdc/inlined_expressions.coverage b/tests/coverage/mcdc/inlined_expressions.coverage
deleted file mode 100644
index cfe398995e4e..000000000000
--- a/tests/coverage/mcdc/inlined_expressions.coverage
+++ /dev/null
@@ -1,40 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=mcdc -Copt-level=z -Cllvm-args=--inline-threshold=0
-   LL|       |//@ llvm-cov-flags: --show-branches=count --show-mcdc
-   LL|       |
-   LL|       |#[inline(always)]
-   LL|      3|fn inlined_instance(a: bool, b: bool) -> bool {
-   LL|      3|    a && b
-                       ^2
-  ------------------
-  |  Branch (LL:5): [True: 2, False: 1]
-  |  Branch (LL:10): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:5) to (LL:11)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:5)
-  |     Condition C2 --> (LL:10)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      3|}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    let _ = inlined_instance(true, false);
-   LL|       |    let _ = inlined_instance(false, true);
-   LL|       |    let _ = inlined_instance(true, true);
-   LL|       |}
-
diff --git a/tests/coverage/mcdc/inlined_expressions.rs b/tests/coverage/mcdc/inlined_expressions.rs
deleted file mode 100644
index 15d4260f6e2c..000000000000
--- a/tests/coverage/mcdc/inlined_expressions.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=mcdc -Copt-level=z -Cllvm-args=--inline-threshold=0
-//@ llvm-cov-flags: --show-branches=count --show-mcdc
-
-#[inline(always)]
-fn inlined_instance(a: bool, b: bool) -> bool {
-    a && b
-}
-
-#[coverage(off)]
-fn main() {
-    let _ = inlined_instance(true, false);
-    let _ = inlined_instance(false, true);
-    let _ = inlined_instance(true, true);
-}
diff --git a/tests/coverage/mcdc/nested_if.cov-map b/tests/coverage/mcdc/nested_if.cov-map
deleted file mode 100644
index 853cdf2c5760..000000000000
--- a/tests/coverage/mcdc/nested_if.cov-map
+++ /dev/null
@@ -1,200 +0,0 @@
-Function name: nested_if::doubly_nested_if_in_condition
-Raw bytes (175): 0x[01, 01, 0f, 01, 05, 05, 11, 05, 09, 05, 37, 09, 0d, 05, 09, 05, 1f, 09, 15, 15, 19, 05, 2b, 09, 19, 09, 0d, 05, 37, 09, 0d, 01, 11, 15, 01, 0e, 01, 00, 45, 01, 01, 08, 00, 09, 28, 09, 02, 00, 08, 00, 4e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 30, 11, 06, 02, 00, 00, 00, 0d, 00, 4e, 05, 00, 10, 00, 11, 28, 06, 02, 00, 10, 00, 36, 30, 09, 16, 01, 00, 02, 00, 10, 00, 11, 30, 0d, 32, 02, 00, 00, 00, 15, 00, 36, 16, 00, 18, 00, 19, 28, 03, 02, 00, 18, 00, 1e, 30, 15, 1a, 01, 02, 00, 00, 18, 00, 19, 15, 00, 1d, 00, 1e, 30, 19, 22, 02, 00, 00, 00, 1d, 00, 1e, 19, 00, 21, 00, 25, 26, 00, 2f, 00, 34, 37, 00, 39, 00, 3e, 32, 00, 48, 00, 4c, 11, 00, 4f, 02, 06, 3a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/nested_if.rs
-Number of expressions: 15
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(4)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Expression(13, Add)
-- expression 4 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 5 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 6 operands: lhs = Counter(1), rhs = Expression(7, Add)
-- expression 7 operands: lhs = Counter(2), rhs = Counter(5)
-- expression 8 operands: lhs = Counter(5), rhs = Counter(6)
-- expression 9 operands: lhs = Counter(1), rhs = Expression(10, Add)
-- expression 10 operands: lhs = Counter(2), rhs = Counter(6)
-- expression 11 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 12 operands: lhs = Counter(1), rhs = Expression(13, Add)
-- expression 13 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 14 operands: lhs = Counter(0), rhs = Counter(4)
-Number of file 0 mappings: 21
-- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 69)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 9, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 78)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- MCDCBranch { true: Counter(4), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 78)
-    true  = c4
-    false = (c1 - c4)
-- Code(Counter(1)) at (prev + 0, 16) to (start + 0, 17)
-- MCDCDecision { bitmap_idx: 6, conditions_num: 2 } at (prev + 0, 16) to (start + 0, 54)
-- MCDCBranch { true: Counter(2), false: Expression(5, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 16) to (start + 0, 17)
-    true  = c2
-    false = (c1 - c2)
-- MCDCBranch { true: Counter(3), false: Expression(12, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 21) to (start + 0, 54)
-    true  = c3
-    false = (c1 - (c2 + c3))
-- Code(Expression(5, Sub)) at (prev + 0, 24) to (start + 0, 25)
-    = (c1 - c2)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 24) to (start + 0, 30)
-- MCDCBranch { true: Counter(5), false: Expression(6, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 24) to (start + 0, 25)
-    true  = c5
-    false = (c1 - (c2 + c5))
-- Code(Counter(5)) at (prev + 0, 29) to (start + 0, 30)
-- MCDCBranch { true: Counter(6), false: Expression(8, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 29) to (start + 0, 30)
-    true  = c6
-    false = (c5 - c6)
-- Code(Counter(6)) at (prev + 0, 33) to (start + 0, 37)
-- Code(Expression(9, Sub)) at (prev + 0, 47) to (start + 0, 52)
-    = (c1 - (c2 + c6))
-- Code(Expression(13, Add)) at (prev + 0, 57) to (start + 0, 62)
-    = (c2 + c3)
-- Code(Expression(12, Sub)) at (prev + 0, 72) to (start + 0, 76)
-    = (c1 - (c2 + c3))
-- Code(Counter(4)) at (prev + 0, 79) to (start + 2, 6)
-- Code(Expression(14, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c4)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c6
-
-Function name: nested_if::nested_if_in_condition
-Raw bytes (123): 0x[01, 01, 0a, 01, 05, 05, 11, 05, 09, 05, 09, 05, 23, 09, 0d, 09, 0d, 05, 23, 09, 0d, 01, 11, 0f, 01, 06, 01, 00, 35, 01, 01, 08, 00, 09, 28, 06, 02, 00, 08, 00, 2e, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 30, 11, 06, 02, 00, 00, 00, 0d, 00, 2e, 05, 00, 10, 00, 11, 28, 03, 02, 00, 10, 00, 16, 30, 09, 0e, 01, 00, 02, 00, 10, 00, 11, 0e, 00, 15, 00, 16, 30, 0d, 1e, 02, 00, 00, 00, 15, 00, 16, 23, 00, 19, 00, 1d, 1e, 00, 27, 00, 2c, 11, 00, 2f, 02, 06, 26, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/nested_if.rs
-Number of expressions: 10
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(4)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 4 operands: lhs = Counter(1), rhs = Expression(8, Add)
-- expression 5 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 6 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 7 operands: lhs = Counter(1), rhs = Expression(8, Add)
-- expression 8 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 9 operands: lhs = Counter(0), rhs = Counter(4)
-Number of file 0 mappings: 15
-- Code(Counter(0)) at (prev + 6, 1) to (start + 0, 53)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 6, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 46)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- MCDCBranch { true: Counter(4), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 46)
-    true  = c4
-    false = (c1 - c4)
-- Code(Counter(1)) at (prev + 0, 16) to (start + 0, 17)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 16) to (start + 0, 22)
-- MCDCBranch { true: Counter(2), false: Expression(3, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 16) to (start + 0, 17)
-    true  = c2
-    false = (c1 - c2)
-- Code(Expression(3, Sub)) at (prev + 0, 21) to (start + 0, 22)
-    = (c1 - c2)
-- MCDCBranch { true: Counter(3), false: Expression(7, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 21) to (start + 0, 22)
-    true  = c3
-    false = (c1 - (c2 + c3))
-- Code(Expression(8, Add)) at (prev + 0, 25) to (start + 0, 29)
-    = (c2 + c3)
-- Code(Expression(7, Sub)) at (prev + 0, 39) to (start + 0, 44)
-    = (c1 - (c2 + c3))
-- Code(Counter(4)) at (prev + 0, 47) to (start + 2, 6)
-- Code(Expression(9, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c4)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c4
-
-Function name: nested_if::nested_in_then_block_in_condition
-Raw bytes (175): 0x[01, 01, 0f, 01, 05, 05, 19, 05, 09, 05, 09, 05, 37, 09, 0d, 09, 0d, 37, 11, 09, 0d, 11, 15, 37, 15, 09, 0d, 05, 37, 09, 0d, 01, 19, 15, 01, 21, 01, 00, 52, 01, 01, 08, 00, 09, 28, 09, 02, 00, 08, 00, 4b, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 30, 19, 06, 02, 00, 00, 00, 0d, 00, 4b, 05, 00, 10, 00, 11, 28, 03, 02, 00, 10, 00, 16, 30, 09, 0e, 01, 00, 02, 00, 10, 00, 11, 0e, 00, 15, 00, 16, 30, 0d, 32, 02, 00, 00, 00, 15, 00, 16, 37, 00, 1c, 00, 1d, 28, 06, 02, 00, 1c, 00, 22, 30, 11, 1e, 01, 02, 00, 00, 1c, 00, 1d, 11, 00, 21, 00, 22, 30, 15, 26, 02, 00, 00, 00, 21, 00, 22, 15, 00, 25, 00, 29, 2a, 00, 33, 00, 38, 32, 00, 44, 00, 49, 19, 00, 4c, 02, 06, 3a, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/nested_if.rs
-Number of expressions: 15
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(6)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 4 operands: lhs = Counter(1), rhs = Expression(13, Add)
-- expression 5 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 6 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 7 operands: lhs = Expression(13, Add), rhs = Counter(4)
-- expression 8 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 9 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 10 operands: lhs = Expression(13, Add), rhs = Counter(5)
-- expression 11 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 12 operands: lhs = Counter(1), rhs = Expression(13, Add)
-- expression 13 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 14 operands: lhs = Counter(0), rhs = Counter(6)
-Number of file 0 mappings: 21
-- Code(Counter(0)) at (prev + 33, 1) to (start + 0, 82)
-- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 9, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 75)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- MCDCBranch { true: Counter(6), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 75)
-    true  = c6
-    false = (c1 - c6)
-- Code(Counter(1)) at (prev + 0, 16) to (start + 0, 17)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 16) to (start + 0, 22)
-- MCDCBranch { true: Counter(2), false: Expression(3, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 16) to (start + 0, 17)
-    true  = c2
-    false = (c1 - c2)
-- Code(Expression(3, Sub)) at (prev + 0, 21) to (start + 0, 22)
-    = (c1 - c2)
-- MCDCBranch { true: Counter(3), false: Expression(12, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 21) to (start + 0, 22)
-    true  = c3
-    false = (c1 - (c2 + c3))
-- Code(Expression(13, Add)) at (prev + 0, 28) to (start + 0, 29)
-    = (c2 + c3)
-- MCDCDecision { bitmap_idx: 6, conditions_num: 2 } at (prev + 0, 28) to (start + 0, 34)
-- MCDCBranch { true: Counter(4), false: Expression(7, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 28) to (start + 0, 29)
-    true  = c4
-    false = ((c2 + c3) - c4)
-- Code(Counter(4)) at (prev + 0, 33) to (start + 0, 34)
-- MCDCBranch { true: Counter(5), false: Expression(9, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 33) to (start + 0, 34)
-    true  = c5
-    false = (c4 - c5)
-- Code(Counter(5)) at (prev + 0, 37) to (start + 0, 41)
-- Code(Expression(10, Sub)) at (prev + 0, 51) to (start + 0, 56)
-    = ((c2 + c3) - c5)
-- Code(Expression(12, Sub)) at (prev + 0, 68) to (start + 0, 73)
-    = (c1 - (c2 + c3))
-- Code(Counter(6)) at (prev + 0, 76) to (start + 2, 6)
-- Code(Expression(14, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c6)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c6
-
-Function name: nested_if::nested_single_condition_decision
-Raw bytes (88): 0x[01, 01, 05, 01, 05, 05, 0d, 05, 09, 05, 09, 01, 0d, 0c, 01, 16, 01, 00, 36, 01, 04, 08, 00, 09, 28, 03, 02, 00, 08, 00, 29, 30, 05, 02, 01, 02, 00, 00, 08, 00, 09, 30, 0d, 06, 02, 00, 00, 00, 0d, 00, 29, 05, 00, 10, 00, 11, 20, 09, 0e, 00, 10, 00, 11, 09, 00, 14, 00, 19, 0e, 00, 23, 00, 27, 0d, 00, 2a, 02, 06, 12, 02, 0c, 02, 06, 01, 03, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/nested_if.rs
-Number of expressions: 5
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 4 operands: lhs = Counter(0), rhs = Counter(3)
-Number of file 0 mappings: 12
-- Code(Counter(0)) at (prev + 22, 1) to (start + 0, 54)
-- Code(Counter(0)) at (prev + 4, 8) to (start + 0, 9)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 8) to (start + 0, 41)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 8) to (start + 0, 9)
-    true  = c1
-    false = (c0 - c1)
-- MCDCBranch { true: Counter(3), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 41)
-    true  = c3
-    false = (c1 - c3)
-- Code(Counter(1)) at (prev + 0, 16) to (start + 0, 17)
-- Branch { true: Counter(2), false: Expression(3, Sub) } at (prev + 0, 16) to (start + 0, 17)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 20) to (start + 0, 25)
-- Code(Expression(3, Sub)) at (prev + 0, 35) to (start + 0, 39)
-    = (c1 - c2)
-- Code(Counter(3)) at (prev + 0, 42) to (start + 2, 6)
-- Code(Expression(4, Sub)) at (prev + 2, 12) to (start + 2, 6)
-    = (c0 - c3)
-- Code(Counter(0)) at (prev + 3, 1) to (start + 0, 2)
-Highest counter ID seen: c3
-
diff --git a/tests/coverage/mcdc/nested_if.coverage b/tests/coverage/mcdc/nested_if.coverage
deleted file mode 100644
index 8b5179b34fee..000000000000
--- a/tests/coverage/mcdc/nested_if.coverage
+++ /dev/null
@@ -1,257 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=mcdc
-   LL|       |//@ llvm-cov-flags: --show-branches=count --show-mcdc
-   LL|       |
-   LL|      4|fn nested_if_in_condition(a: bool, b: bool, c: bool) {
-   LL|      4|    if a && if b || c { true } else { false } {
-                             ^3   ^2  ^2            ^1
-  ------------------
-  |  Branch (LL:8): [True: 3, False: 1]
-  |  Branch (LL:13): [True: 2, False: 1]
-  |  Branch (LL:16): [True: 1, False: 2]
-  |  Branch (LL:21): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:46)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  |---> MC/DC Decision Region (LL:16) to (LL:22)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:16)
-  |     Condition C2 --> (LL:21)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  F  = F      }
-  |  2 { F,  T  = T      }
-  |  3 { T,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (1,2)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      2|        say("yes");
-   LL|      2|    } else {
-   LL|      2|        say("no");
-   LL|      2|    }
-   LL|      4|}
-   LL|       |
-   LL|      4|fn doubly_nested_if_in_condition(a: bool, b: bool, c: bool, d: bool) {
-   LL|      4|    if a && if b || if c && d { true } else { false } { false } else { true } {
-                             ^3      ^2   ^1  ^1            ^1        ^2             ^1
-  ------------------
-  |  Branch (LL:8): [True: 3, False: 1]
-  |  Branch (LL:13): [True: 1, False: 2]
-  |  Branch (LL:16): [True: 1, False: 2]
-  |  Branch (LL:21): [True: 1, False: 1]
-  |  Branch (LL:24): [True: 1, False: 1]
-  |  Branch (LL:29): [True: 1, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:78)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  |---> MC/DC Decision Region (LL:16) to (LL:54)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:16)
-  |     Condition C2 --> (LL:21)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  F  = F      }
-  |  2 { F,  T  = T      }
-  |  3 { T,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (1,2)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  |---> MC/DC Decision Region (LL:24) to (LL:30)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:24)
-  |     Condition C2 --> (LL:29)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,2)
-  |  C2-Pair: not covered
-  |  MC/DC Coverage for Decision: 50.00%
-  |
-  ------------------
-   LL|      1|        say("yes");
-   LL|      3|    } else {
-   LL|      3|        say("no");
-   LL|      3|    }
-   LL|      4|}
-   LL|       |
-   LL|      3|fn nested_single_condition_decision(a: bool, b: bool) {
-   LL|       |    // Decision with only 1 decision should not be instrumented by MCDC because
-   LL|       |    // branch-coverage is equivalent to MCDC coverage in this case, and we don't
-   LL|       |    // want to waste bitmap space for this.
-   LL|      3|    if a && if b { false } else { true } {
-                             ^2  ^1             ^1
-  ------------------
-  |  Branch (LL:8): [True: 2, False: 1]
-  |  Branch (LL:13): [True: 1, False: 1]
-  |  Branch (LL:16): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:41)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      1|        say("yes");
-   LL|      2|    } else {
-   LL|      2|        say("no");
-   LL|      2|    }
-   LL|      3|}
-   LL|       |
-   LL|      7|fn nested_in_then_block_in_condition(a: bool, b: bool, c: bool, d: bool, e: bool) {
-   LL|      7|    if a && if b || c { if d && e { true } else { false } } else { false } {
-                             ^6   ^5     ^5   ^2  ^1            ^4               ^1
-  ------------------
-  |  Branch (LL:8): [True: 6, False: 1]
-  |  Branch (LL:13): [True: 1, False: 5]
-  |  Branch (LL:16): [True: 1, False: 5]
-  |  Branch (LL:21): [True: 4, False: 1]
-  |  Branch (LL:28): [True: 2, False: 3]
-  |  Branch (LL:33): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:8) to (LL:75)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:8)
-  |     Condition C2 --> (LL:13)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  |---> MC/DC Decision Region (LL:16) to (LL:22)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:16)
-  |     Condition C2 --> (LL:21)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  F  = F      }
-  |  2 { F,  T  = T      }
-  |  3 { T,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (1,2)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  |---> MC/DC Decision Region (LL:28) to (LL:34)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:28)
-  |     Condition C2 --> (LL:33)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      1|        say("yes");
-   LL|      6|    } else {
-   LL|      6|        say("no");
-   LL|      6|    }
-   LL|      7|}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    nested_if_in_condition(true, false, false);
-   LL|       |    nested_if_in_condition(true, true, true);
-   LL|       |    nested_if_in_condition(true, false, true);
-   LL|       |    nested_if_in_condition(false, true, true);
-   LL|       |
-   LL|       |    doubly_nested_if_in_condition(true, false, false, true);
-   LL|       |    doubly_nested_if_in_condition(true, true, true, true);
-   LL|       |    doubly_nested_if_in_condition(true, false, true, true);
-   LL|       |    doubly_nested_if_in_condition(false, true, true, true);
-   LL|       |
-   LL|       |    nested_single_condition_decision(true, true);
-   LL|       |    nested_single_condition_decision(true, false);
-   LL|       |    nested_single_condition_decision(false, false);
-   LL|       |
-   LL|       |    nested_in_then_block_in_condition(false, false, false, false, false);
-   LL|       |    nested_in_then_block_in_condition(true, false, false, false, false);
-   LL|       |    nested_in_then_block_in_condition(true, true, false, false, false);
-   LL|       |    nested_in_then_block_in_condition(true, false, true, false, false);
-   LL|       |    nested_in_then_block_in_condition(true, false, true, true, false);
-   LL|       |    nested_in_then_block_in_condition(true, false, true, false, true);
-   LL|       |    nested_in_then_block_in_condition(true, false, true, true, true);
-   LL|       |}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn say(message: &str) {
-   LL|       |    core::hint::black_box(message);
-   LL|       |}
-
diff --git a/tests/coverage/mcdc/nested_if.rs b/tests/coverage/mcdc/nested_if.rs
deleted file mode 100644
index db02aece33c9..000000000000
--- a/tests/coverage/mcdc/nested_if.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=mcdc
-//@ llvm-cov-flags: --show-branches=count --show-mcdc
-
-fn nested_if_in_condition(a: bool, b: bool, c: bool) {
-    if a && if b || c { true } else { false } {
-        say("yes");
-    } else {
-        say("no");
-    }
-}
-
-fn doubly_nested_if_in_condition(a: bool, b: bool, c: bool, d: bool) {
-    if a && if b || if c && d { true } else { false } { false } else { true } {
-        say("yes");
-    } else {
-        say("no");
-    }
-}
-
-fn nested_single_condition_decision(a: bool, b: bool) {
-    // Decision with only 1 decision should not be instrumented by MCDC because
-    // branch-coverage is equivalent to MCDC coverage in this case, and we don't
-    // want to waste bitmap space for this.
-    if a && if b { false } else { true } {
-        say("yes");
-    } else {
-        say("no");
-    }
-}
-
-fn nested_in_then_block_in_condition(a: bool, b: bool, c: bool, d: bool, e: bool) {
-    if a && if b || c { if d && e { true } else { false } } else { false } {
-        say("yes");
-    } else {
-        say("no");
-    }
-}
-
-#[coverage(off)]
-fn main() {
-    nested_if_in_condition(true, false, false);
-    nested_if_in_condition(true, true, true);
-    nested_if_in_condition(true, false, true);
-    nested_if_in_condition(false, true, true);
-
-    doubly_nested_if_in_condition(true, false, false, true);
-    doubly_nested_if_in_condition(true, true, true, true);
-    doubly_nested_if_in_condition(true, false, true, true);
-    doubly_nested_if_in_condition(false, true, true, true);
-
-    nested_single_condition_decision(true, true);
-    nested_single_condition_decision(true, false);
-    nested_single_condition_decision(false, false);
-
-    nested_in_then_block_in_condition(false, false, false, false, false);
-    nested_in_then_block_in_condition(true, false, false, false, false);
-    nested_in_then_block_in_condition(true, true, false, false, false);
-    nested_in_then_block_in_condition(true, false, true, false, false);
-    nested_in_then_block_in_condition(true, false, true, true, false);
-    nested_in_then_block_in_condition(true, false, true, false, true);
-    nested_in_then_block_in_condition(true, false, true, true, true);
-}
-
-#[coverage(off)]
-fn say(message: &str) {
-    core::hint::black_box(message);
-}
diff --git a/tests/coverage/mcdc/non_control_flow.cov-map b/tests/coverage/mcdc/non_control_flow.cov-map
deleted file mode 100644
index f06bc2ed8168..000000000000
--- a/tests/coverage/mcdc/non_control_flow.cov-map
+++ /dev/null
@@ -1,186 +0,0 @@
-Function name: non_control_flow::assign_3
-Raw bytes (89): 0x[01, 01, 04, 01, 05, 01, 0b, 05, 09, 09, 0d, 0c, 01, 15, 01, 00, 27, 01, 01, 09, 00, 0a, 01, 00, 0d, 00, 0e, 28, 04, 03, 00, 0d, 00, 18, 30, 05, 02, 01, 00, 02, 00, 0d, 00, 0e, 02, 00, 12, 00, 13, 30, 09, 06, 02, 03, 00, 00, 12, 00, 13, 09, 00, 17, 00, 18, 30, 0d, 0e, 03, 00, 00, 00, 17, 00, 18, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 4
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(0), rhs = Expression(2, Add)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(2), rhs = Counter(3)
-Number of file 0 mappings: 12
-- Code(Counter(0)) at (prev + 21, 1) to (start + 0, 39)
-- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 10)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCDecision { bitmap_idx: 4, conditions_num: 3 } at (prev + 0, 13) to (start + 0, 24)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c1
-    false = (c0 - c1)
-- Code(Expression(0, Sub)) at (prev + 0, 18) to (start + 0, 19)
-    = (c0 - c1)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 3, false_next_id: 0 } at (prev + 0, 18) to (start + 0, 19)
-    true  = c2
-    false = (c0 - (c1 + c2))
-- Code(Counter(2)) at (prev + 0, 23) to (start + 0, 24)
-- MCDCBranch { true: Counter(3), false: Expression(3, Sub), condition_id: 3, true_next_id: 0, false_next_id: 0 } at (prev + 0, 23) to (start + 0, 24)
-    true  = c3
-    false = (c2 - c3)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c3
-
-Function name: non_control_flow::assign_3_bis
-Raw bytes (91): 0x[01, 01, 05, 01, 05, 05, 09, 01, 09, 01, 13, 09, 0d, 0c, 01, 1a, 01, 00, 2b, 01, 01, 09, 00, 0a, 01, 00, 0d, 00, 0e, 28, 05, 03, 00, 0d, 00, 18, 30, 05, 02, 01, 03, 02, 00, 0d, 00, 0e, 05, 00, 12, 00, 13, 30, 09, 06, 03, 00, 02, 00, 12, 00, 13, 0a, 00, 17, 00, 18, 30, 0d, 0e, 02, 00, 00, 00, 17, 00, 18, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 5
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(0), rhs = Counter(2)
-- expression 3 operands: lhs = Counter(0), rhs = Expression(4, Add)
-- expression 4 operands: lhs = Counter(2), rhs = Counter(3)
-Number of file 0 mappings: 12
-- Code(Counter(0)) at (prev + 26, 1) to (start + 0, 43)
-- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 10)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCDecision { bitmap_idx: 5, conditions_num: 3 } at (prev + 0, 13) to (start + 0, 24)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 3, false_next_id: 2 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 18) to (start + 0, 19)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 3, true_next_id: 0, false_next_id: 2 } at (prev + 0, 18) to (start + 0, 19)
-    true  = c2
-    false = (c1 - c2)
-- Code(Expression(2, Sub)) at (prev + 0, 23) to (start + 0, 24)
-    = (c0 - c2)
-- MCDCBranch { true: Counter(3), false: Expression(3, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 23) to (start + 0, 24)
-    true  = c3
-    false = (c0 - (c2 + c3))
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c3
-
-Function name: non_control_flow::assign_and
-Raw bytes (70): 0x[01, 01, 02, 01, 05, 05, 09, 0a, 01, 0b, 01, 00, 20, 01, 01, 09, 00, 0a, 01, 00, 0d, 00, 0e, 28, 03, 02, 00, 0d, 00, 13, 30, 05, 02, 01, 02, 00, 00, 0d, 00, 0e, 05, 00, 12, 00, 13, 30, 09, 06, 02, 00, 00, 00, 12, 00, 13, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-Number of file 0 mappings: 10
-- Code(Counter(0)) at (prev + 11, 1) to (start + 0, 32)
-- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 10)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 13) to (start + 0, 19)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 18) to (start + 0, 19)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 18) to (start + 0, 19)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: non_control_flow::assign_or
-Raw bytes (72): 0x[01, 01, 03, 01, 05, 01, 0b, 05, 09, 0a, 01, 10, 01, 00, 1f, 01, 01, 09, 00, 0a, 01, 00, 0d, 00, 0e, 28, 03, 02, 00, 0d, 00, 13, 30, 05, 02, 01, 00, 02, 00, 0d, 00, 0e, 02, 00, 12, 00, 13, 30, 09, 06, 02, 00, 00, 00, 12, 00, 13, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 3
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(0), rhs = Expression(2, Add)
-- expression 2 operands: lhs = Counter(1), rhs = Counter(2)
-Number of file 0 mappings: 10
-- Code(Counter(0)) at (prev + 16, 1) to (start + 0, 31)
-- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 10)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 13) to (start + 0, 19)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 0, false_next_id: 2 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c1
-    false = (c0 - c1)
-- Code(Expression(0, Sub)) at (prev + 0, 18) to (start + 0, 19)
-    = (c0 - c1)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 18) to (start + 0, 19)
-    true  = c2
-    false = (c0 - (c1 + c2))
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: non_control_flow::foo
-Raw bytes (24): 0x[01, 01, 00, 04, 01, 24, 01, 00, 18, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 0
-Number of file 0 mappings: 4
-- Code(Counter(0)) at (prev + 36, 1) to (start + 0, 24)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c0
-
-Function name: non_control_flow::func_call
-Raw bytes (60): 0x[01, 01, 02, 01, 05, 05, 09, 08, 01, 28, 01, 00, 1f, 01, 01, 05, 00, 08, 01, 00, 09, 00, 0a, 28, 03, 02, 00, 09, 00, 0f, 30, 05, 02, 01, 02, 00, 00, 09, 00, 0a, 05, 00, 0e, 00, 0f, 30, 09, 06, 02, 00, 00, 00, 0e, 00, 0f, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-Number of file 0 mappings: 8
-- Code(Counter(0)) at (prev + 40, 1) to (start + 0, 31)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 8)
-- Code(Counter(0)) at (prev + 0, 9) to (start + 0, 10)
-- MCDCDecision { bitmap_idx: 3, conditions_num: 2 } at (prev + 0, 9) to (start + 0, 15)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 9) to (start + 0, 10)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 14) to (start + 0, 15)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 0, false_next_id: 0 } at (prev + 0, 14) to (start + 0, 15)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c2
-
-Function name: non_control_flow::right_comb_tree
-Raw bytes (121): 0x[01, 01, 05, 01, 05, 05, 09, 09, 0d, 0d, 11, 11, 15, 10, 01, 1f, 01, 00, 40, 01, 01, 09, 00, 0a, 01, 00, 0d, 00, 0e, 28, 06, 05, 00, 0d, 00, 2a, 30, 05, 02, 01, 02, 00, 00, 0d, 00, 0e, 05, 00, 13, 00, 14, 30, 09, 06, 02, 03, 00, 00, 13, 00, 14, 09, 00, 19, 00, 1a, 30, 0d, 0a, 03, 04, 00, 00, 19, 00, 1a, 0d, 00, 1f, 00, 20, 30, 11, 0e, 04, 05, 00, 00, 1f, 00, 20, 11, 00, 24, 00, 27, 30, 15, 12, 05, 00, 00, 00, 24, 00, 27, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 10, 01, 01, 01, 00, 02]
-Number of files: 1
-- file 0 => $DIR/non_control_flow.rs
-Number of expressions: 5
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 2 operands: lhs = Counter(2), rhs = Counter(3)
-- expression 3 operands: lhs = Counter(3), rhs = Counter(4)
-- expression 4 operands: lhs = Counter(4), rhs = Counter(5)
-Number of file 0 mappings: 16
-- Code(Counter(0)) at (prev + 31, 1) to (start + 0, 64)
-- Code(Counter(0)) at (prev + 1, 9) to (start + 0, 10)
-- Code(Counter(0)) at (prev + 0, 13) to (start + 0, 14)
-- MCDCDecision { bitmap_idx: 6, conditions_num: 5 } at (prev + 0, 13) to (start + 0, 42)
-- MCDCBranch { true: Counter(1), false: Expression(0, Sub), condition_id: 1, true_next_id: 2, false_next_id: 0 } at (prev + 0, 13) to (start + 0, 14)
-    true  = c1
-    false = (c0 - c1)
-- Code(Counter(1)) at (prev + 0, 19) to (start + 0, 20)
-- MCDCBranch { true: Counter(2), false: Expression(1, Sub), condition_id: 2, true_next_id: 3, false_next_id: 0 } at (prev + 0, 19) to (start + 0, 20)
-    true  = c2
-    false = (c1 - c2)
-- Code(Counter(2)) at (prev + 0, 25) to (start + 0, 26)
-- MCDCBranch { true: Counter(3), false: Expression(2, Sub), condition_id: 3, true_next_id: 4, false_next_id: 0 } at (prev + 0, 25) to (start + 0, 26)
-    true  = c3
-    false = (c2 - c3)
-- Code(Counter(3)) at (prev + 0, 31) to (start + 0, 32)
-- MCDCBranch { true: Counter(4), false: Expression(3, Sub), condition_id: 4, true_next_id: 5, false_next_id: 0 } at (prev + 0, 31) to (start + 0, 32)
-    true  = c4
-    false = (c3 - c4)
-- Code(Counter(4)) at (prev + 0, 36) to (start + 0, 39)
-- MCDCBranch { true: Counter(5), false: Expression(4, Sub), condition_id: 5, true_next_id: 0, false_next_id: 0 } at (prev + 0, 36) to (start + 0, 39)
-    true  = c5
-    false = (c4 - c5)
-- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14)
-- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 16)
-- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2)
-Highest counter ID seen: c5
-
diff --git a/tests/coverage/mcdc/non_control_flow.coverage b/tests/coverage/mcdc/non_control_flow.coverage
deleted file mode 100644
index 419d40bb6f99..000000000000
--- a/tests/coverage/mcdc/non_control_flow.coverage
+++ /dev/null
@@ -1,224 +0,0 @@
-   LL|       |#![feature(coverage_attribute)]
-   LL|       |//@ edition: 2021
-   LL|       |//@ compile-flags: -Zcoverage-options=mcdc
-   LL|       |//@ llvm-cov-flags: --show-branches=count --show-mcdc
-   LL|       |
-   LL|       |// This test ensures that boolean expressions that are not inside control flow
-   LL|       |// decisions are correctly instrumented.
-   LL|       |
-   LL|       |use core::hint::black_box;
-   LL|       |
-   LL|      3|fn assign_and(a: bool, b: bool) {
-   LL|      3|    let x = a && b;
-                               ^2
-  ------------------
-  |  Branch (LL:13): [True: 2, False: 1]
-  |  Branch (LL:18): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:13) to (LL:19)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:13)
-  |     Condition C2 --> (LL:18)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      3|    black_box(x);
-   LL|      3|}
-   LL|       |
-   LL|      3|fn assign_or(a: bool, b: bool) {
-   LL|      3|    let x = a || b;
-                               ^1
-  ------------------
-  |  Branch (LL:13): [True: 2, False: 1]
-  |  Branch (LL:18): [True: 0, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:13) to (LL:19)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:13)
-  |     Condition C2 --> (LL:18)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  F  = F      }
-  |  2 { T,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,2)
-  |  C2-Pair: not covered
-  |  MC/DC Coverage for Decision: 50.00%
-  |
-  ------------------
-   LL|      3|    black_box(x);
-   LL|      3|}
-   LL|       |
-   LL|      4|fn assign_3(a: bool, b: bool, c: bool) {
-   LL|      4|    let x = a || b && c;
-                               ^2   ^1
-  ------------------
-  |  Branch (LL:13): [True: 2, False: 2]
-  |  Branch (LL:18): [True: 1, False: 1]
-  |  Branch (LL:23): [True: 1, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:13) to (LL:24)
-  |
-  |  Number of Conditions: 3
-  |     Condition C1 --> (LL:13)
-  |     Condition C2 --> (LL:18)
-  |     Condition C3 --> (LL:23)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3    Result
-  |  1 { F,  F,  -  = F      }
-  |  2 { F,  T,  T  = T      }
-  |  3 { T,  -,  -  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (1,2)
-  |  C3-Pair: not covered
-  |  MC/DC Coverage for Decision: 66.67%
-  |
-  ------------------
-   LL|      4|    black_box(x);
-   LL|      4|}
-   LL|       |
-   LL|      4|fn assign_3_bis(a: bool, b: bool, c: bool) {
-   LL|      4|    let x = a && b || c;
-                               ^2   ^3
-  ------------------
-  |  Branch (LL:13): [True: 2, False: 2]
-  |  Branch (LL:18): [True: 1, False: 1]
-  |  Branch (LL:23): [True: 2, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:13) to (LL:24)
-  |
-  |  Number of Conditions: 3
-  |     Condition C1 --> (LL:13)
-  |     Condition C2 --> (LL:18)
-  |     Condition C3 --> (LL:23)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3    Result
-  |  1 { T,  F,  F  = F      }
-  |  2 { F,  -,  T  = T      }
-  |  3 { T,  T,  -  = T      }
-  |
-  |  C1-Pair: not covered
-  |  C2-Pair: covered: (1,3)
-  |  C3-Pair: not covered
-  |  MC/DC Coverage for Decision: 33.33%
-  |
-  ------------------
-   LL|      4|    black_box(x);
-   LL|      4|}
-   LL|       |
-   LL|      3|fn right_comb_tree(a: bool, b: bool, c: bool, d: bool, e: bool) {
-   LL|      3|    let x = a && (b && (c && (d && (e))));
-                                ^2    ^1    ^1   ^1
-  ------------------
-  |  Branch (LL:13): [True: 2, False: 1]
-  |  Branch (LL:19): [True: 1, False: 1]
-  |  Branch (LL:25): [True: 1, False: 0]
-  |  Branch (LL:31): [True: 1, False: 0]
-  |  Branch (LL:36): [True: 1, False: 0]
-  ------------------
-  |---> MC/DC Decision Region (LL:13) to (LL:42)
-  |
-  |  Number of Conditions: 5
-  |     Condition C1 --> (LL:13)
-  |     Condition C2 --> (LL:19)
-  |     Condition C3 --> (LL:25)
-  |     Condition C4 --> (LL:31)
-  |     Condition C5 --> (LL:36)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2, C3, C4, C5    Result
-  |  1 { F,  -,  -,  -,  -  = F      }
-  |  2 { T,  F,  -,  -,  -  = F      }
-  |  3 { T,  T,  T,  T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  C3-Pair: not covered
-  |  C4-Pair: not covered
-  |  C5-Pair: not covered
-  |  MC/DC Coverage for Decision: 40.00%
-  |
-  ------------------
-   LL|      3|    black_box(x);
-   LL|      3|}
-   LL|       |
-   LL|      3|fn foo(a: bool) -> bool {
-   LL|      3|    black_box(a)
-   LL|      3|}
-   LL|       |
-   LL|      3|fn func_call(a: bool, b: bool) {
-   LL|      3|    foo(a && b);
-                           ^2
-  ------------------
-  |  Branch (LL:9): [True: 2, False: 1]
-  |  Branch (LL:14): [True: 1, False: 1]
-  ------------------
-  |---> MC/DC Decision Region (LL:9) to (LL:15)
-  |
-  |  Number of Conditions: 2
-  |     Condition C1 --> (LL:9)
-  |     Condition C2 --> (LL:14)
-  |
-  |  Executed MC/DC Test Vectors:
-  |
-  |     C1, C2    Result
-  |  1 { F,  -  = F      }
-  |  2 { T,  F  = F      }
-  |  3 { T,  T  = T      }
-  |
-  |  C1-Pair: covered: (1,3)
-  |  C2-Pair: covered: (2,3)
-  |  MC/DC Coverage for Decision: 100.00%
-  |
-  ------------------
-   LL|      3|}
-   LL|       |
-   LL|       |#[coverage(off)]
-   LL|       |fn main() {
-   LL|       |    assign_and(true, false);
-   LL|       |    assign_and(true, true);
-   LL|       |    assign_and(false, false);
-   LL|       |
-   LL|       |    assign_or(true, false);
-   LL|       |    assign_or(true, true);
-   LL|       |    assign_or(false, false);
-   LL|       |
-   LL|       |    assign_3(true, false, false);
-   LL|       |    assign_3(true, true, false);
-   LL|       |    assign_3(false, false, true);
-   LL|       |    assign_3(false, true, true);
-   LL|       |
-   LL|       |    assign_3_bis(true, false, false);
-   LL|       |    assign_3_bis(true, true, false);
-   LL|       |    assign_3_bis(false, false, true);
-   LL|       |    assign_3_bis(false, true, true);
-   LL|       |
-   LL|       |    right_comb_tree(false, false, false, true, true);
-   LL|       |    right_comb_tree(true, false, false, true, true);
-   LL|       |    right_comb_tree(true, true, true, true, true);
-   LL|       |
-   LL|       |    func_call(true, false);
-   LL|       |    func_call(true, true);
-   LL|       |    func_call(false, false);
-   LL|       |}
-
diff --git a/tests/coverage/mcdc/non_control_flow.rs b/tests/coverage/mcdc/non_control_flow.rs
deleted file mode 100644
index 863bb8a5756d..000000000000
--- a/tests/coverage/mcdc/non_control_flow.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-#![feature(coverage_attribute)]
-//@ edition: 2021
-//@ compile-flags: -Zcoverage-options=mcdc
-//@ llvm-cov-flags: --show-branches=count --show-mcdc
-
-// This test ensures that boolean expressions that are not inside control flow
-// decisions are correctly instrumented.
-
-use core::hint::black_box;
-
-fn assign_and(a: bool, b: bool) {
-    let x = a && b;
-    black_box(x);
-}
-
-fn assign_or(a: bool, b: bool) {
-    let x = a || b;
-    black_box(x);
-}
-
-fn assign_3(a: bool, b: bool, c: bool) {
-    let x = a || b && c;
-    black_box(x);
-}
-
-fn assign_3_bis(a: bool, b: bool, c: bool) {
-    let x = a && b || c;
-    black_box(x);
-}
-
-fn right_comb_tree(a: bool, b: bool, c: bool, d: bool, e: bool) {
-    let x = a && (b && (c && (d && (e))));
-    black_box(x);
-}
-
-fn foo(a: bool) -> bool {
-    black_box(a)
-}
-
-fn func_call(a: bool, b: bool) {
-    foo(a && b);
-}
-
-#[coverage(off)]
-fn main() {
-    assign_and(true, false);
-    assign_and(true, true);
-    assign_and(false, false);
-
-    assign_or(true, false);
-    assign_or(true, true);
-    assign_or(false, false);
-
-    assign_3(true, false, false);
-    assign_3(true, true, false);
-    assign_3(false, false, true);
-    assign_3(false, true, true);
-
-    assign_3_bis(true, false, false);
-    assign_3_bis(true, true, false);
-    assign_3_bis(false, false, true);
-    assign_3_bis(false, true, true);
-
-    right_comb_tree(false, false, false, true, true);
-    right_comb_tree(true, false, false, true, true);
-    right_comb_tree(true, true, true, true, true);
-
-    func_call(true, false);
-    func_call(true, true);
-    func_call(false, false);
-}
diff --git a/tests/coverage/remap-path-prefix.rs b/tests/coverage/remap-path-prefix.rs
new file mode 100644
index 000000000000..29c5826989c4
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.rs
@@ -0,0 +1,17 @@
+// This test makes sure that the files used in the coverage are remapped by
+// `--remap-path-prefix` and the `coverage` <- `object` scopes.
+//
+// We also test the `macro` scope to make sure it does not affect coverage.
+
+// When coverage paths are remapped, the coverage-run mode can't find source files (because
+// it doesn't know about the remapping), so it produces an empty coverage report. The empty
+// report (i.e. no `.coverage` files) helps to demonstrate that remapping was indeed performed.
+
+//@ revisions: with_remap with_coverage_scope with_object_scope with_macro_scope
+//@ compile-flags: --remap-path-prefix={{src-base}}=remapped
+//
+//@[with_coverage_scope] compile-flags: -Zremap-path-scope=coverage
+//@[with_object_scope] compile-flags: -Zremap-path-scope=object
+//@[with_macro_scope] compile-flags: -Zremap-path-scope=macro
+
+fn main() {}
diff --git a/tests/coverage/remap-path-prefix.with_coverage_scope.cov-map b/tests/coverage/remap-path-prefix.with_coverage_scope.cov-map
new file mode 100644
index 000000000000..35731d71116f
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.with_coverage_scope.cov-map
@@ -0,0 +1,10 @@
+Function name: remap_path_prefix::main
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 11, 01, 00, 0a, 01, 00, 0c, 00, 0d]
+Number of files: 1
+- file 0 => remapped/remap-path-prefix.rs
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 17, 1) to (start + 0, 10)
+- Code(Counter(0)) at (prev + 0, 12) to (start + 0, 13)
+Highest counter ID seen: c0
+
diff --git a/tests/coverage/remap-path-prefix.with_macro_scope.cov-map b/tests/coverage/remap-path-prefix.with_macro_scope.cov-map
new file mode 100644
index 000000000000..551e28676885
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.with_macro_scope.cov-map
@@ -0,0 +1,10 @@
+Function name: remap_path_prefix::main
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 11, 01, 00, 0a, 01, 00, 0c, 00, 0d]
+Number of files: 1
+- file 0 => $DIR/remap-path-prefix.rs
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 17, 1) to (start + 0, 10)
+- Code(Counter(0)) at (prev + 0, 12) to (start + 0, 13)
+Highest counter ID seen: c0
+
diff --git a/tests/coverage/remap-path-prefix.with_macro_scope.coverage b/tests/coverage/remap-path-prefix.with_macro_scope.coverage
new file mode 100644
index 000000000000..63979d8fe15a
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.with_macro_scope.coverage
@@ -0,0 +1,18 @@
+   LL|       |// This test makes sure that the files used in the coverage are remapped by
+   LL|       |// `--remap-path-prefix` and the `coverage` <- `object` scopes.
+   LL|       |//
+   LL|       |// We also test the `macro` scope to make sure it does not affect coverage.
+   LL|       |
+   LL|       |// When coverage paths are remapped, the coverage-run mode can't find source files (because
+   LL|       |// it doesn't know about the remapping), so it produces an empty coverage report. The empty
+   LL|       |// report (i.e. no `.coverage` files) helps to demonstrate that remapping was indeed performed.
+   LL|       |
+   LL|       |//@ revisions: with_remap with_coverage_scope with_object_scope with_macro_scope
+   LL|       |//@ compile-flags: --remap-path-prefix={{src-base}}=remapped
+   LL|       |//
+   LL|       |//@[with_coverage_scope] compile-flags: -Zremap-path-scope=coverage
+   LL|       |//@[with_object_scope] compile-flags: -Zremap-path-scope=object
+   LL|       |//@[with_macro_scope] compile-flags: -Zremap-path-scope=macro
+   LL|       |
+   LL|      1|fn main() {}
+
diff --git a/tests/coverage/remap-path-prefix.with_object_scope.cov-map b/tests/coverage/remap-path-prefix.with_object_scope.cov-map
new file mode 100644
index 000000000000..35731d71116f
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.with_object_scope.cov-map
@@ -0,0 +1,10 @@
+Function name: remap_path_prefix::main
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 11, 01, 00, 0a, 01, 00, 0c, 00, 0d]
+Number of files: 1
+- file 0 => remapped/remap-path-prefix.rs
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 17, 1) to (start + 0, 10)
+- Code(Counter(0)) at (prev + 0, 12) to (start + 0, 13)
+Highest counter ID seen: c0
+
diff --git a/tests/coverage/remap-path-prefix.with_remap.cov-map b/tests/coverage/remap-path-prefix.with_remap.cov-map
new file mode 100644
index 000000000000..35731d71116f
--- /dev/null
+++ b/tests/coverage/remap-path-prefix.with_remap.cov-map
@@ -0,0 +1,10 @@
+Function name: remap_path_prefix::main
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 11, 01, 00, 0a, 01, 00, 0c, 00, 0d]
+Number of files: 1
+- file 0 => remapped/remap-path-prefix.rs
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 17, 1) to (start + 0, 10)
+- Code(Counter(0)) at (prev + 0, 12) to (start + 0, 13)
+Highest counter ID seen: c0
+
diff --git a/tests/coverage/rustfmt-skip.cov-map b/tests/coverage/rustfmt-skip.cov-map
new file mode 100644
index 000000000000..bb673a411bfd
--- /dev/null
+++ b/tests/coverage/rustfmt-skip.cov-map
@@ -0,0 +1,12 @@
+Function name: rustfmt_skip::main
+Raw bytes (24): 0x[01, 01, 00, 04, 01, 0a, 01, 00, 0a, 01, 02, 05, 00, 0d, 01, 03, 09, 00, 10, 01, 02, 01, 00, 02]
+Number of files: 1
+- file 0 => $DIR/rustfmt-skip.rs
+Number of expressions: 0
+Number of file 0 mappings: 4
+- Code(Counter(0)) at (prev + 10, 1) to (start + 0, 10)
+- Code(Counter(0)) at (prev + 2, 5) to (start + 0, 13)
+- Code(Counter(0)) at (prev + 3, 9) to (start + 0, 16)
+- Code(Counter(0)) at (prev + 2, 1) to (start + 0, 2)
+Highest counter ID seen: c0
+
diff --git a/tests/coverage/rustfmt-skip.coverage b/tests/coverage/rustfmt-skip.coverage
new file mode 100644
index 000000000000..b7276cf0ee8b
--- /dev/null
+++ b/tests/coverage/rustfmt-skip.coverage
@@ -0,0 +1,18 @@
+   LL|       |//@ edition: 2024
+   LL|       |
+   LL|       |// The presence of `#[rustfmt::skip]` on a function should not cause macros
+   LL|       |// within that function to mysteriously not be instrumented.
+   LL|       |//
+   LL|       |// This test detects problems that can occur when building an expansion tree
+   LL|       |// based on `ExpnData::parent` instead of `ExpnData::call_site`, for example.
+   LL|       |
+   LL|       |#[rustfmt::skip]
+   LL|      1|fn main() {
+   LL|       |    // Ensure a gap between the body start and the first statement.
+   LL|      1|    println!(
+   LL|       |        // Keep this on a separate line, to distinguish instrumentation of
+   LL|       |        // `println!` from instrumentation of its arguments.
+   LL|      1|        "hello"
+   LL|       |    );
+   LL|      1|}
+
diff --git a/tests/coverage/rustfmt-skip.rs b/tests/coverage/rustfmt-skip.rs
new file mode 100644
index 000000000000..6f6874c9aa0a
--- /dev/null
+++ b/tests/coverage/rustfmt-skip.rs
@@ -0,0 +1,17 @@
+//@ edition: 2024
+
+// The presence of `#[rustfmt::skip]` on a function should not cause macros
+// within that function to mysteriously not be instrumented.
+//
+// This test detects problems that can occur when building an expansion tree
+// based on `ExpnData::parent` instead of `ExpnData::call_site`, for example.
+
+#[rustfmt::skip]
+fn main() {
+    // Ensure a gap between the body start and the first statement.
+    println!(
+        // Keep this on a separate line, to distinguish instrumentation of
+        // `println!` from instrumentation of its arguments.
+        "hello"
+    );
+}
diff --git a/tests/coverage/try-in-macro.attr.cov-map b/tests/coverage/try-in-macro.attr.cov-map
index 7111e89637ce..1b6c97cf145a 100644
--- a/tests/coverage/try-in-macro.attr.cov-map
+++ b/tests/coverage/try-in-macro.attr.cov-map
@@ -1,12 +1,3 @@
-Function name: ::try_size_hint
-Raw bytes (9): 0x[01, 01, 00, 01, 00, 1e, 2a, 00, 2b]
-Number of files: 1
-- file 0 => $DIR/try-in-macro.rs
-Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 30, 42) to (start + 0, 43)
-Highest counter ID seen: (none)
-
 Function name: try_in_macro::main
 Raw bytes (19): 0x[01, 01, 00, 03, 01, 29, 01, 00, 0a, 01, 01, 05, 00, 1a, 01, 01, 01, 00, 02]
 Number of files: 1
diff --git a/tests/coverage/try-in-macro.bang.cov-map b/tests/coverage/try-in-macro.bang.cov-map
index 80bd91a993c0..1b6c97cf145a 100644
--- a/tests/coverage/try-in-macro.bang.cov-map
+++ b/tests/coverage/try-in-macro.bang.cov-map
@@ -1,12 +1,3 @@
-Function name: ::try_size_hint
-Raw bytes (9): 0x[01, 01, 00, 01, 00, 23, 1c, 00, 1d]
-Number of files: 1
-- file 0 => $DIR/try-in-macro.rs
-Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 35, 28) to (start + 0, 29)
-Highest counter ID seen: (none)
-
 Function name: try_in_macro::main
 Raw bytes (19): 0x[01, 01, 00, 03, 01, 29, 01, 00, 0a, 01, 01, 05, 00, 1a, 01, 01, 01, 00, 02]
 Number of files: 1
diff --git a/tests/coverage/try-in-macro.derive.cov-map b/tests/coverage/try-in-macro.derive.cov-map
index 6646b6693bae..1b6c97cf145a 100644
--- a/tests/coverage/try-in-macro.derive.cov-map
+++ b/tests/coverage/try-in-macro.derive.cov-map
@@ -1,12 +1,3 @@
-Function name: ::try_size_hint
-Raw bytes (9): 0x[01, 01, 00, 01, 00, 26, 38, 00, 39]
-Number of files: 1
-- file 0 => $DIR/try-in-macro.rs
-Number of expressions: 0
-Number of file 0 mappings: 1
-- Code(Zero) at (prev + 38, 56) to (start + 0, 57)
-Highest counter ID seen: (none)
-
 Function name: try_in_macro::main
 Raw bytes (19): 0x[01, 01, 00, 03, 01, 29, 01, 00, 0a, 01, 01, 05, 00, 1a, 01, 01, 01, 00, 02]
 Number of files: 1
diff --git a/tests/crashes/105275.rs b/tests/crashes/105275.rs
deleted file mode 100644
index a97f36d19872..000000000000
--- a/tests/crashes/105275.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//@ known-bug: #105275
-//@ compile-flags: -Copt-level=0
-
-pub fn encode_num(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
-    if n > 15 {
-        encode_num(n / 16, &mut writer)?;
-    }
-    Ok(())
-}
-
-pub trait ExampleWriter {
-    type Error;
-}
-
-impl<'a, T: ExampleWriter> ExampleWriter for &'a mut T {
-    type Error = T::Error;
-}
-
-struct Error;
-
-impl ExampleWriter for Error {
-    type Error = ();
-}
-
-fn main() {
-    encode_num(69, &mut Error).unwrap();
-}
diff --git a/tests/crashes/105937.rs b/tests/crashes/105937.rs
deleted file mode 100644
index ffd1a493e46d..000000000000
--- a/tests/crashes/105937.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//@ known-bug: #105937
-//@ compile-flags: -Copt-level=0
-
-pub fn encode_num(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
-    if n > 15 {
-        encode_num(n / 16, &mut writer)?;
-    }
-    Ok(())
-}
-
-pub trait ExampleWriter {
-    type Error;
-}
-
-impl<'a, T: ExampleWriter> ExampleWriter for &'a mut T {
-    type Error = T::Error;
-}
-
-struct Error;
-
-impl ExampleWriter for Error {
-    type Error = ();
-}
-
-fn main() {
-    encode_num(69, &mut Error).unwrap();
-}
diff --git a/tests/crashes/114663.rs b/tests/crashes/114663.rs
deleted file mode 100644
index 406371f12e46..000000000000
--- a/tests/crashes/114663.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-//@ known-bug: #114663
-//@ edition:2021
-
-#![feature(generic_const_exprs)]
-
-use core::fmt::Debug;
-
-struct Inline
-where
-    [u8; ::core::mem::size_of::() + 1]:,
-{
-    _phantom: PhantomData,
-}
-
-fn main() {
-    let dst = Inline::::new(0); // BANG!
-}
diff --git a/tests/crashes/117629.rs b/tests/crashes/117629.rs
deleted file mode 100644
index f63365395c6b..000000000000
--- a/tests/crashes/117629.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-//@ known-bug: #117629
-//@ edition:2021
-
-#![feature(const_trait_impl)]
-
-const trait Tr {
-    async fn ft1() {}
-}
-
-fn main() {}
diff --git a/tests/crashes/117696-1.rs b/tests/crashes/117696-1.rs
deleted file mode 100644
index dfca39177855..000000000000
--- a/tests/crashes/117696-1.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-//@ known-bug: #117696
-fn main() {
-    let mut it = (Empty);
-    rec(&mut it);
-}
-
-struct Empty;
-
-impl Iterator for Empty {
-    type Item = ();
-    fn next<'a>(&'a mut self) -> core::option::Option<()> {
-        None
-    }
-}
-
-fn identity(x: T) -> T {
-    x
-}
-
-fn rec(mut it: T)
-where
-    T: Iterator,
-{
-    if () == () {
-        T::count(it);
-    } else {
-        rec(identity(&mut it))
-    }
-}
diff --git a/tests/crashes/117696-2.rs b/tests/crashes/117696-2.rs
deleted file mode 100644
index 9c2a68d3a915..000000000000
--- a/tests/crashes/117696-2.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: #117696
-//@ compile-flags: -Copt-level=0
-fn main() {
-    rec(&mut None::<()>.into_iter());
-}
-
-fn rec(mut it: T) {
-    if true {
-        it.next();
-    } else {
-        rec(&mut it);
-    }
-}
diff --git a/tests/crashes/117808.rs b/tests/crashes/117808.rs
deleted file mode 100644
index 2c727986dd07..000000000000
--- a/tests/crashes/117808.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//@ known-bug: #117808
-//@ edition:2021
-//@ needs-rustc-debug-assertions
-
-use std::future::Future;
-
-fn hrc AsyncClosure<'a, (), R>>(f: F) -> F {
-    f
-}
-
-fn main() {
-    hrc(|x| async {});
-}
-
-trait AsyncClosure<'a, I, R>
-where
-    I: 'a,
-{
-}
-
-impl<'a, I, R, Fut, F> AsyncClosure<'a, I, R> for F
-where
-    I: 'a,
-    F: Fn(&'a I) -> Fut,
-    Fut: Future + Send + 'a,
-{
-}
diff --git a/tests/crashes/118244.rs b/tests/crashes/118244.rs
index bfa1f5b7dd48..0330d0d838ab 100644
--- a/tests/crashes/118244.rs
+++ b/tests/crashes/118244.rs
@@ -1,5 +1,6 @@
 //@ known-bug: #118244
 //@ compile-flags: -Cdebuginfo=2
+//@ ignore-backends: gcc
 
 #![allow(incomplete_features)]
 #![feature(generic_const_exprs)]
diff --git a/tests/crashes/118590.rs b/tests/crashes/118590.rs
deleted file mode 100644
index 829c16582dc9..000000000000
--- a/tests/crashes/118590.rs
+++ /dev/null
@@ -1,11 +0,0 @@
-//@ known-bug: #118590
-
-fn main() {
-    recurse(std::iter::empty::<()>())
-}
-
-fn recurse(nums: impl Iterator) {
-    if true { return }
-
-    recurse(nums.skip(42).peekable())
-}
diff --git a/tests/crashes/119729.rs b/tests/crashes/119729.rs
deleted file mode 100644
index ed07c58e89fd..000000000000
--- a/tests/crashes/119729.rs
+++ /dev/null
@@ -1,12 +0,0 @@
-//@ known-bug: #119729
-#![feature(generic_const_exprs)]
-
-trait Size {}
-
-impl Size<{ std::mem::size_of::() }> for T {}
-
-struct A + ?Sized> {
-    x: std::marker::PhantomData,
-}
-
-fn foo(x: A) {}
diff --git a/tests/crashes/120016.rs b/tests/crashes/120016.rs
index 7eda330e7ade..12f54dbc3d92 100644
--- a/tests/crashes/120016.rs
+++ b/tests/crashes/120016.rs
@@ -1,19 +1,19 @@
 //@ known-bug: #120016
-//@ compile-flags: -Zcrate-attr=feature(const_async_blocks)
+//@ compile-flags: -Zvalidate-mir
 //@ edition: 2021
 
-#![feature(type_alias_impl_trait, const_async_blocks)]
+#![feature(type_alias_impl_trait)]
 
 struct Bug {
     V1: [(); {
-        type F = impl std::future::Future;
+        type F = impl Sized;
         #[define_opaque(F)]
         fn concrete_use() -> F {
-            //~^ ERROR to be a future that resolves to `u8`, but it resolves to `()`
-            async {}
+            //~^ ERROR
+            1i32
         }
-        let f: F = async { 1 };
-        //~^ ERROR `async` blocks are not allowed in constants
+        let f: F = 0u32;
+
         1
     }],
 }
diff --git a/tests/crashes/120175.rs b/tests/crashes/120175.rs
index c6e7203ff988..e06da5a8e0ae 100644
--- a/tests/crashes/120175.rs
+++ b/tests/crashes/120175.rs
@@ -1,9 +1,11 @@
 //@ known-bug: #120175
 //@ needs-rustc-debug-assertions
+//@ ignore-apple (raw-dylib doesn't work on Apple targets yet)
 
 #![feature(extern_types)]
+#![feature(raw_dylib_elf)]
 
-#[link(name = "bar", import_name_type = "decorated", kind = "raw-dylib")]
+#[link(name = "bar", kind = "raw-dylib")]
 extern "C" {
     pub type CrossCrate;
 }
diff --git a/tests/crashes/122630.rs b/tests/crashes/122630.rs
deleted file mode 100644
index e66624431c51..000000000000
--- a/tests/crashes/122630.rs
+++ /dev/null
@@ -1,22 +0,0 @@
-//@ known-bug: #122630
-//@ compile-flags: -Zvalidate-mir
-
-use std::ops::Coroutine;
-
-const FOO_SIZE: usize = 1024;
-struct Foo([u8; FOO_SIZE]);
-
-impl Drop for Foo {
-    fn move_before_yield_with_noop() -> impl Coroutine {}
-}
-
-fn overlap_move_points() -> impl Coroutine {
-    static || {
-        let first = Foo([0; FOO_SIZE]);
-        yield;
-        let second = first;
-        yield;
-        let second = first;
-        yield;
-    }
-}
diff --git a/tests/crashes/122823.rs b/tests/crashes/122823.rs
deleted file mode 100644
index ec22b331ad92..000000000000
--- a/tests/crashes/122823.rs
+++ /dev/null
@@ -1,69 +0,0 @@
-//@ known-bug: #122823
-//@ compile-flags: -Copt-level=0
-// ignore-tidy-linelength
-
-use std::vec::Vec;
-use std::iter::Peekable;
-
-pub fn main() {
-    let packet = decode(vec![1,0,1,0]);
-}
-
-pub fn decode(bitstream: Vec) -> Packet {
-    let mut bitstream_itr = bitstream.into_iter().peekable();
-    return match decode_packet(&mut bitstream_itr) {
-        Some(p) => p,
-        None    => panic!("expected outer packet"),
-    }
-}
-
-pub fn decode_packets>(itr: &mut Peekable) -> Vec {
-    let mut res = Vec::new();
-    loop {
-        match decode_packet(itr) {
-            Some(p) => { res.push(p); },
-            None    => break
-        }
-    }
-
-    return res;
-}
-
-pub fn decode_packet>(itr: &mut Peekable) -> Option {
-    // get version digits
-    let version = extend_number(0, itr, 3)?;
-    let type_id = extend_number(0, itr, 3)?;
-    return operator_packet(version, type_id, itr);
-}
-
-pub fn operator_packet>(version: u64, type_id: u64, itr: &mut Peekable) -> Option {
-    let p = OperatorPacket {
-        version: version,
-        type_id: type_id,
-        packets: decode_packets(&mut itr.take(0).peekable()),
-    };
-
-    return Some(Packet::Operator(p));
-}
-
-pub fn extend_number>(num: u64, itr: &mut Peekable, take: u64) -> Option {
-    let mut value = num;
-    for _ in 0..take {
-        value *= 2;
-        value += itr.next()?;
-    }
-
-    return Some(value);
-}
-
-#[derive(Debug)]
-pub enum Packet {
-    Operator(OperatorPacket),
-}
-
-#[derive(Debug)]
-pub struct OperatorPacket {
-    version: u64,
-    type_id: u64,
-    packets: Vec
-}
diff --git a/tests/crashes/122904-2.rs b/tests/crashes/122904-2.rs
deleted file mode 100644
index db66b8625db0..000000000000
--- a/tests/crashes/122904-2.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-//@ known-bug: #122904
-trait T {}
-
-type Alias<'a> = impl T;
-
-struct S;
-impl<'a> T for &'a S {}
-
-#[define_opaque(Alias)]
-fn with_positive(fun: impl Fn(Alias<'_>)) {
-    with_positive(|&n| ());
-}
-
-#[define_opaque(Alias)]
-fn main(Alias<'_>) {
-    with_positive(|&a| ());
-}
diff --git a/tests/crashes/124352.rs b/tests/crashes/124352.rs
deleted file mode 100644
index e9eb4419e6a4..000000000000
--- a/tests/crashes/124352.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-//@ known-bug: #124352
-#![rustc_never_type_options(: Unsize = "hi")]
-
-fn main() {}
diff --git a/tests/crashes/125185.rs b/tests/crashes/125185.rs
deleted file mode 100644
index e77666ca73d3..000000000000
--- a/tests/crashes/125185.rs
+++ /dev/null
@@ -1,26 +0,0 @@
-//@ known-bug: rust-lang/rust#125185
-//@ compile-flags: -Zvalidate-mir
-
-#![feature(type_alias_impl_trait)]
-
-type Foo = impl Send;
-
-struct A;
-
-#[define_opaque(Foo)]
-const fn foo() -> Foo {
-    value()
-}
-
-const VALUE: Foo = foo();
-
-#[define_opaque(Foo)]
-fn test(foo: Foo, f: impl for<'b> FnMut()) {
-    match VALUE {
-        0 | 0 => {}
-
-        _ => (),
-    }
-}
-
-fn main() {}
diff --git a/tests/crashes/125323.rs b/tests/crashes/125323.rs
deleted file mode 100644
index 180b7bbad097..000000000000
--- a/tests/crashes/125323.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-//@ known-bug: rust-lang/rust#125323
-fn main() {
-    for _ in 0..0 {
-        [(); loop {}];
-    }
-}
diff --git a/tests/crashes/125772.rs b/tests/crashes/125772.rs
index 2965cfc9e7c3..2b6cffd9463a 100644
--- a/tests/crashes/125772.rs
+++ b/tests/crashes/125772.rs
@@ -1,5 +1,5 @@
 //@ known-bug: rust-lang/rust#125772
-//@ only-x86_64
+//@ only-64bit
 #![feature(generic_const_exprs)]
 
 struct Outer();
diff --git a/tests/crashes/129095.rs b/tests/crashes/129095.rs
deleted file mode 100644
index b1bb74708c2d..000000000000
--- a/tests/crashes/129095.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: rust-lang/rust#129095
-//@ compile-flags: -Zmir-enable-passes=+GVN -Zmir-enable-passes=+Inline -Zvalidate-mir
-
-#![feature(adt_const_params, unsized_const_params)]
-#![allow(incomplete_features)]
-
-pub fn function_with_bytes() -> &'static [u8] {
-    BYTES
-}
-
-pub fn main() {
-    assert_eq!(function_with_bytes::(), &[0x41, 0x41, 0x41, 0x41]);
-}
diff --git a/tests/crashes/131292.rs b/tests/crashes/131292.rs
index 01e0eca0bd6d..05b93d06b055 100644
--- a/tests/crashes/131292.rs
+++ b/tests/crashes/131292.rs
@@ -1,5 +1,5 @@
 //@ known-bug: #131292
-//@ only-x86_64
+//@ needs-asm-support
 use std::arch::asm;
 
 unsafe fn f6() {
diff --git a/tests/crashes/131342.rs b/tests/crashes/131342.rs
deleted file mode 100644
index f4404092917a..000000000000
--- a/tests/crashes/131342.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-//@ known-bug: #131342
-
-fn main() {
-    let mut items = vec![1, 2, 3, 4, 5].into_iter();
-    problem_thingy(&mut items);
-}
-
-fn problem_thingy(items: &mut impl Iterator) {
-    let mut peeker = items.peekable();
-    match peeker.peek() {
-        Some(_) => (),
-        None => return (),
-    }
-    problem_thingy(&mut peeker);
-}
diff --git a/tests/crashes/132142.rs b/tests/crashes/132142.rs
deleted file mode 100644
index 813bf0bf0a8e..000000000000
--- a/tests/crashes/132142.rs
+++ /dev/null
@@ -1,3 +0,0 @@
-//@ known-bug: #132142
-
-async extern "cmse-nonsecure-entry" fn fun(...) {}
diff --git a/tests/crashes/134174.rs b/tests/crashes/134174.rs
deleted file mode 100644
index 899cdc6faf35..000000000000
--- a/tests/crashes/134174.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-//@ known-bug: #134175
-//@compile-flags: -Zvalidate-mir -Zinline-mir=yes
-use std::vec::IntoIter;
-
-pub(crate) trait Foo: Iterator::Key> {
-    type Key;
-}
-
-impl Foo for IntoIter {}
-
-fn sum_foo>(f: F) -> i32 {
-    f.fold(0, |a, b| a + b)
-}
-
-fn main() {
-    let x = sum_foo(vec![11, 10, 1].into_iter());
-}
diff --git a/tests/crashes/134479.rs b/tests/crashes/134479.rs
index 0e4ddb2bfd56..02047050cec4 100644
--- a/tests/crashes/134479.rs
+++ b/tests/crashes/134479.rs
@@ -1,5 +1,6 @@
 //@ known-bug: #134479
 //@ compile-flags: -Csymbol-mangling-version=v0 -Cdebuginfo=1
+//@ ignore-backends: gcc
 
 #![feature(generic_const_exprs)]
 
diff --git a/tests/crashes/134654.rs b/tests/crashes/134654.rs
deleted file mode 100644
index f2323fe4ecdc..000000000000
--- a/tests/crashes/134654.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-//@ known-bug: #134654
-//@ compile-flags: -Zmir-enable-passes=+GVN -Zmir-enable-passes=+Inline -Zvalidate-mir
-//@ only-x86_64
-
-#![feature(adt_const_params, unsized_const_params)]
-#![allow(incomplete_features)]
-
-fn function_with_bytes() -> &'static [u8] {
-    BYTES
-}
-
-fn main() {
-    function_with_bytes::() == &[];
-}
diff --git a/tests/crashes/135528.rs b/tests/crashes/135528.rs
deleted file mode 100644
index 171550e209e5..000000000000
--- a/tests/crashes/135528.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-//@ known-bug: #135528
-//@ compile-flags: -Zvalidate-mir -Zinline-mir=yes
-#![feature(type_alias_impl_trait)]
-type Tait = impl Copy;
-
-fn set(x: &isize) -> isize {
-    *x
-}
-
-#[define_opaque(Tait)]
-fn d(x: Tait) {
-    set(x);
-}
-
-#[define_opaque(Tait)]
-fn other_define() -> Tait {
-    ()
-}
-
-fn main() {}
diff --git a/tests/crashes/135570.rs b/tests/crashes/135570.rs
deleted file mode 100644
index 7919ceb26d50..000000000000
--- a/tests/crashes/135570.rs
+++ /dev/null
@@ -1,15 +0,0 @@
-//@ known-bug: #135570
-//@compile-flags: -Zvalidate-mir -Zmir-enable-passes=+Inline -Copt-level=0 -Zmir-enable-passes=+GVN
-//@ only-x86_64
-
-#![feature(adt_const_params, unsized_const_params)]
-#![allow(incomplete_features)]
-
-fn function_with_bytes(
-) -> &'static [u8] {
-    BYTES
-}
-
-fn main() {
-    function_with_bytes::() == &[];
-}
diff --git a/tests/crashes/135646.rs b/tests/crashes/135646.rs
deleted file mode 100644
index 841ea5b81b41..000000000000
--- a/tests/crashes/135646.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-//@ known-bug: #135646
-//@ compile-flags: -Zpolonius=next
-//@ edition: 2024
-
-fn main() {
-    &{ [1, 2, 3][4] };
-}
diff --git a/tests/crashes/136175.rs b/tests/crashes/136175.rs
deleted file mode 100644
index 0b5f2fdaa922..000000000000
--- a/tests/crashes/136175.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: #136175
-#![feature(generic_const_exprs)]
-#![allow(incomplete_features)]
-
-trait Trait {}
-
-struct A(T)
-where
-    [(); size_of::()]:;
-
-fn main() {
-    let x: A;
-}
diff --git a/tests/crashes/136381.rs b/tests/crashes/136381.rs
deleted file mode 100644
index 13ccc14a2c5b..000000000000
--- a/tests/crashes/136381.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-//@ known-bug: #136381
-//@ compile-flags: -Zvalidate-mir -Zmir-enable-passes=+GVN
-#![feature(trait_upcasting)]
-
-trait A {}
-trait B: A {
-    fn c(&self);
-}
-impl B for i32 {
-    fn c(self) {
-        todo!();
-    }
-}
-
-fn main() {
-    let baz: &dyn B = &1;
-    let bar: &dyn A = baz;
-}
diff --git a/tests/crashes/137190-1.rs b/tests/crashes/137190-1.rs
deleted file mode 100644
index bdfe883b7120..000000000000
--- a/tests/crashes/137190-1.rs
+++ /dev/null
@@ -1,10 +0,0 @@
-//@ known-bug: #137190
-//@ compile-flags: -Zmir-opt-level=2 -Zvalidate-mir
-trait A {
-    fn b(&self);
-}
-trait C: A {}
-impl C for () {}
-fn main() {
-    (&() as &dyn C as &dyn A).b();
-}
diff --git a/tests/crashes/137468.rs b/tests/crashes/137468.rs
deleted file mode 100644
index cceb0502bd21..000000000000
--- a/tests/crashes/137468.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-//@ known-bug: #137468
-//@ compile-flags: -Copt-level=0 -Zmir-enable-passes=+GVN -Zvalidate-mir
-trait Supertrait {}
-
-trait Identity {
-    type Selff;
-}
-
-trait Trait
: Supertrait<()> + Supertrait<
::Selff> {}
-
-impl
 Trait
 for () {}
-
-fn main() {
-    let x: &dyn Trait<()> = &();
-    let x: &dyn Supertrait<()> = x;
-}
diff --git a/tests/crashes/137580.rs b/tests/crashes/137580.rs
deleted file mode 100644
index 246c80ef5560..000000000000
--- a/tests/crashes/137580.rs
+++ /dev/null
@@ -1,4 +0,0 @@
-//@ known-bug: #137580
-fn main() {
-    println!("%65536$", 1);
-}
diff --git a/tests/crashes/138510.rs b/tests/crashes/138510.rs
deleted file mode 100644
index f429e8bb33b5..000000000000
--- a/tests/crashes/138510.rs
+++ /dev/null
@@ -1,7 +0,0 @@
-//@ known-bug: #138510
-fn main()
-where
-    #[repr()]
-    _: Sized,
-{
-}
diff --git a/tests/crashes/139462.rs b/tests/crashes/139462.rs
index 05bb246d7be0..d3be14b2be2a 100644
--- a/tests/crashes/139462.rs
+++ b/tests/crashes/139462.rs
@@ -1,5 +1,6 @@
 //@ known-bug: #139462
 //@ compile-flags: -Cdebuginfo=2
+//@ ignore-backends: gcc
 #![feature(unsafe_binders)]
 use std::unsafe_binder::wrap_binder;
 fn main() {
diff --git a/tests/crashes/139556.rs b/tests/crashes/139556.rs
deleted file mode 100644
index 60dc8d7c3afc..000000000000
--- a/tests/crashes/139556.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: #139556
-
-trait T {}
-
-type Alias<'a> = impl T;
-
-struct S;
-impl<'a> T for &'a S {}
-
-#[define_opaque(Alias)]
-fn with_positive(fun: impl Fn(Alias<'_>)) {
-    with_positive(|&n| ());
-}
diff --git a/tests/crashes/139659.rs b/tests/crashes/139659.rs
deleted file mode 100644
index 7fc33f7e6a7c..000000000000
--- a/tests/crashes/139659.rs
+++ /dev/null
@@ -1,29 +0,0 @@
-//@ known-bug: #139659
-//@compile-flags: -Cdebuginfo=2 -Copt-level=0 --crate-type lib
-trait Trait {
-    type Output;
-}
-
-impl O> Trait for F {
-    type Output = O;
-}
-
-struct Wrap
(P);
-struct WrapOutput(O);
-
-impl Trait for Wrap
 {
-    type Output = WrapOutput;
-}
-
-fn wrap(x: P) -> impl Trait {
-    Wrap(x)
-}
-
-fn consume(_: P) -> P::Output {
-    unimplemented!()
-}
-
-pub fn recurse() -> impl Sized {
-    consume(wrap(recurse))
-}
-pub fn main() {}
diff --git a/tests/crashes/139815.rs b/tests/crashes/139815.rs
deleted file mode 100644
index 9094acdc94b2..000000000000
--- a/tests/crashes/139815.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-//@ known-bug: #139815
-
-#![feature(generic_const_exprs)]
-fn is_123(
-    x: [u32; {
-        N + 1;
-        5
-    }],
-) -> bool {
-    match x {
-        [1, 2] => true,
-        _ => false,
-    }
-}
diff --git a/tests/crashes/140479.rs b/tests/crashes/140479.rs
deleted file mode 100644
index ed3ca887546f..000000000000
--- a/tests/crashes/140479.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-//@ known-bug: #140479
-macro_rules! a { ( $( { $ [ $b:c ] } )) => ( $(${ concat(d, $b)} ))}
-fn e() {
-    a!({})
-}
diff --git a/tests/crashes/140609.rs b/tests/crashes/140609.rs
deleted file mode 100644
index ee8a4bb30489..000000000000
--- a/tests/crashes/140609.rs
+++ /dev/null
@@ -1,13 +0,0 @@
-//@ known-bug: #140609
-#![feature(with_negative_coherence)]
-#![feature(generic_const_exprs)]
-#![crate_type = "lib"]
-trait Trait {}
-struct A;
-
-trait C {}
-
-impl Trait for E where A<{ D <= 2 }>: FnOnce(&isize) {}
-struct E;
-
-impl Trait for E where A<{ D <= 2 }>: C {}
diff --git a/tests/crashes/144293-indirect-ops-llvm.rs b/tests/crashes/144293-indirect-ops-llvm.rs
new file mode 100644
index 000000000000..490a0116d7d2
--- /dev/null
+++ b/tests/crashes/144293-indirect-ops-llvm.rs
@@ -0,0 +1,42 @@
+//@ known-bug: #144293
+// Same as recursion-etc but eggs LLVM emission into giving indirect arguments.
+#![expect(incomplete_features)]
+#![feature(explicit_tail_calls)]
+
+use std::hint::black_box;
+
+struct U64Wrapper {
+    pub x: u64,
+    pub arbitrary: String,
+}
+
+fn count(curr: U64Wrapper, top: U64Wrapper) -> U64Wrapper {
+    if black_box(curr.x) >= top.x {
+        curr
+    } else {
+        become count(
+            U64Wrapper {
+                x: curr.x + 1,
+                arbitrary: curr.arbitrary,
+            },
+            top,
+        )
+    }
+}
+
+fn main() {
+    println!(
+        "{}",
+        count(
+            U64Wrapper {
+                x: 0,
+                arbitrary: "hello!".into()
+            },
+            black_box(U64Wrapper {
+                x: 1000000,
+                arbitrary: "goodbye!".into()
+            })
+        )
+        .x
+    );
+}
diff --git a/tests/crashes/146261.rs b/tests/crashes/146261.rs
new file mode 100644
index 000000000000..f901497a769c
--- /dev/null
+++ b/tests/crashes/146261.rs
@@ -0,0 +1,13 @@
+// This is part of series of regression tests for some diagnostics ICEs encountered in the wild with
+// suggestions having overlapping parts under https://github.com/rust-lang/rust/pull/146121.
+
+//@ needs-rustc-debug-assertions
+//@ known-bug: #146261
+
+enum U {
+    B(),
+}
+
+fn main() {
+    A(U::C)
+}
diff --git a/tests/crashes/146706.rs b/tests/crashes/146706.rs
new file mode 100644
index 000000000000..358ce3e8600a
--- /dev/null
+++ b/tests/crashes/146706.rs
@@ -0,0 +1,15 @@
+// This is part of series of regression tests for some diagnostics ICEs encountered in the wild with
+// suggestions having overlapping parts under https://github.com/rust-lang/rust/pull/146121.
+
+//@ needs-rustc-debug-assertions
+//@ known-bug: #146706
+
+type Alias<'a, T> = Foo;
+
+enum Foo {
+    Bar { t: T },
+}
+
+fn main() {
+    Alias::Bar:: { t: 0 };
+}
diff --git a/tests/crashes/147973.rs b/tests/crashes/147973.rs
new file mode 100644
index 000000000000..7271c54846f1
--- /dev/null
+++ b/tests/crashes/147973.rs
@@ -0,0 +1,14 @@
+// This is part of series of regression tests for some diagnostics ICEs encountered in the wild with
+// suggestions having overlapping parts under https://github.com/rust-lang/rust/pull/146121.
+// This is one MCVE from the beta crater run regressions from issue 147973.
+
+//@ needs-rustc-debug-assertions
+//@ known-bug: #147973
+
+//@ aux-build: overlapping_spans_helper.rs
+extern crate overlapping_spans_helper;
+
+fn main() {
+    let _name = Some(1);
+    overlapping_spans_helper::do_loop!(_name);
+}
diff --git a/tests/crashes/34127.rs b/tests/crashes/34127.rs
index ea36b48ecba0..58725bff4c08 100644
--- a/tests/crashes/34127.rs
+++ b/tests/crashes/34127.rs
@@ -1,6 +1,7 @@
 //@ compile-flags: -g -Copt-level=0 -Z verify-llvm-ir
 //@ known-bug: #34127
-//@ only-x86_64
+//@ only-64bit
+//@ ignore-backends: gcc
 
 pub fn main() {
 let _a = [(); 1 << 63];
diff --git a/tests/crashes/92004.rs b/tests/crashes/92004.rs
deleted file mode 100644
index bc2ca2a7ba38..000000000000
--- a/tests/crashes/92004.rs
+++ /dev/null
@@ -1,70 +0,0 @@
-//@ known-bug: #102310
-//@ compile-flags: -Copt-level=0
-//@ edition:2021
-// ignore-tidy-linelength
-
-use std::vec::Vec;
-use std::iter::Peekable;
-
-pub fn main() {
-    let packet = decode(vec![1,0,1,0]);
-}
-
-pub fn decode(bitstream: Vec) -> Packet {
-    let mut bitstream_itr = bitstream.into_iter().peekable();
-    return match decode_packet(&mut bitstream_itr) {
-        Some(p) => p,
-        None    => panic!("expected outer packet"),
-    }
-}
-
-pub fn decode_packets>(itr: &mut Peekable) -> Vec {
-    let mut res = Vec::new();
-    loop {
-        match decode_packet(itr) {
-            Some(p) => { res.push(p); },
-            None    => break
-        }
-    }
-
-    return res;
-}
-
-pub fn decode_packet>(itr: &mut Peekable) -> Option {
-    // get version digits
-    let version = extend_number(0, itr, 3)?;
-    let type_id = extend_number(0, itr, 3)?;
-    return operator_packet(version, type_id, itr);
-}
-
-pub fn operator_packet>(version: u64, type_id: u64, itr: &mut Peekable) -> Option {
-    let p = OperatorPacket {
-        version: version,
-        type_id: type_id,
-        packets: decode_packets(&mut itr.take(0).peekable()),
-    };
-
-    return Some(Packet::Operator(p));
-}
-
-pub fn extend_number>(num: u64, itr: &mut Peekable, take: u64) -> Option {
-    let mut value = num;
-    for _ in 0..take {
-        value *= 2;
-        value += itr.next()?;
-    }
-
-    return Some(value);
-}
-
-#[derive(Debug)]
-pub enum Packet {
-    Operator(OperatorPacket),
-}
-
-#[derive(Debug)]
-pub struct OperatorPacket {
-    version: u64,
-    type_id: u64,
-    packets: Vec
-}
diff --git a/tests/crashes/92470.rs b/tests/crashes/92470.rs
deleted file mode 100644
index a3c518f5ec62..000000000000
--- a/tests/crashes/92470.rs
+++ /dev/null
@@ -1,31 +0,0 @@
-//@ known-bug: #92470
-fn main() {
-    encode(&mut EncoderImpl);
-}
-
-pub trait Encoder {
-    type W;
-
-    fn writer(&self) -> Self::W;
-}
-
-fn encode(mut encoder: E) {
-    encoder.writer();
-    encode(&mut encoder);
-}
-
-struct EncoderImpl;
-
-impl Encoder for EncoderImpl {
-    type W = ();
-
-    fn writer(&self) {}
-}
-
-impl<'a, T: Encoder> Encoder for &'a mut T {
-    type W = T::W;
-
-    fn writer(&self) -> Self::W {
-        panic!()
-    }
-}
diff --git a/tests/crashes/95134.rs b/tests/crashes/95134.rs
deleted file mode 100644
index bcd88b1076fd..000000000000
--- a/tests/crashes/95134.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//@ known-bug: #95134
-//@ compile-flags: -Copt-level=0
-
-pub fn encode_num(n: u32, mut writer: Writer) -> Result<(), Writer::Error> {
-    if n > 15 {
-        encode_num(n / 16, &mut writer)?;
-    }
-    Ok(())
-}
-
-pub trait ExampleWriter {
-    type Error;
-}
-
-impl<'a, T: ExampleWriter> ExampleWriter for &'a mut T {
-    type Error = T::Error;
-}
-
-struct EmptyWriter;
-
-impl ExampleWriter for EmptyWriter {
-    type Error = ();
-}
-
-fn main() {
-    encode_num(69, &mut EmptyWriter).unwrap();
-}
diff --git a/tests/crashes/auxiliary/overlapping_spans_helper.rs b/tests/crashes/auxiliary/overlapping_spans_helper.rs
new file mode 100644
index 000000000000..e449fcd36c37
--- /dev/null
+++ b/tests/crashes/auxiliary/overlapping_spans_helper.rs
@@ -0,0 +1,15 @@
+// Auxiliary lib for the issue 147973 regression test with ICEs due to overlapping spans.
+
+#[macro_export]
+macro_rules! identity {
+    ($x:ident) => {
+        $x
+    };
+}
+
+#[macro_export]
+macro_rules! do_loop {
+    ($x:ident) => {
+        for $crate::identity!($x) in $x {}
+    };
+}
diff --git a/tests/debuginfo/associated-types.rs b/tests/debuginfo/associated-types.rs
index b20bd5209368..e74a592662d2 100644
--- a/tests/debuginfo/associated-types.rs
+++ b/tests/debuginfo/associated-types.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 // gdb-command:run
@@ -68,8 +70,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 trait TraitWithAssocType {
     type Type;
diff --git a/tests/debuginfo/auxiliary/cross_crate_spans.rs b/tests/debuginfo/auxiliary/cross_crate_spans.rs
index af853ee0b003..d0d32c2cbe32 100644
--- a/tests/debuginfo/auxiliary/cross_crate_spans.rs
+++ b/tests/debuginfo/auxiliary/cross_crate_spans.rs
@@ -1,8 +1,6 @@
 #![crate_type = "rlib"]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 //@ no-prefer-dynamic
 //@ compile-flags:-g
diff --git a/tests/debuginfo/basic-stepping.rs b/tests/debuginfo/basic-stepping.rs
new file mode 100644
index 000000000000..aa9f29259d05
--- /dev/null
+++ b/tests/debuginfo/basic-stepping.rs
@@ -0,0 +1,50 @@
+//! Test that stepping through a simple program with a debugger one line at a
+//! time works intuitively, e.g. that `next` takes you to the next source line.
+//! Regression test for .
+
+//@ ignore-aarch64: Doesn't work yet.
+//@ ignore-loongarch64: Doesn't work yet.
+//@ ignore-riscv64: Doesn't work yet.
+//@ compile-flags: -g
+//@ ignore-backends: gcc
+
+// gdb-command: run
+// FIXME(#97083): Should we be able to break on initialization of zero-sized types?
+// FIXME(#97083): Right now the first breakable line is:
+// gdb-check:   let mut c = 27;
+// gdb-command: next
+// gdb-check:   let d = c = 99;
+// gdb-command: next
+// FIXME(#33013): gdb-check:   let e = "hi bob";
+// FIXME(#33013): gdb-command: next
+// FIXME(#33013): gdb-check:   let f = b"hi bob";
+// FIXME(#33013): gdb-command: next
+// FIXME(#33013): gdb-check:   let g = b'9';
+// FIXME(#33013): gdb-command: next
+// FIXME(#33013): gdb-check:   let h = ["whatever"; 8];
+// FIXME(#33013): gdb-command: next
+// gdb-check:   let i = [1,2,3,4];
+// gdb-command: next
+// gdb-check:   let j = (23, "hi");
+// gdb-command: next
+// gdb-check:   let k = 2..3;
+// gdb-command: next
+// gdb-check:   let l = &i[k];
+// gdb-command: next
+// gdb-check:   let m: *const() = &a;
+
+fn main () {
+    let a = (); // #break
+    let b : [i32; 0] = [];
+    let mut c = 27;
+    let d = c = 99;
+    let e = "hi bob";
+    let f = b"hi bob";
+    let g = b'9';
+    let h = ["whatever"; 8];
+    let i = [1,2,3,4];
+    let j = (23, "hi");
+    let k = 2..3;
+    let l = &i[k];
+    let m: *const() = &a;
+}
diff --git a/tests/debuginfo/basic-types-globals-metadata.rs b/tests/debuginfo/basic-types-globals-metadata.rs
index aec8ff183ad7..cc32b338da72 100644
--- a/tests/debuginfo/basic-types-globals-metadata.rs
+++ b/tests/debuginfo/basic-types-globals-metadata.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 // gdb-command:whatis basic_types_globals_metadata::B
@@ -35,8 +37,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 // N.B. These are `mut` only so they don't constant fold away.
diff --git a/tests/debuginfo/basic-types-globals.rs b/tests/debuginfo/basic-types-globals.rs
index 15a0deb64c12..d8997b3f75a9 100644
--- a/tests/debuginfo/basic-types-globals.rs
+++ b/tests/debuginfo/basic-types-globals.rs
@@ -1,9 +1,43 @@
 //@ revisions: lto no-lto
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 //@ [lto] compile-flags:-C lto
 //@ [lto] no-prefer-dynamic
+//@ ignore-backends: gcc
+
+// lldb-command:run
+// lldb-command:v B
+// lldb-check: ::B::[...] = false
+// lldb-command:v I
+// lldb-check: ::I::[...] = -1
+// lldb-command:v --format=d C
+// lldb-check: ::C::[...] = 97
+// lldb-command:v --format=d I8
+// lldb-check: ::I8::[...] = 68
+// lldb-command:v I16
+// lldb-check: ::I16::[...] = -16
+// lldb-command:v I32
+// lldb-check: ::I32::[...] = -32
+// lldb-command:v I64
+// lldb-check: ::I64::[...] = -64
+// lldb-command:v U
+// lldb-check: ::U::[...] = 1
+// lldb-command:v --format=d U8
+// lldb-check: ::U8::[...] = 100
+// lldb-command:v U16
+// lldb-check: ::U16::[...] = 16
+// lldb-command:v U32
+// lldb-check: ::U32::[...] = 32
+// lldb-command:v U64
+// lldb-check: ::U64::[...] = 64
+// lldb-command:v F16
+// lldb-check: ::F16::[...] = 1.5
+// lldb-command:v F32
+// lldb-check: ::F32::[...] = 2.5
+// lldb-command:v F64
+// lldb-check: ::F64::[...] = 3.5
 
 // gdb-command:run
 // gdb-command:print B
@@ -39,8 +73,6 @@
 // gdb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 // N.B. These are `mut` only so they don't constant fold away.
diff --git a/tests/debuginfo/basic-types-metadata.rs b/tests/debuginfo/basic-types-metadata.rs
index 6b7cfbdebca9..495ef81a16c6 100644
--- a/tests/debuginfo/basic-types-metadata.rs
+++ b/tests/debuginfo/basic-types-metadata.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 // gdb-command:whatis unit
@@ -53,8 +55,6 @@
 // gdb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 fn main() {
diff --git a/tests/debuginfo/basic-types-mut-globals.rs b/tests/debuginfo/basic-types-mut-globals.rs
index f6a2399d230a..6617f91c3df5 100644
--- a/tests/debuginfo/basic-types-mut-globals.rs
+++ b/tests/debuginfo/basic-types-mut-globals.rs
@@ -5,6 +5,8 @@
 // its numerical value.
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 
@@ -74,8 +76,6 @@
 // gdb-check:$30 = 9.25
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 static mut B: bool = false;
diff --git a/tests/debuginfo/basic-types.rs b/tests/debuginfo/basic-types.rs
index fea5262bc41d..9b1452fab413 100644
--- a/tests/debuginfo/basic-types.rs
+++ b/tests/debuginfo/basic-types.rs
@@ -5,6 +5,8 @@
 // its numerical value.
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -112,8 +114,6 @@
 // cdb-check:s                : [...] [Type: ref$]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 fn main() {
diff --git a/tests/debuginfo/borrowed-basic.rs b/tests/debuginfo/borrowed-basic.rs
index 91de691e78e1..86c4df12866e 100644
--- a/tests/debuginfo/borrowed-basic.rs
+++ b/tests/debuginfo/borrowed-basic.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -96,8 +98,6 @@
 // lldb-check:[...] 3.5
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 fn main() {
diff --git a/tests/debuginfo/borrowed-c-style-enum.rs b/tests/debuginfo/borrowed-c-style-enum.rs
index 6a91d4f96504..b331775743b4 100644
--- a/tests/debuginfo/borrowed-c-style-enum.rs
+++ b/tests/debuginfo/borrowed-c-style-enum.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -28,8 +30,6 @@
 // lldb-check:[...] TheC
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 enum ABC { TheA, TheB, TheC }
 
diff --git a/tests/debuginfo/borrowed-enum.rs b/tests/debuginfo/borrowed-enum.rs
index c5a795fdede6..48dce139a2f3 100644
--- a/tests/debuginfo/borrowed-enum.rs
+++ b/tests/debuginfo/borrowed-enum.rs
@@ -1,6 +1,8 @@
 //@ min-lldb-version: 1800
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -21,15 +23,13 @@
 // lldb-command:run
 
 // lldb-command:v *the_a_ref
-// lldb-check:(borrowed_enum::ABC) *the_a_ref = { value = { x = 0 y = 8970181431921507452 } $discr$ = 0 }
+// lldb-check:(borrowed_enum::ABC) *the_a_ref = { TheA = { x = 0 y = 8970181431921507452 } }
 // lldb-command:v *the_b_ref
-// lldb-check:(borrowed_enum::ABC) *the_b_ref = { value = { 0 = 0 1 = 286331153 2 = 286331153 } $discr$ = 1 }
+// lldb-check:(borrowed_enum::ABC) *the_b_ref = { TheB = { 0 = 0 1 = 286331153 2 = 286331153 } }
 // lldb-command:v *univariant_ref
-// lldb-check:(borrowed_enum::Univariant) *univariant_ref = { value = { 0 = 4820353753753434 } }
+// lldb-check:(borrowed_enum::Univariant) *univariant_ref = { TheOnlyCase = { 0 = 4820353753753434 } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // The first element is to ensure proper alignment, irrespective of the machines word size. Since
 // the size of the discriminant value is machine dependent, this has be taken into account when
diff --git a/tests/debuginfo/borrowed-struct.rs b/tests/debuginfo/borrowed-struct.rs
index 245af35f5055..876ec30b1401 100644
--- a/tests/debuginfo/borrowed-struct.rs
+++ b/tests/debuginfo/borrowed-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -52,8 +54,6 @@
 // lldb-check:[...] 26.5
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct SomeStruct {
     x: isize,
diff --git a/tests/debuginfo/borrowed-tuple.rs b/tests/debuginfo/borrowed-tuple.rs
index 9e4ceec033ec..42964c79485a 100644
--- a/tests/debuginfo/borrowed-tuple.rs
+++ b/tests/debuginfo/borrowed-tuple.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -29,8 +31,6 @@
 
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn main() {
     let stack_val: (i16, f32) = (-14, -19f32);
diff --git a/tests/debuginfo/borrowed-unique-basic.rs b/tests/debuginfo/borrowed-unique-basic.rs
index 7a9b4d1df825..c230ceae1c75 100644
--- a/tests/debuginfo/borrowed-unique-basic.rs
+++ b/tests/debuginfo/borrowed-unique-basic.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -100,8 +102,6 @@
 // lldb-check:[...] 3.5
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 fn main() {
diff --git a/tests/debuginfo/box.rs b/tests/debuginfo/box.rs
index d22566c0b179..c5034410f49f 100644
--- a/tests/debuginfo/box.rs
+++ b/tests/debuginfo/box.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -19,8 +21,6 @@
 // lldb-check:[...] { 0 = 2 1 = 3.5 }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn main() {
     let a = Box::new(1);
diff --git a/tests/debuginfo/boxed-struct.rs b/tests/debuginfo/boxed-struct.rs
index 158609fb2ed7..fc676e8ce618 100644
--- a/tests/debuginfo/boxed-struct.rs
+++ b/tests/debuginfo/boxed-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -22,8 +24,6 @@
 // lldb-check:[...] { x = 77 y = 777 z = 7777 w = 77777 }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct StructWithSomePadding {
     x: i16,
diff --git a/tests/debuginfo/by-value-non-immediate-argument.rs b/tests/debuginfo/by-value-non-immediate-argument.rs
index deacea5f6ccd..b5b0df73a68a 100644
--- a/tests/debuginfo/by-value-non-immediate-argument.rs
+++ b/tests/debuginfo/by-value-non-immediate-argument.rs
@@ -1,6 +1,7 @@
 //@ min-lldb-version: 1800
 //@ min-gdb-version: 13.0
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 //@ ignore-windows-gnu: #128973
 //@ ignore-aarch64-unknown-linux-gnu (gdb tries to read from 0x0; FIXME: #128973)
 //@ ignore-powerpc64: #128973 on both -gnu and -musl
@@ -62,9 +63,6 @@
 // lldb-check:[...] Case1 { x: 0, y: 8970181431921507452 }
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Clone)]
 struct Struct {
     a: isize,
diff --git a/tests/debuginfo/by-value-self-argument-in-trait-impl.rs b/tests/debuginfo/by-value-self-argument-in-trait-impl.rs
index 6981fdfc9e11..409cfc533c87 100644
--- a/tests/debuginfo/by-value-self-argument-in-trait-impl.rs
+++ b/tests/debuginfo/by-value-self-argument-in-trait-impl.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -33,9 +35,6 @@
 // lldb-check:[...] { 0 = 4444.5 1 = 5555 2 = 6666 3 = 7777.5 }
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 trait Trait {
     fn method(self) -> Self;
 }
diff --git a/tests/debuginfo/c-style-enum-in-composite.rs b/tests/debuginfo/c-style-enum-in-composite.rs
index 642879cf3b67..dd5e4f8b65d7 100644
--- a/tests/debuginfo/c-style-enum-in-composite.rs
+++ b/tests/debuginfo/c-style-enum-in-composite.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -51,8 +53,6 @@
 // lldb-check:[...] { 0 = { a = OneHundred b = Vienna } 1 = 9 }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::AnEnum::{OneHundred, OneThousand, OneMillion};
 use self::AnotherEnum::{MountainView, Toronto, Vienna};
diff --git a/tests/debuginfo/c-style-enum.rs b/tests/debuginfo/c-style-enum.rs
index 08378f7af181..79438e3f2d43 100644
--- a/tests/debuginfo/c-style-enum.rs
+++ b/tests/debuginfo/c-style-enum.rs
@@ -1,6 +1,8 @@
 //@ ignore-aarch64
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -88,8 +90,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::AutoDiscriminant::{One, Two, Three};
 use self::ManualDiscriminant::{OneHundred, OneThousand, OneMillion};
diff --git a/tests/debuginfo/captured-fields-1.rs b/tests/debuginfo/captured-fields-1.rs
index 69ca3ecd812d..53f77d314685 100644
--- a/tests/debuginfo/captured-fields-1.rs
+++ b/tests/debuginfo/captured-fields-1.rs
@@ -1,5 +1,6 @@
 //@ compile-flags:-g
 //@ edition:2021
+//@ ignore-backends: gcc
 // === GDB TESTS ===================================================================================
 
 // gdb-command:run
diff --git a/tests/debuginfo/captured-fields-2.rs b/tests/debuginfo/captured-fields-2.rs
index 24bff1d3f35d..cdebeaf3a975 100644
--- a/tests/debuginfo/captured-fields-2.rs
+++ b/tests/debuginfo/captured-fields-2.rs
@@ -1,5 +1,6 @@
 //@ compile-flags:-g
 //@ edition:2021
+//@ ignore-backends: gcc
 // === GDB TESTS ===================================================================================
 
 // gdb-command:run
diff --git a/tests/debuginfo/closure-in-generic-function.rs b/tests/debuginfo/closure-in-generic-function.rs
index 0c6a6fdfca1b..88769399f089 100644
--- a/tests/debuginfo/closure-in-generic-function.rs
+++ b/tests/debuginfo/closure-in-generic-function.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -33,9 +35,6 @@
 // lldb-check:[...] 110
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn some_generic_fun(a: T1, b: T2) -> (T2, T1) {
 
     let closure = |x, y| {
diff --git a/tests/debuginfo/collapse-debuginfo-in-non-collapse-macro.rs b/tests/debuginfo/collapse-debuginfo-in-non-collapse-macro.rs
index 1aafcffa3047..bafa5408b197 100644
--- a/tests/debuginfo/collapse-debuginfo-in-non-collapse-macro.rs
+++ b/tests/debuginfo/collapse-debuginfo-in-non-collapse-macro.rs
@@ -6,6 +6,7 @@
 // without collapse_debuginfo attribute.
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/collapse-debuginfo-static-external.rs b/tests/debuginfo/collapse-debuginfo-static-external.rs
index 2209bb9bd948..6389bb734ddf 100644
--- a/tests/debuginfo/collapse-debuginfo-static-external.rs
+++ b/tests/debuginfo/collapse-debuginfo-static-external.rs
@@ -3,11 +3,12 @@
 // Test that static debug info is not collapsed with #[collapse_debuginfo(external)]
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
 // gdb-command:info line collapse_debuginfo_static_external::FOO
-// gdb-check:[...]Line 15[...]
+// gdb-check:[...]Line 16[...]
 
 #[collapse_debuginfo(external)]
 macro_rules! decl_foo {
diff --git a/tests/debuginfo/collapse-debuginfo-static.rs b/tests/debuginfo/collapse-debuginfo-static.rs
index e6469da4785e..a5542a44091a 100644
--- a/tests/debuginfo/collapse-debuginfo-static.rs
+++ b/tests/debuginfo/collapse-debuginfo-static.rs
@@ -3,11 +3,12 @@
 // Test that static debug info is collapsed with #[collapse_debuginfo(yes)]
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
 // gdb-command:info line collapse_debuginfo_static::FOO
-// gdb-check:[...]Line 19[...]
+// gdb-check:[...]Line 20[...]
 
 #[collapse_debuginfo(yes)]
 macro_rules! decl_foo {
diff --git a/tests/debuginfo/constant-debug-locs.rs b/tests/debuginfo/constant-debug-locs.rs
index 81115fc3c384..d13b8648b18a 100644
--- a/tests/debuginfo/constant-debug-locs.rs
+++ b/tests/debuginfo/constant-debug-locs.rs
@@ -1,8 +1,7 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 #![allow(dead_code, unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // This test makes sure that the compiler doesn't crash when trying to assign
 // debug locations to const-expressions.
diff --git a/tests/debuginfo/constant-in-match-pattern.rs b/tests/debuginfo/constant-in-match-pattern.rs
index 952db216debf..922e0a5d8da2 100644
--- a/tests/debuginfo/constant-in-match-pattern.rs
+++ b/tests/debuginfo/constant-in-match-pattern.rs
@@ -1,8 +1,7 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 #![allow(dead_code, unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // This test makes sure that the compiler doesn't crash when trying to assign
 // debug locations to 'constant' patterns in match expressions.
diff --git a/tests/debuginfo/constant-ordering-prologue.rs b/tests/debuginfo/constant-ordering-prologue.rs
index 3136aff238a2..1d4674b1d32b 100644
--- a/tests/debuginfo/constant-ordering-prologue.rs
+++ b/tests/debuginfo/constant-ordering-prologue.rs
@@ -1,6 +1,7 @@
 //@ min-lldb-version: 310
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/coroutine-locals.rs b/tests/debuginfo/coroutine-locals.rs
index f3593adc9453..d63ea4bde367 100644
--- a/tests/debuginfo/coroutine-locals.rs
+++ b/tests/debuginfo/coroutine-locals.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -44,8 +46,7 @@
 // lldb-command:v c
 // lldb-check:(int) c = 6
 
-#![feature(omit_gdb_pretty_printer_section, coroutines, coroutine_trait, stmt_expr_attributes)]
-#![omit_gdb_pretty_printer_section]
+#![feature(coroutines, coroutine_trait, stmt_expr_attributes)]
 
 use std::ops::Coroutine;
 use std::pin::Pin;
diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs
index 242c76c2989e..f1165de4bfaa 100644
--- a/tests/debuginfo/coroutine-objects.rs
+++ b/tests/debuginfo/coroutine-objects.rs
@@ -5,6 +5,8 @@
 // ensure that LLDB won't crash at least (like #57822).
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -53,8 +55,7 @@
 // cdb-check: b                : Returned [Type: enum2$]
 // cdb-check:    [+0x[...]] _ref__a          : 0x[...] : 6 [Type: int *]
 
-#![feature(omit_gdb_pretty_printer_section, coroutines, coroutine_trait, stmt_expr_attributes)]
-#![omit_gdb_pretty_printer_section]
+#![feature(coroutines, coroutine_trait, stmt_expr_attributes)]
 
 use std::ops::Coroutine;
 use std::pin::Pin;
diff --git a/tests/debuginfo/cross-crate-spans.rs b/tests/debuginfo/cross-crate-spans.rs
index e337aaf5a6c9..6ef17061070e 100644
--- a/tests/debuginfo/cross-crate-spans.rs
+++ b/tests/debuginfo/cross-crate-spans.rs
@@ -1,15 +1,14 @@
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 //@ aux-build:cross_crate_spans.rs
 extern crate cross_crate_spans;
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 
 // === GDB TESTS ===================================================================================
 
-// gdb-command:break cross_crate_spans.rs:14
+// gdb-command:break cross_crate_spans.rs:12
 // gdb-command:run
 
 // gdb-command:print result
@@ -32,7 +31,7 @@ extern crate cross_crate_spans;
 
 // === LLDB TESTS ==================================================================================
 
-// lldb-command:b cross_crate_spans.rs:14
+// lldb-command:b cross_crate_spans.rs:12
 // lldb-command:run
 
 // lldb-command:v result
diff --git a/tests/debuginfo/cross-crate-type-uniquing.rs b/tests/debuginfo/cross-crate-type-uniquing.rs
index 28ebc3438846..c29757ae7793 100644
--- a/tests/debuginfo/cross-crate-type-uniquing.rs
+++ b/tests/debuginfo/cross-crate-type-uniquing.rs
@@ -3,6 +3,7 @@ extern crate cross_crate_debuginfo_type_uniquing;
 
 //@ no-prefer-dynamic
 //@ compile-flags:-g -C lto
+//@ ignore-backends: gcc
 
 pub struct C;
 pub fn p() -> C {
diff --git a/tests/debuginfo/destructured-fn-argument.rs b/tests/debuginfo/destructured-fn-argument.rs
index 37a7bb2b7786..27910ed46882 100644
--- a/tests/debuginfo/destructured-fn-argument.rs
+++ b/tests/debuginfo/destructured-fn-argument.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -299,8 +301,6 @@
 
 #![allow(unused_variables)]
 #![feature(box_patterns)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Univariant::Unit;
 
diff --git a/tests/debuginfo/destructured-for-loop-variable.rs b/tests/debuginfo/destructured-for-loop-variable.rs
index cc16be1268ae..12a9ff417258 100644
--- a/tests/debuginfo/destructured-for-loop-variable.rs
+++ b/tests/debuginfo/destructured-for-loop-variable.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -141,8 +143,6 @@
 
 #![allow(unused_variables)]
 #![feature(box_patterns)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct {
     x: i16,
diff --git a/tests/debuginfo/destructured-local.rs b/tests/debuginfo/destructured-local.rs
index fad96ca7d4b6..ef595468b473 100644
--- a/tests/debuginfo/destructured-local.rs
+++ b/tests/debuginfo/destructured-local.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -232,8 +234,6 @@
 
 #![allow(unused_variables)]
 #![feature(box_patterns)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Univariant::Unit;
 
diff --git a/tests/debuginfo/dummy_span.rs b/tests/debuginfo/dummy_span.rs
index d02eead470fa..23822222006c 100644
--- a/tests/debuginfo/dummy_span.rs
+++ b/tests/debuginfo/dummy_span.rs
@@ -1,6 +1,7 @@
 //@ min-lldb-version: 310
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/embedded-visualizer.rs b/tests/debuginfo/embedded-visualizer.rs
index cbd8691394d5..3652036ae8b6 100644
--- a/tests/debuginfo/embedded-visualizer.rs
+++ b/tests/debuginfo/embedded-visualizer.rs
@@ -63,6 +63,7 @@
 #![debugger_visualizer(gdb_script_file = "embedded-visualizer.py")]
 
 //@ aux-build: dependency-with-embedded-visualizers.rs
+//@ ignore-backends: gcc
 extern crate dependency_with_embedded_visualizers;
 
 use dependency_with_embedded_visualizers::Person;
diff --git a/tests/debuginfo/empty-string.rs b/tests/debuginfo/empty-string.rs
index 6cf61e177714..d9368001b631 100644
--- a/tests/debuginfo/empty-string.rs
+++ b/tests/debuginfo/empty-string.rs
@@ -1,6 +1,7 @@
 //@ ignore-windows-gnu: #128981
 //@ ignore-android: FIXME(#10381)
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/enum-thinlto.rs b/tests/debuginfo/enum-thinlto.rs
index af77145c312d..789755438a6b 100644
--- a/tests/debuginfo/enum-thinlto.rs
+++ b/tests/debuginfo/enum-thinlto.rs
@@ -1,5 +1,7 @@
 //@ min-lldb-version: 1800
 //@ compile-flags:-g -Z thinlto
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -16,8 +18,6 @@
 // lldb-check:(enum_thinlto::ABC) *abc = { value = { x = 0 y = 8970181431921507452 } $discr$ = 0 }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // The first element is to ensure proper alignment, irrespective of the machines word size. Since
 // the size of the discriminant value is machine dependent, this has be taken into account when
diff --git a/tests/debuginfo/evec-in-struct.rs b/tests/debuginfo/evec-in-struct.rs
index 303669cf06cd..1c0c0a36bace 100644
--- a/tests/debuginfo/evec-in-struct.rs
+++ b/tests/debuginfo/evec-in-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -38,8 +40,6 @@
 // lldb-check:[...] { x = { [0] = 22 [1] = 23 } y = { [0] = 24 [1] = 25 } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct NoPadding1 {
     x: [u32; 3],
diff --git a/tests/debuginfo/extern-c-fn.rs b/tests/debuginfo/extern-c-fn.rs
index 4642073faabc..3a8a110afa2c 100644
--- a/tests/debuginfo/extern-c-fn.rs
+++ b/tests/debuginfo/extern-c-fn.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 // gdb-command:run
@@ -32,9 +34,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 
 #[no_mangle]
 pub unsafe extern "C" fn fn_with_c_abi(s: *const u8, len: i32) -> i32 {
diff --git a/tests/debuginfo/function-arg-initialization.rs b/tests/debuginfo/function-arg-initialization.rs
index ae54d56623c6..1a681c772116 100644
--- a/tests/debuginfo/function-arg-initialization.rs
+++ b/tests/debuginfo/function-arg-initialization.rs
@@ -8,6 +8,8 @@
 //@ min-lldb-version: 1800
 //@ compile-flags:-g -Zmir-enable-passes=-SingleUseConsts
 // SingleUseConsts shouldn't need to be disabled, see #128945
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -214,8 +216,6 @@
 
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn immediate_args(a: isize, b: bool, c: f64) {
     zzz(); // #break
diff --git a/tests/debuginfo/function-arguments.rs b/tests/debuginfo/function-arguments.rs
index 21c0c7d859cc..47c6ebe11554 100644
--- a/tests/debuginfo/function-arguments.rs
+++ b/tests/debuginfo/function-arguments.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -32,10 +34,6 @@
 // lldb-check:[...] 3000
 // lldb-command:continue
 
-
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     fun(111102, true);
diff --git a/tests/debuginfo/function-call.rs b/tests/debuginfo/function-call.rs
index 3d8d798928d7..c0b23dc551ea 100644
--- a/tests/debuginfo/function-call.rs
+++ b/tests/debuginfo/function-call.rs
@@ -2,6 +2,7 @@
 //@ min-gdb-version: 10.1
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/function-names.rs b/tests/debuginfo/function-names.rs
index c51884451e56..75fc14129761 100644
--- a/tests/debuginfo/function-names.rs
+++ b/tests/debuginfo/function-names.rs
@@ -2,6 +2,8 @@
 //@ min-gdb-version: 10.1
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -78,8 +80,6 @@
 // cdb-check:[...] a!function_names::const_generic_fn_bool (void)
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(adt_const_params, coroutines, coroutine_trait, stmt_expr_attributes)]
 #![allow(incomplete_features)]
 
diff --git a/tests/debuginfo/function-prologue-stepping-regular.rs b/tests/debuginfo/function-prologue-stepping-regular.rs
index 07b9356fb507..f61128ca2df1 100644
--- a/tests/debuginfo/function-prologue-stepping-regular.rs
+++ b/tests/debuginfo/function-prologue-stepping-regular.rs
@@ -4,6 +4,7 @@
 //@ min-lldb-version: 1800
 //@ ignore-gdb
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // lldb-command:breakpoint set --name immediate_args
 // lldb-command:breakpoint set --name non_immediate_args
@@ -116,8 +117,6 @@
 // lldb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn immediate_args(a: isize, b: bool, c: f64) {
     ()
diff --git a/tests/debuginfo/gdb-char.rs b/tests/debuginfo/gdb-char.rs
index 7d8608d4f511..5f44fa5c4233 100644
--- a/tests/debuginfo/gdb-char.rs
+++ b/tests/debuginfo/gdb-char.rs
@@ -3,6 +3,8 @@
 
 //@ min-gdb-version: 11.2
 //@ compile-flags: -g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -11,8 +13,6 @@
 // gdb-check:$1 = 97 'a'
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn main() {
     let ch: char = 'a';
diff --git a/tests/debuginfo/gdb-pretty-struct-and-enums.rs b/tests/debuginfo/gdb-pretty-struct-and-enums.rs
index 08e01333a376..ab6aea6b6316 100644
--- a/tests/debuginfo/gdb-pretty-struct-and-enums.rs
+++ b/tests/debuginfo/gdb-pretty-struct-and-enums.rs
@@ -2,6 +2,7 @@
 //@ ignore-android: FIXME(#10381)
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // gdb-command: run
 
diff --git a/tests/debuginfo/generic-enum-with-different-disr-sizes.rs b/tests/debuginfo/generic-enum-with-different-disr-sizes.rs
index e723543a37b2..07a92619e211 100644
--- a/tests/debuginfo/generic-enum-with-different-disr-sizes.rs
+++ b/tests/debuginfo/generic-enum-with-different-disr-sizes.rs
@@ -1,6 +1,8 @@
 //@ ignore-lldb: FIXME(#27089)
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 // gdb-command:run
@@ -57,8 +59,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // This test case makes sure that we get correct type descriptions for the enum
 // discriminant of different instantiations of the same generic enum type where,
diff --git a/tests/debuginfo/generic-function.rs b/tests/debuginfo/generic-function.rs
index 4be8d5ad45aa..9ca9eb32a2e1 100644
--- a/tests/debuginfo/generic-function.rs
+++ b/tests/debuginfo/generic-function.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -44,9 +46,6 @@
 // lldb-check:[...] { a = 6 b = 7.5 }
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Clone)]
 struct Struct {
     a: isize,
diff --git a/tests/debuginfo/generic-functions-nested.rs b/tests/debuginfo/generic-functions-nested.rs
index 7e0c20f89038..1a87827c7759 100644
--- a/tests/debuginfo/generic-functions-nested.rs
+++ b/tests/debuginfo/generic-functions-nested.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -57,10 +59,6 @@
 // lldb-check:[...] 2.5
 // lldb-command:continue
 
-
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn outer(a: TA) {
     inner(a.clone(), 1);
     inner(a.clone(), 2.5f64);
diff --git a/tests/debuginfo/generic-method-on-generic-struct.rs b/tests/debuginfo/generic-method-on-generic-struct.rs
index 9c587ca2839b..c549a1415365 100644
--- a/tests/debuginfo/generic-method-on-generic-struct.rs
+++ b/tests/debuginfo/generic-method-on-generic-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10.5
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: T
diff --git a/tests/debuginfo/generic-static-method-on-struct-and-enum.rs b/tests/debuginfo/generic-static-method-on-struct-and-enum.rs
index 79fe2144cf4b..be9f394f133d 100644
--- a/tests/debuginfo/generic-static-method-on-struct-and-enum.rs
+++ b/tests/debuginfo/generic-static-method-on-struct-and-enum.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 
@@ -19,9 +21,6 @@
 // gdb-command:continue
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct Struct {
     x: isize
 }
diff --git a/tests/debuginfo/generic-struct-style-enum.rs b/tests/debuginfo/generic-struct-style-enum.rs
index a5529ab8027d..7b7a36effd57 100644
--- a/tests/debuginfo/generic-struct-style-enum.rs
+++ b/tests/debuginfo/generic-struct-style-enum.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:set print union on
 // gdb-command:run
@@ -16,9 +18,6 @@
 // gdb-check:$4 = generic_struct_style_enum::Univariant::TheOnlyCase{a: -1}
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 use self::Regular::{Case1, Case2, Case3};
 use self::Univariant::TheOnlyCase;
 
diff --git a/tests/debuginfo/generic-struct.rs b/tests/debuginfo/generic-struct.rs
index f26d823d4f2e..daf6554fab0a 100644
--- a/tests/debuginfo/generic-struct.rs
+++ b/tests/debuginfo/generic-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -49,9 +51,6 @@
 // cdb-check:[...]value            [Type: generic_struct::AGenericStruct]
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct AGenericStruct {
     key: TKey,
     value: TValue
diff --git a/tests/debuginfo/generic-tuple-style-enum.rs b/tests/debuginfo/generic-tuple-style-enum.rs
index 4a5996645cb4..de8681bc5361 100644
--- a/tests/debuginfo/generic-tuple-style-enum.rs
+++ b/tests/debuginfo/generic-tuple-style-enum.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -30,8 +32,6 @@
 
 // lldb-command:v univariant
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Regular::{Case1, Case2, Case3};
 use self::Univariant::TheOnlyCase;
diff --git a/tests/debuginfo/include_string.rs b/tests/debuginfo/include_string.rs
index 704b85e1ac2d..565bd09f89e9 100644
--- a/tests/debuginfo/include_string.rs
+++ b/tests/debuginfo/include_string.rs
@@ -2,6 +2,8 @@
 // ^ test temporarily disabled as it fails under gdb 15
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 // gdb-command:run
 // gdb-command:print string1.length
 // gdb-check:$1 = 48
@@ -26,8 +28,6 @@
 // lldb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // This test case makes sure that debug info does not ICE when include_str is
 // used multiple times (see issue #11322).
diff --git a/tests/debuginfo/issue-12886.rs b/tests/debuginfo/issue-12886.rs
index 48250e885374..5574294cd57c 100644
--- a/tests/debuginfo/issue-12886.rs
+++ b/tests/debuginfo/issue-12886.rs
@@ -2,14 +2,13 @@
 //@ ignore-aarch64
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // gdb-command:run
 // gdb-command:next
-// gdb-check:[...]23[...]let s = Some(5).unwrap(); // #break
+// gdb-check:[...]22[...]let s = Some(5).unwrap(); // #break
 // gdb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // IF YOU MODIFY THIS FILE, BE CAREFUL TO ADAPT THE LINE NUMBERS IN THE DEBUGGER COMMANDS
 
diff --git a/tests/debuginfo/issue-22656.rs b/tests/debuginfo/issue-22656.rs
index eb0b38cfa4d6..3407c0524ebb 100644
--- a/tests/debuginfo/issue-22656.rs
+++ b/tests/debuginfo/issue-22656.rs
@@ -5,6 +5,7 @@
 //@ ignore-gdb
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // === LLDB TESTS ==================================================================================
 // lldb-command:run
@@ -16,8 +17,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct ZeroSizedStruct;
 
diff --git a/tests/debuginfo/issue-57822.rs b/tests/debuginfo/issue-57822.rs
index 7abac1c14d32..59c0759737dd 100644
--- a/tests/debuginfo/issue-57822.rs
+++ b/tests/debuginfo/issue-57822.rs
@@ -3,6 +3,8 @@
 
 //@ min-lldb-version: 1800
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -24,8 +26,7 @@
 // lldb-command:v b
 // lldb-check:(issue_57822::main::{coroutine_env#3}) b = { value = { a = { value = { y = 2 } $discr$ = '\x02' } } $discr$ = '\x02' }
 
-#![feature(omit_gdb_pretty_printer_section, coroutines, coroutine_trait, stmt_expr_attributes)]
-#![omit_gdb_pretty_printer_section]
+#![feature(coroutines, coroutine_trait, stmt_expr_attributes)]
 
 use std::ops::Coroutine;
 use std::pin::Pin;
diff --git a/tests/debuginfo/lexical-scope-in-for-loop.rs b/tests/debuginfo/lexical-scope-in-for-loop.rs
index 08f244f89a02..c2e97ad469ef 100644
--- a/tests/debuginfo/lexical-scope-in-for-loop.rs
+++ b/tests/debuginfo/lexical-scope-in-for-loop.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -73,9 +75,6 @@
 // lldb-check:[...] 1000000
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let range = [1, 2, 3];
diff --git a/tests/debuginfo/lexical-scope-in-if-let.rs b/tests/debuginfo/lexical-scope-in-if-let.rs
index b2c7790eab23..3bc0c231b4d3 100644
--- a/tests/debuginfo/lexical-scope-in-if-let.rs
+++ b/tests/debuginfo/lexical-scope-in-if-let.rs
@@ -1,4 +1,5 @@
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ==================================================================================
 
diff --git a/tests/debuginfo/lexical-scope-in-if.rs b/tests/debuginfo/lexical-scope-in-if.rs
index c0e1f2f3e05c..622188269307 100644
--- a/tests/debuginfo/lexical-scope-in-if.rs
+++ b/tests/debuginfo/lexical-scope-in-if.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -121,9 +123,6 @@
 // lldb-check:[...] -1
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let x = 999;
diff --git a/tests/debuginfo/lexical-scope-in-match.rs b/tests/debuginfo/lexical-scope-in-match.rs
index 9169c19c6a33..329d959bc848 100644
--- a/tests/debuginfo/lexical-scope-in-match.rs
+++ b/tests/debuginfo/lexical-scope-in-match.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -113,9 +115,6 @@
 // lldb-check:[...] 232
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct Struct {
     x: isize,
     y: isize
diff --git a/tests/debuginfo/lexical-scope-in-stack-closure.rs b/tests/debuginfo/lexical-scope-in-stack-closure.rs
index d01162c39d69..b6abf507f26b 100644
--- a/tests/debuginfo/lexical-scope-in-stack-closure.rs
+++ b/tests/debuginfo/lexical-scope-in-stack-closure.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -57,9 +59,6 @@
 // lldb-check:[...] false
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let x = false;
diff --git a/tests/debuginfo/lexical-scope-in-unconditional-loop.rs b/tests/debuginfo/lexical-scope-in-unconditional-loop.rs
index dfec570218f3..270fbcf6ebf5 100644
--- a/tests/debuginfo/lexical-scope-in-unconditional-loop.rs
+++ b/tests/debuginfo/lexical-scope-in-unconditional-loop.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -119,9 +121,6 @@
 // lldb-check:[...] 2
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let mut x = 0;
diff --git a/tests/debuginfo/lexical-scope-in-unique-closure.rs b/tests/debuginfo/lexical-scope-in-unique-closure.rs
index db84005121af..b4909a97bea5 100644
--- a/tests/debuginfo/lexical-scope-in-unique-closure.rs
+++ b/tests/debuginfo/lexical-scope-in-unique-closure.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -58,9 +60,6 @@
 // lldb-command:continue
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let x = false;
diff --git a/tests/debuginfo/lexical-scope-in-while.rs b/tests/debuginfo/lexical-scope-in-while.rs
index d6536d77545b..e8e66d2835cd 100644
--- a/tests/debuginfo/lexical-scope-in-while.rs
+++ b/tests/debuginfo/lexical-scope-in-while.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -119,9 +121,6 @@
 // lldb-check:[...] 2
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
 
     let mut x = 0;
diff --git a/tests/debuginfo/lexical-scope-with-macro.rs b/tests/debuginfo/lexical-scope-with-macro.rs
index 6e8fef201ead..8c302a94ea2a 100644
--- a/tests/debuginfo/lexical-scope-with-macro.rs
+++ b/tests/debuginfo/lexical-scope-with-macro.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -97,9 +99,6 @@
 // lldb-check:[...] 400
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 macro_rules! trivial {
     ($e1:expr) => ($e1)
 }
diff --git a/tests/debuginfo/lexical-scopes-in-block-expression.rs b/tests/debuginfo/lexical-scopes-in-block-expression.rs
index cd27c88db58e..9a7fba30397e 100644
--- a/tests/debuginfo/lexical-scopes-in-block-expression.rs
+++ b/tests/debuginfo/lexical-scopes-in-block-expression.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -336,8 +338,6 @@
 
 #![allow(unused_variables)]
 #![allow(unused_assignments)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 static mut MUT_INT: isize = 0;
 
diff --git a/tests/debuginfo/limited-debuginfo.rs b/tests/debuginfo/limited-debuginfo.rs
index fb453d8078ce..136953641a4d 100644
--- a/tests/debuginfo/limited-debuginfo.rs
+++ b/tests/debuginfo/limited-debuginfo.rs
@@ -1,6 +1,8 @@
 //@ ignore-lldb
 
 //@ compile-flags:-C debuginfo=1
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // Make sure functions have proper names
 // gdb-command:info functions
@@ -18,8 +20,6 @@
 
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct {
     a: i64,
diff --git a/tests/debuginfo/macro-stepping.rs b/tests/debuginfo/macro-stepping.rs
index 35bb6de4fef3..e58975764e53 100644
--- a/tests/debuginfo/macro-stepping.rs
+++ b/tests/debuginfo/macro-stepping.rs
@@ -1,5 +1,4 @@
 //@ ignore-android
-//@ ignore-aarch64
 //@ min-lldb-version: 1800
 //@ min-gdb-version: 13.0
 
diff --git a/tests/debuginfo/method-on-enum.rs b/tests/debuginfo/method-on-enum.rs
index 754b4a2dc26c..f86cf8ccfdf7 100644
--- a/tests/debuginfo/method-on-enum.rs
+++ b/tests/debuginfo/method-on-enum.rs
@@ -2,6 +2,7 @@
 //@ min-gdb-version: 13.0
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 //@ ignore-windows-gnu: #128973
 
@@ -104,9 +105,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 enum Enum {
     Variant1 { x: u16, y: u16 },
diff --git a/tests/debuginfo/method-on-generic-struct.rs b/tests/debuginfo/method-on-generic-struct.rs
index 1e6c9d66178a..b9627f27b912 100644
--- a/tests/debuginfo/method-on-generic-struct.rs
+++ b/tests/debuginfo/method-on-generic-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: T
diff --git a/tests/debuginfo/method-on-struct.rs b/tests/debuginfo/method-on-struct.rs
index 91f609365e9a..4ccb55d3704b 100644
--- a/tests/debuginfo/method-on-struct.rs
+++ b/tests/debuginfo/method-on-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: isize
diff --git a/tests/debuginfo/method-on-trait.rs b/tests/debuginfo/method-on-trait.rs
index 7b95e1f81c70..baf1a8567bbf 100644
--- a/tests/debuginfo/method-on-trait.rs
+++ b/tests/debuginfo/method-on-trait.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: isize
diff --git a/tests/debuginfo/method-on-tuple-struct.rs b/tests/debuginfo/method-on-tuple-struct.rs
index 04c00d883025..077c0bc13d90 100644
--- a/tests/debuginfo/method-on-tuple-struct.rs
+++ b/tests/debuginfo/method-on-tuple-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct TupleStruct(isize, f64);
 
diff --git a/tests/debuginfo/multi-cgu.rs b/tests/debuginfo/multi-cgu.rs
index 3bb5269adead..e3fa68d77659 100644
--- a/tests/debuginfo/multi-cgu.rs
+++ b/tests/debuginfo/multi-cgu.rs
@@ -2,6 +2,8 @@
 // compiled with multiple codegen units. (see #39160)
 
 //@ compile-flags:-g -Ccodegen-units=2
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===============================================================
 
@@ -29,9 +31,6 @@
 // lldb-command:continue
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 mod a {
     pub fn foo(xxx: u32) {
         super::_zzz(); // #break
diff --git a/tests/debuginfo/multiple-functions-equal-var-names.rs b/tests/debuginfo/multiple-functions-equal-var-names.rs
index 6ae9225d55c3..9df1fe1b9ca1 100644
--- a/tests/debuginfo/multiple-functions-equal-var-names.rs
+++ b/tests/debuginfo/multiple-functions-equal-var-names.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -32,8 +34,6 @@
 // lldb-check:[...] 30303
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn function_one() {
     let abc = 10101;
diff --git a/tests/debuginfo/multiple-functions.rs b/tests/debuginfo/multiple-functions.rs
index 3f7a0ded91b0..a5fe01f9dd2c 100644
--- a/tests/debuginfo/multiple-functions.rs
+++ b/tests/debuginfo/multiple-functions.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -32,8 +34,6 @@
 // lldb-check:[...] 30303
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn function_one() {
     let a = 10101;
diff --git a/tests/debuginfo/name-shadowing-and-scope-nesting.rs b/tests/debuginfo/name-shadowing-and-scope-nesting.rs
index d3829b60713d..9a3393a71e83 100644
--- a/tests/debuginfo/name-shadowing-and-scope-nesting.rs
+++ b/tests/debuginfo/name-shadowing-and-scope-nesting.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -81,9 +83,6 @@
 // lldb-check:[...] 20
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
     let x = false;
     let y = true;
diff --git a/tests/debuginfo/no_mangle-info.rs b/tests/debuginfo/no_mangle-info.rs
index 06c65a71b2e9..d1daef669cee 100644
--- a/tests/debuginfo/no_mangle-info.rs
+++ b/tests/debuginfo/no_mangle-info.rs
@@ -1,5 +1,6 @@
 //@ compile-flags:-g
 //@ min-gdb-version: 10.1
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 // gdb-command:run
diff --git a/tests/debuginfo/numeric-types.rs b/tests/debuginfo/numeric-types.rs
index 9a0fd01d7e23..9f7ef5c537d2 100644
--- a/tests/debuginfo/numeric-types.rs
+++ b/tests/debuginfo/numeric-types.rs
@@ -9,6 +9,7 @@
 // unblock the tree.
 //@ only-64bit
 //@ min-cdb-version: 10.0.26100.2161
+//@ ignore-backends: gcc
 
 // Tests the visualizations for `NonZero`, `Wrapping` and
 // `Atomic{Bool,I8,I16,I32,I64,Isize,U8,U16,U32,U64,Usize}` located in `libcore.natvis`.
diff --git a/tests/debuginfo/opt/dead_refs.rs b/tests/debuginfo/opt/dead_refs.rs
new file mode 100644
index 000000000000..61e741573346
--- /dev/null
+++ b/tests/debuginfo/opt/dead_refs.rs
@@ -0,0 +1,50 @@
+//@ min-lldb-version: 1800
+//@ min-gdb-version: 13.0
+//@ compile-flags: -g -Copt-level=3
+//@ disable-gdb-pretty-printers
+
+// Checks that we still can access dead variables from debuginfos.
+
+// === GDB TESTS ===================================================================================
+
+// gdb-command:run
+// gdb-command:print *ref_v0
+// gdb-check:$1 = 0
+
+// gdb-command:print *ref_v1
+// gdb-check:$2 = 1
+
+// gdb-command:print *ref_v2
+// gdb-check:$3 = 2
+
+// === LLDB TESTS ==================================================================================
+
+// lldb-command:run
+// lldb-command:v *ref_v0
+// lldb-check:[...] 0
+
+// lldb-command:v *ref_v1
+// lldb-check:[...] 1
+
+// lldb-command:v *ref_v2
+// lldb-check:[...] 2
+
+#![allow(unused_variables)]
+
+use std::hint::black_box;
+
+pub struct Foo(i32, i64, i32);
+
+#[inline(never)]
+#[no_mangle]
+fn test_ref(ref_foo: &Foo) -> i32 {
+    let ref_v0 = &ref_foo.0;
+    let ref_v1 = &ref_foo.1;
+    let ref_v2 = &ref_foo.2;
+    ref_foo.0 // #break
+}
+
+fn main() {
+    let foo = black_box(Foo(0, 1, 2));
+    black_box(test_ref(&foo));
+}
diff --git a/tests/debuginfo/option-like-enum.rs b/tests/debuginfo/option-like-enum.rs
index 72a41986dcef..5a55b143fbbd 100644
--- a/tests/debuginfo/option-like-enum.rs
+++ b/tests/debuginfo/option-like-enum.rs
@@ -2,6 +2,7 @@
 //@ min-gdb-version: 13.0
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // === GDB TESTS ===================================================================================
 
@@ -69,9 +70,6 @@
 // lldb-check:[...] Nope
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 // If a struct has exactly two variants, one of them is empty, and the other one
 // contains a non-nullable pointer, then this value is used as the discriminator.
 // The test cases in this file make sure that something readable is generated for
diff --git a/tests/debuginfo/packed-struct-with-destructor.rs b/tests/debuginfo/packed-struct-with-destructor.rs
index f923d36953c7..2150ec53501c 100644
--- a/tests/debuginfo/packed-struct-with-destructor.rs
+++ b/tests/debuginfo/packed-struct-with-destructor.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -60,8 +62,6 @@
 
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 #[repr(packed)]
 struct Packed {
diff --git a/tests/debuginfo/packed-struct.rs b/tests/debuginfo/packed-struct.rs
index 2b3652fe8611..427289c84774 100644
--- a/tests/debuginfo/packed-struct.rs
+++ b/tests/debuginfo/packed-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -46,8 +48,6 @@
 // lldb-check:[...] 40
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 #[repr(packed)]
 struct Packed {
diff --git a/tests/debuginfo/pretty-huge-vec.rs b/tests/debuginfo/pretty-huge-vec.rs
index 6938158e365e..c73491bf2806 100644
--- a/tests/debuginfo/pretty-huge-vec.rs
+++ b/tests/debuginfo/pretty-huge-vec.rs
@@ -2,6 +2,7 @@
 //@ ignore-android: FIXME(#10381)
 //@ ignore-aix: FIXME(#137965)
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/pretty-slices.rs b/tests/debuginfo/pretty-slices.rs
index f1aad836434e..fd347a4eb933 100644
--- a/tests/debuginfo/pretty-slices.rs
+++ b/tests/debuginfo/pretty-slices.rs
@@ -1,6 +1,7 @@
 //@ ignore-android: FIXME(#10381)
 //@ ignore-windows-gnu: #128981
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // gdb-command: run
 
diff --git a/tests/debuginfo/pretty-std-collections.rs b/tests/debuginfo/pretty-std-collections.rs
index 5e133ee718e2..568ca0fd2303 100644
--- a/tests/debuginfo/pretty-std-collections.rs
+++ b/tests/debuginfo/pretty-std-collections.rs
@@ -1,6 +1,7 @@
 //@ ignore-android: FIXME(#10381)
 //@ ignore-windows-gnu: #128981
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/pretty-std.rs b/tests/debuginfo/pretty-std.rs
index 53835d41be24..2620557eaf99 100644
--- a/tests/debuginfo/pretty-std.rs
+++ b/tests/debuginfo/pretty-std.rs
@@ -4,6 +4,7 @@
 //@ compile-flags:-g
 //@ min-lldb-version: 1800
 //@ min-cdb-version: 10.0.18317.1001
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/pretty-uninitialized-vec.rs b/tests/debuginfo/pretty-uninitialized-vec.rs
index cea2f26ef58f..e5a2df5c9345 100644
--- a/tests/debuginfo/pretty-uninitialized-vec.rs
+++ b/tests/debuginfo/pretty-uninitialized-vec.rs
@@ -1,6 +1,7 @@
 //@ ignore-windows-gnu: #128981
 //@ ignore-android: FIXME(#10381)
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/rc_arc.rs b/tests/debuginfo/rc_arc.rs
index f636c60702cd..c439ff7484fe 100644
--- a/tests/debuginfo/rc_arc.rs
+++ b/tests/debuginfo/rc_arc.rs
@@ -2,6 +2,7 @@
 //@ compile-flags:-g
 
 //@ min-cdb-version: 10.0.18317.1001
+//@ ignore-backends: gcc
 
 // === GDB TESTS ==================================================================================
 
diff --git a/tests/debuginfo/recursive-enum.rs b/tests/debuginfo/recursive-enum.rs
index b861e6d617c8..5fb339f54f39 100644
--- a/tests/debuginfo/recursive-enum.rs
+++ b/tests/debuginfo/recursive-enum.rs
@@ -1,14 +1,13 @@
 //@ ignore-lldb
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 // gdb-command:run
 
 // Test whether compiling a recursive enum definition crashes debug info generation. The test case
 // is taken from issue #11083 and #135093.
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 pub struct Window<'a> {
     callbacks: WindowCallbacks<'a>
diff --git a/tests/debuginfo/recursive-struct.rs b/tests/debuginfo/recursive-struct.rs
index a97eb295eb43..723ab06a8b26 100644
--- a/tests/debuginfo/recursive-struct.rs
+++ b/tests/debuginfo/recursive-struct.rs
@@ -1,6 +1,8 @@
 //@ ignore-lldb
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 
@@ -58,11 +60,10 @@
 // gdb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Opt::{Empty, Val};
 use std::boxed::Box as B;
+use std::marker::PhantomData;
 
 enum Opt {
     Empty,
@@ -99,6 +100,11 @@ struct LongCycleWithAnonymousTypes {
     value: usize,
 }
 
+struct Expanding {
+    a: PhantomData,
+    b: *const Expanding<(T, T)>,
+}
+
 // This test case makes sure that recursive structs are properly described. The Node structs are
 // generic so that we can have a new type (that newly needs to be described) for the different
 // cases. The potential problem with recursive types is that the DI generation algorithm gets
@@ -206,6 +212,9 @@ fn main() {
         value: 30
     })))));
 
+    // This type can generate new instances infinitely if not handled properly.
+    std::hint::black_box(Expanding::<()> { a: PhantomData, b: std::ptr::null() });
+
     zzz(); // #break
 }
 
diff --git a/tests/debuginfo/reference-debuginfo.rs b/tests/debuginfo/reference-debuginfo.rs
index 773c3ae4bc36..75acbc131a8c 100644
--- a/tests/debuginfo/reference-debuginfo.rs
+++ b/tests/debuginfo/reference-debuginfo.rs
@@ -3,6 +3,8 @@
 // and leaves codegen to create a ladder of allocations so as `*a == b`.
 //
 //@ compile-flags:-g -Zmir-enable-passes=+ReferencePropagation,-ConstDebugInfo
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -106,8 +108,6 @@
 // lldb-check:[...] 3.5
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(f16)]
 
 fn main() {
diff --git a/tests/debuginfo/regression-bad-location-list-67992.rs b/tests/debuginfo/regression-bad-location-list-67992.rs
index 0ec474b5b5af..a5617a23c931 100644
--- a/tests/debuginfo/regression-bad-location-list-67992.rs
+++ b/tests/debuginfo/regression-bad-location-list-67992.rs
@@ -1,4 +1,5 @@
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/self-in-default-method.rs b/tests/debuginfo/self-in-default-method.rs
index 02fc01d96eb3..6e08ef4cd83a 100644
--- a/tests/debuginfo/self-in-default-method.rs
+++ b/tests/debuginfo/self-in-default-method.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: isize
diff --git a/tests/debuginfo/self-in-generic-default-method.rs b/tests/debuginfo/self-in-generic-default-method.rs
index 65018e549ee0..a06731a54bd2 100644
--- a/tests/debuginfo/self-in-generic-default-method.rs
+++ b/tests/debuginfo/self-in-generic-default-method.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -99,9 +101,6 @@
 // lldb-check:[...] -10.5
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 #[derive(Copy, Clone)]
 struct Struct {
     x: isize
diff --git a/tests/debuginfo/shadowed-argument.rs b/tests/debuginfo/shadowed-argument.rs
index 3a575b4addfd..7a7e54545b88 100644
--- a/tests/debuginfo/shadowed-argument.rs
+++ b/tests/debuginfo/shadowed-argument.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -46,9 +48,6 @@
 // lldb-command:continue
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn a_function(x: bool, y: bool) {
     zzz(); // #break
     sentinel();
diff --git a/tests/debuginfo/shadowed-variable.rs b/tests/debuginfo/shadowed-variable.rs
index 752e4c233f12..c55f22df0037 100644
--- a/tests/debuginfo/shadowed-variable.rs
+++ b/tests/debuginfo/shadowed-variable.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -68,9 +70,6 @@
 // lldb-check:[...] 20
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
     let x = false;
     let y = true;
diff --git a/tests/debuginfo/simd.rs b/tests/debuginfo/simd.rs
index 12675a71a570..888a78117b54 100644
--- a/tests/debuginfo/simd.rs
+++ b/tests/debuginfo/simd.rs
@@ -7,6 +7,8 @@
 //@ ignore-s390x
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 // gdb-command:run
 
 // gdb-command:print vi8x16
@@ -35,8 +37,6 @@
 // gdb-command:continue
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(repr_simd)]
 
 #[repr(simd)]
diff --git a/tests/debuginfo/simple-lexical-scope.rs b/tests/debuginfo/simple-lexical-scope.rs
index 6008489bd65d..194da9169d9a 100644
--- a/tests/debuginfo/simple-lexical-scope.rs
+++ b/tests/debuginfo/simple-lexical-scope.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -66,9 +68,6 @@
 // lldb-command:continue
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 fn main() {
     let x = false;
 
diff --git a/tests/debuginfo/simple-struct.rs b/tests/debuginfo/simple-struct.rs
index bb6b2b798102..665bb2fdb579 100644
--- a/tests/debuginfo/simple-struct.rs
+++ b/tests/debuginfo/simple-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags: -g -Zmir-enable-passes=-CheckAlignment
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -84,8 +86,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct NoPadding16 {
     x: u16,
diff --git a/tests/debuginfo/simple-tuple.rs b/tests/debuginfo/simple-tuple.rs
index 82467ef3bcf2..c1509a8a9b4c 100644
--- a/tests/debuginfo/simple-tuple.rs
+++ b/tests/debuginfo/simple-tuple.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -121,8 +123,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 static mut NO_PADDING_8: (i8, u8) = (-50, 50);
 static mut NO_PADDING_16: (i16, i16, u16) = (-1, 2, 3);
diff --git a/tests/debuginfo/skip_second_statement.rs b/tests/debuginfo/skip_second_statement.rs
index ba182de03854..452b6db3882d 100644
--- a/tests/debuginfo/skip_second_statement.rs
+++ b/tests/debuginfo/skip_second_statement.rs
@@ -4,6 +4,7 @@
 // Performed step-over and step-into debug stepping through call statements.
 
 //@ compile-flags:-g -C collapse-macro-debuginfo=yes
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/skip_second_statement_collapse.rs b/tests/debuginfo/skip_second_statement_collapse.rs
index db90cee8bfba..52a85c5107e4 100644
--- a/tests/debuginfo/skip_second_statement_collapse.rs
+++ b/tests/debuginfo/skip_second_statement_collapse.rs
@@ -4,6 +4,7 @@
 // Performed step-over and step-into debug stepping through call statements.
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/static-method-on-struct-and-enum.rs b/tests/debuginfo/static-method-on-struct-and-enum.rs
index b487512a52f1..87b959f9df05 100644
--- a/tests/debuginfo/static-method-on-struct-and-enum.rs
+++ b/tests/debuginfo/static-method-on-struct-and-enum.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -41,9 +43,6 @@
 // lldb-check:[...] 5
 // lldb-command:continue
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct Struct {
     x: isize
 }
diff --git a/tests/debuginfo/step-into-match.rs b/tests/debuginfo/step-into-match.rs
index d4b7ea201195..a05ed8653be2 100644
--- a/tests/debuginfo/step-into-match.rs
+++ b/tests/debuginfo/step-into-match.rs
@@ -4,6 +4,7 @@
 // On Arm64 Windows, stepping at the end of a function on goes to the callsite, not the instruction
 // after it.
 //@ ignore-aarch64-pc-windows-msvc: Stepping out of functions behaves differently.
+//@ ignore-backends: gcc
 
 // === GDB TESTS ==============================================================
 
diff --git a/tests/debuginfo/strings-and-strs.rs b/tests/debuginfo/strings-and-strs.rs
index 7d550408bec3..392cf697e110 100644
--- a/tests/debuginfo/strings-and-strs.rs
+++ b/tests/debuginfo/strings-and-strs.rs
@@ -2,6 +2,7 @@
 //@ min-lldb-version: 1800
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // === GDB TESTS ===================================================================================
 // gdb-command:run
@@ -40,8 +41,6 @@
 
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 pub struct Foo<'a> {
     inner: &'a str,
diff --git a/tests/debuginfo/struct-in-enum.rs b/tests/debuginfo/struct-in-enum.rs
index bc2c59fe4aa9..bd61e7cb1437 100644
--- a/tests/debuginfo/struct-in-enum.rs
+++ b/tests/debuginfo/struct-in-enum.rs
@@ -1,6 +1,8 @@
 //@ min-lldb-version: 1800
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -30,8 +32,6 @@
 // lldb-check:[...] TheOnlyCase(Struct { x: 123, y: 456, z: 789 })
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Regular::{Case1, Case2};
 use self::Univariant::TheOnlyCase;
diff --git a/tests/debuginfo/struct-in-struct.rs b/tests/debuginfo/struct-in-struct.rs
index 3cf48470391b..4f770e7e56c8 100644
--- a/tests/debuginfo/struct-in-struct.rs
+++ b/tests/debuginfo/struct-in-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -43,8 +45,6 @@
 // lldb-check:[...] { x = { x = 25 } y = { x = { x = 26 y = 27 } y = { x = 28 y = 29 } z = { x = 30 y = 31 } } z = { x = { x = { x = 32 } } } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Simple {
     x: i32
diff --git a/tests/debuginfo/struct-namespace.rs b/tests/debuginfo/struct-namespace.rs
index 957884191003..d56c84c4f13b 100644
--- a/tests/debuginfo/struct-namespace.rs
+++ b/tests/debuginfo/struct-namespace.rs
@@ -1,5 +1,6 @@
 //@ ignore-gdb
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 // Check that structs get placed in the correct namespace
 
@@ -16,8 +17,6 @@
 
 #![allow(unused_variables)]
 #![allow(dead_code)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct1 {
     a: u32,
diff --git a/tests/debuginfo/struct-style-enum.rs b/tests/debuginfo/struct-style-enum.rs
index cea9f3def8bc..48ef97896ac1 100644
--- a/tests/debuginfo/struct-style-enum.rs
+++ b/tests/debuginfo/struct-style-enum.rs
@@ -1,5 +1,7 @@
 //@ min-lldb-version: 1800
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -36,8 +38,6 @@
 // lldb-check:(struct_style_enum::Univariant) univariant = { value = { a = -1 } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Regular::{Case1, Case2, Case3};
 use self::Univariant::TheOnlyCase;
diff --git a/tests/debuginfo/struct-with-destructor.rs b/tests/debuginfo/struct-with-destructor.rs
index c159824980a2..2cdac43ba500 100644
--- a/tests/debuginfo/struct-with-destructor.rs
+++ b/tests/debuginfo/struct-with-destructor.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -32,8 +34,6 @@
 // lldb-check:[...] { a = { a = { x = 7890 y = 9870 } } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct NoDestructor {
     x: i32,
diff --git a/tests/debuginfo/trait-pointers.rs b/tests/debuginfo/trait-pointers.rs
index 71da71b897ad..5cdefe94e507 100644
--- a/tests/debuginfo/trait-pointers.rs
+++ b/tests/debuginfo/trait-pointers.rs
@@ -1,10 +1,9 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 // gdb-command:run
 // lldb-command:run
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 trait Trait {
     fn method(&self) -> isize { 0 }
diff --git a/tests/debuginfo/tuple-in-struct.rs b/tests/debuginfo/tuple-in-struct.rs
index a74d6203f5f5..3223532b16a4 100644
--- a/tests/debuginfo/tuple-in-struct.rs
+++ b/tests/debuginfo/tuple-in-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // gdb-command:run
 
@@ -28,8 +30,6 @@
 // gdb-check:$10 = tuple_in_struct::MixedPadding {x: ((40, 41, 42), (43, 44)), y: (45, 46, 47, 48)}
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct NoPadding1 {
     x: (i32, i32),
diff --git a/tests/debuginfo/tuple-in-tuple.rs b/tests/debuginfo/tuple-in-tuple.rs
index d4388095ad72..e906cbed67db 100644
--- a/tests/debuginfo/tuple-in-tuple.rs
+++ b/tests/debuginfo/tuple-in-tuple.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -111,8 +113,6 @@
 // cdb-check:[...][1]              : 22 [Type: [...]]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 fn main() {
     let no_padding1: ((u32, u32), u32, u32) = ((0, 1), 2, 3);
diff --git a/tests/debuginfo/tuple-struct.rs b/tests/debuginfo/tuple-struct.rs
index 0110203a7c79..44fc862c43c9 100644
--- a/tests/debuginfo/tuple-struct.rs
+++ b/tests/debuginfo/tuple-struct.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -50,9 +52,6 @@
 // structs.
 
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct NoPadding16(u16, i16);
 struct NoPadding32(i32, f32, u32);
 struct NoPadding64(f64, i64, u64);
diff --git a/tests/debuginfo/tuple-style-enum.rs b/tests/debuginfo/tuple-style-enum.rs
index a759ad61c056..a452c57b3042 100644
--- a/tests/debuginfo/tuple-style-enum.rs
+++ b/tests/debuginfo/tuple-style-enum.rs
@@ -1,6 +1,8 @@
 //@ min-lldb-version: 1800
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -37,8 +39,6 @@
 // lldb-check:(tuple_style_enum::Univariant) univariant = { value = { 0 = -1 } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 use self::Regular::{Case1, Case2, Case3};
 use self::Univariant::TheOnlyCase;
diff --git a/tests/debuginfo/type-names.rs b/tests/debuginfo/type-names.rs
index ac61fef48fe1..341112b0b88e 100644
--- a/tests/debuginfo/type-names.rs
+++ b/tests/debuginfo/type-names.rs
@@ -6,6 +6,8 @@
 //@ min-gdb-version: 9.2
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ==================================================================================
 
@@ -271,8 +273,6 @@
 // cdb-check:struct type_names::mod1::extern$0::ForeignType2 * foreign2 = [...]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 #![feature(extern_types)]
 
 use std::marker::PhantomData;
diff --git a/tests/debuginfo/union-smoke.rs b/tests/debuginfo/union-smoke.rs
index 6043240069e3..beaf4c8fa8fe 100644
--- a/tests/debuginfo/union-smoke.rs
+++ b/tests/debuginfo/union-smoke.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -18,8 +20,6 @@
 // lldb-check:[...] { a = { 0 = '\x01' 1 = '\x01' } b = 257 }
 
 #![allow(unused)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 union U {
     a: (u8, u8),
diff --git a/tests/debuginfo/unique-enum.rs b/tests/debuginfo/unique-enum.rs
index 230429278aa5..a9b928456e93 100644
--- a/tests/debuginfo/unique-enum.rs
+++ b/tests/debuginfo/unique-enum.rs
@@ -1,6 +1,8 @@
 //@ min-lldb-version: 1800
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -30,8 +32,6 @@
 // lldb-check:(unique_enum::Univariant) *univariant = { value = { 0 = 123234 } }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // The first element is to ensure proper alignment, irrespective of the machines word size. Since
 // the size of the discriminant value is machine dependent, this has be taken into account when
diff --git a/tests/debuginfo/unit-type.rs b/tests/debuginfo/unit-type.rs
index 0ffa2fe490a5..ce2a39521d90 100644
--- a/tests/debuginfo/unit-type.rs
+++ b/tests/debuginfo/unit-type.rs
@@ -2,6 +2,7 @@
 
 // FIXME(jieyouxu): triple check if this works in CI
 //@ min-cdb-version: 10.0.26100.2161
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/debuginfo/unreachable-locals.rs b/tests/debuginfo/unreachable-locals.rs
index d4416387e0b9..4d3f01fe423b 100644
--- a/tests/debuginfo/unreachable-locals.rs
+++ b/tests/debuginfo/unreachable-locals.rs
@@ -1,8 +1,7 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 // No need to actually run the debugger, just make sure that the compiler can
 // handle locals in unreachable code.
diff --git a/tests/debuginfo/unsized.rs b/tests/debuginfo/unsized.rs
index edd9f5af557d..eb0fb8193bf7 100644
--- a/tests/debuginfo/unsized.rs
+++ b/tests/debuginfo/unsized.rs
@@ -1,5 +1,7 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
 //@ ignore-gdb-version: 13.1 - 99.0
+//@ ignore-backends: gcc
 // ^ https://sourceware.org/bugzilla/show_bug.cgi?id=30330
 
 // === GDB TESTS ===================================================================================
@@ -41,9 +43,6 @@
 // cdb-check:[+0x000] pointer          : 0x[...] [Type: unsized::Foo > *]
 // cdb-check:[...] vtable           : 0x[...] [Type: unsigned [...]int[...] (*)[4]]
 
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
-
 struct Foo {
     value: T,
 }
diff --git a/tests/debuginfo/var-captured-in-nested-closure.rs b/tests/debuginfo/var-captured-in-nested-closure.rs
index 4e8700015ba6..0343aea309d5 100644
--- a/tests/debuginfo/var-captured-in-nested-closure.rs
+++ b/tests/debuginfo/var-captured-in-nested-closure.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -115,8 +117,6 @@
 // cdb-check:closure_local    : 8 [Type: [...]]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct {
     a: isize,
diff --git a/tests/debuginfo/var-captured-in-sendable-closure.rs b/tests/debuginfo/var-captured-in-sendable-closure.rs
index cbb09daeb5f7..ad6aa5fb9c74 100644
--- a/tests/debuginfo/var-captured-in-sendable-closure.rs
+++ b/tests/debuginfo/var-captured-in-sendable-closure.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -28,8 +30,6 @@
 // lldb-check:[...] 5
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct {
     a: isize,
diff --git a/tests/debuginfo/var-captured-in-stack-closure.rs b/tests/debuginfo/var-captured-in-stack-closure.rs
index 0f84ea57b007..cc7e2eff81b4 100644
--- a/tests/debuginfo/var-captured-in-stack-closure.rs
+++ b/tests/debuginfo/var-captured-in-stack-closure.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -100,8 +102,6 @@
 // cdb-check:owned            : 0x[...] : 6 [Type: [...] *]
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct Struct {
     a: isize,
diff --git a/tests/debuginfo/vec-slices.rs b/tests/debuginfo/vec-slices.rs
index 2b4d624976ab..7ed6b0cb6640 100644
--- a/tests/debuginfo/vec-slices.rs
+++ b/tests/debuginfo/vec-slices.rs
@@ -2,6 +2,8 @@
 // ^ test temporarily disabled as it fails under gdb 15
 
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -71,8 +73,6 @@
 // lldb-check:[...] size=2 { [0] = { x = 10 y = 11 z = 12 } [1] = { x = 13 y = 14 z = 15 } }
 
 #![allow(dead_code, unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 struct AStruct {
     x: i16,
diff --git a/tests/debuginfo/vec.rs b/tests/debuginfo/vec.rs
index 1093e38d8780..c0e9f802f1e3 100644
--- a/tests/debuginfo/vec.rs
+++ b/tests/debuginfo/vec.rs
@@ -1,4 +1,6 @@
 //@ compile-flags:-g
+//@ disable-gdb-pretty-printers
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
@@ -16,8 +18,6 @@
 // lldb-check:[...] { [0] = 1 [1] = 2 [2] = 3 }
 
 #![allow(unused_variables)]
-#![feature(omit_gdb_pretty_printer_section)]
-#![omit_gdb_pretty_printer_section]
 
 static mut VECT: [i32; 3] = [1, 2, 3];
 
diff --git a/tests/debuginfo/zst-interferes-with-prologue.rs b/tests/debuginfo/zst-interferes-with-prologue.rs
index 09041a3bb72c..4d87df1abe7a 100644
--- a/tests/debuginfo/zst-interferes-with-prologue.rs
+++ b/tests/debuginfo/zst-interferes-with-prologue.rs
@@ -1,6 +1,7 @@
 //@ min-lldb-version: 310
 
 //@ compile-flags:-g
+//@ ignore-backends: gcc
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/incremental/add_private_fn_at_krate_root_cc/struct_point.rs b/tests/incremental/add_private_fn_at_krate_root_cc/struct_point.rs
index 65d363b35a67..0367af8d53b8 100644
--- a/tests/incremental/add_private_fn_at_krate_root_cc/struct_point.rs
+++ b/tests/incremental/add_private_fn_at_krate_root_cc/struct_point.rs
@@ -6,6 +6,7 @@
 //@ compile-flags: -Z query-dep-graph
 //@ aux-build:point.rs
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![feature(stmt_expr_attributes)]
diff --git a/tests/incremental/async-lifetimes.rs b/tests/incremental/async-lifetimes.rs
index 8d55e7293df8..f12f99519387 100644
--- a/tests/incremental/async-lifetimes.rs
+++ b/tests/incremental/async-lifetimes.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2
 //@ edition:2021
+//@ ignore-backends: gcc
 
 // See https://github.com/rust-lang/rust/issues/98890
 
diff --git a/tests/incremental/callee_caller_cross_crate/b.rs b/tests/incremental/callee_caller_cross_crate/b.rs
index b9012ecf7f77..78f3d8b3958f 100644
--- a/tests/incremental/callee_caller_cross_crate/b.rs
+++ b/tests/incremental/callee_caller_cross_crate/b.rs
@@ -1,6 +1,7 @@
 //@ aux-build:a.rs
 //@ revisions:rpass1 rpass2
 //@ compile-flags:-Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/change_add_field/struct_point.rs b/tests/incremental/change_add_field/struct_point.rs
index 8ff992771fdc..fb363c9ce496 100644
--- a/tests/incremental/change_add_field/struct_point.rs
+++ b/tests/incremental/change_add_field/struct_point.rs
@@ -6,6 +6,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![feature(stmt_expr_attributes)]
diff --git a/tests/incremental/change_crate_dep_kind.rs b/tests/incremental/change_crate_dep_kind.rs
index abca8de47198..4fd0c345aad2 100644
--- a/tests/incremental/change_crate_dep_kind.rs
+++ b/tests/incremental/change_crate_dep_kind.rs
@@ -6,6 +6,7 @@
 //@ compile-flags: -Z query-dep-graph -Cpanic=unwind
 //@ needs-unwind
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![feature(panic_unwind)]
 
diff --git a/tests/incremental/change_crate_order/main.rs b/tests/incremental/change_crate_order/main.rs
index 23f1782578e9..38a2541a76b5 100644
--- a/tests/incremental/change_crate_order/main.rs
+++ b/tests/incremental/change_crate_order/main.rs
@@ -1,6 +1,7 @@
 //@ aux-build:a.rs
 //@ aux-build:b.rs
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/change_implementation_cross_crate/main.rs b/tests/incremental/change_implementation_cross_crate/main.rs
index 9919b0d01cb6..5c3b0815115c 100644
--- a/tests/incremental/change_implementation_cross_crate/main.rs
+++ b/tests/incremental/change_implementation_cross_crate/main.rs
@@ -4,6 +4,7 @@
 //@ revisions: rpass1 rpass2
 //@ aux-build: a.rs
 //@ compile-flags: -Zquery-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type = "bin"]
diff --git a/tests/incremental/change_name_of_static_in_fn.rs b/tests/incremental/change_name_of_static_in_fn.rs
index 808194501866..c6160efed7b9 100644
--- a/tests/incremental/change_name_of_static_in_fn.rs
+++ b/tests/incremental/change_name_of_static_in_fn.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2 rpass3
+//@ ignore-backends: gcc
 
 // See issue #57692.
 
diff --git a/tests/incremental/change_private_fn/struct_point.rs b/tests/incremental/change_private_fn/struct_point.rs
index a4c0a8db4098..cce26fa7a3e6 100644
--- a/tests/incremental/change_private_fn/struct_point.rs
+++ b/tests/incremental/change_private_fn/struct_point.rs
@@ -4,6 +4,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![feature(stmt_expr_attributes)]
diff --git a/tests/incremental/change_private_fn_cc/struct_point.rs b/tests/incremental/change_private_fn_cc/struct_point.rs
index 6262e32a807d..87392ca857ed 100644
--- a/tests/incremental/change_private_fn_cc/struct_point.rs
+++ b/tests/incremental/change_private_fn_cc/struct_point.rs
@@ -5,6 +5,7 @@
 //@ compile-flags: -Z query-dep-graph
 //@ aux-build:point.rs
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/change_private_impl_method/struct_point.rs b/tests/incremental/change_private_impl_method/struct_point.rs
index 953d58101510..dc0ba82c0a31 100644
--- a/tests/incremental/change_private_impl_method/struct_point.rs
+++ b/tests/incremental/change_private_impl_method/struct_point.rs
@@ -4,6 +4,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![feature(stmt_expr_attributes)]
diff --git a/tests/incremental/change_private_impl_method_cc/struct_point.rs b/tests/incremental/change_private_impl_method_cc/struct_point.rs
index 42cc41da4c8b..eb51af720949 100644
--- a/tests/incremental/change_private_impl_method_cc/struct_point.rs
+++ b/tests/incremental/change_private_impl_method_cc/struct_point.rs
@@ -5,6 +5,7 @@
 //@ compile-flags: -Z query-dep-graph
 //@ aux-build:point.rs
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/change_pub_inherent_method_body/struct_point.rs b/tests/incremental/change_pub_inherent_method_body/struct_point.rs
index e112b25df5c5..b8e06d070a3c 100644
--- a/tests/incremental/change_pub_inherent_method_body/struct_point.rs
+++ b/tests/incremental/change_pub_inherent_method_body/struct_point.rs
@@ -3,6 +3,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/change_pub_inherent_method_sig/struct_point.rs b/tests/incremental/change_pub_inherent_method_sig/struct_point.rs
index d67011f1e88c..3672ec268010 100644
--- a/tests/incremental/change_pub_inherent_method_sig/struct_point.rs
+++ b/tests/incremental/change_pub_inherent_method_sig/struct_point.rs
@@ -3,6 +3,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/change_symbol_export_status.rs b/tests/incremental/change_symbol_export_status.rs
index 5fc6afa74119..37f8b9b52e5f 100644
--- a/tests/incremental/change_symbol_export_status.rs
+++ b/tests/incremental/change_symbol_export_status.rs
@@ -2,6 +2,7 @@
 //@ compile-flags: -Zquery-dep-graph
 //@ [rpass1]compile-flags: -Zincremental-ignore-spans
 //@ [rpass2]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![rustc_partition_reused(module = "change_symbol_export_status-mod1", cfg = "rpass2")]
diff --git a/tests/incremental/commandline-args.rs b/tests/incremental/commandline-args.rs
index f7aabe1e6d00..eb7c8b6daf05 100644
--- a/tests/incremental/commandline-args.rs
+++ b/tests/incremental/commandline-args.rs
@@ -18,6 +18,7 @@
 //@[rpass2] compile-flags: -C debuginfo=2
 //@[rpass3] compile-flags: -C debuginfo=2 --diagnostic-width=80
 //@[rpass4] compile-flags: -C debuginfo=2 --diagnostic-width=80 --remap-path-prefix=/home/bors/r=src
+//@ ignore-backends: gcc
 
 pub fn main() {
     // empty
diff --git a/tests/incremental/const-generics/change-const-param-gat.rs b/tests/incremental/const-generics/change-const-param-gat.rs
index 9fdac0142935..0aa628f4bd6b 100644
--- a/tests/incremental/const-generics/change-const-param-gat.rs
+++ b/tests/incremental/const-generics/change-const-param-gat.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2 rpass3
 //@ compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 #![feature(generic_associated_types)]
 
 // This test unsures that with_opt_const_param returns the
diff --git a/tests/incremental/const-generics/change-const-param-type.rs b/tests/incremental/const-generics/change-const-param-type.rs
index 989ad56990db..2dc1194f5128 100644
--- a/tests/incremental/const-generics/change-const-param-type.rs
+++ b/tests/incremental/const-generics/change-const-param-type.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2 rpass3
 //@ compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 enum Foo {
     Variant,
diff --git a/tests/incremental/crate_hash_reorder.rs b/tests/incremental/crate_hash_reorder.rs
index bd26d604a825..07b38014ff85 100644
--- a/tests/incremental/crate_hash_reorder.rs
+++ b/tests/incremental/crate_hash_reorder.rs
@@ -2,6 +2,7 @@
 
 //@ revisions:rpass1 rpass2 rpass3
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/decl_macro.rs b/tests/incremental/decl_macro.rs
index 74810ae42274..3ddf86a068f4 100644
--- a/tests/incremental/decl_macro.rs
+++ b/tests/incremental/decl_macro.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 // issue#112680
 
diff --git a/tests/incremental/dirty_clean.rs b/tests/incremental/dirty_clean.rs
index 24aedb011c6b..1188b46fa0ad 100644
--- a/tests/incremental/dirty_clean.rs
+++ b/tests/incremental/dirty_clean.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 cfail2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/env/env_macro.rs b/tests/incremental/env/env_macro.rs
index 0c026328874d..8d6bd2416b1f 100644
--- a/tests/incremental/env/env_macro.rs
+++ b/tests/incremental/env/env_macro.rs
@@ -10,6 +10,7 @@
 //@ [rpass3]rustc-env:EXAMPLE_ENV=two
 //@ [rpass3]exec-env:EXAMPLE_ENV=two
 //@ [cfail4]unset-rustc-env:EXAMPLE_ENV
+//@ ignore-backends: gcc
 
 fn main() {
     assert_eq!(env!("EXAMPLE_ENV"), std::env::var("EXAMPLE_ENV").unwrap());
diff --git a/tests/incremental/env/option_env_macro.rs b/tests/incremental/env/option_env_macro.rs
index 44c3bfd69e05..3b9110a5f4e0 100644
--- a/tests/incremental/env/option_env_macro.rs
+++ b/tests/incremental/env/option_env_macro.rs
@@ -12,6 +12,7 @@
 //@ [rpass3]exec-env:EXAMPLE_ENV=two
 //@ [rpass4]unset-rustc-env:EXAMPLE_ENV
 //@ [rpass4]unset-exec-env:EXAMPLE_ENV
+//@ ignore-backends: gcc
 
 fn main() {
     assert_eq!(option_env!("EXAMPLE_ENV"), std::env::var("EXAMPLE_ENV").ok().as_deref());
diff --git a/tests/incremental/extern_static/issue-49153.rs b/tests/incremental/extern_static/issue-49153.rs
index 72340173b5ca..386c175be523 100644
--- a/tests/incremental/extern_static/issue-49153.rs
+++ b/tests/incremental/extern_static/issue-49153.rs
@@ -1,6 +1,7 @@
 // https://github.com/rust-lang/rust/issues/49153
 
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 extern "C" {
     pub static __ImageBase: u8;
diff --git a/tests/incremental/hash-module-order.rs b/tests/incremental/hash-module-order.rs
index c29e1e5b64c9..5261965e8f3f 100644
--- a/tests/incremental/hash-module-order.rs
+++ b/tests/incremental/hash-module-order.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2
 //@ compile-flags: -Z incremental-ignore-spans -Z query-dep-graph
+//@ ignore-backends: gcc
 
 // Tests that module hashing depends on the order of the items
 // (since the order is exposed through `Mod.item_ids`).
diff --git a/tests/incremental/hashes/call_expressions.rs b/tests/incremental/hashes/call_expressions.rs
index 71423ef6aff6..d58bf0416c6f 100644
--- a/tests/incremental/hashes/call_expressions.rs
+++ b/tests/incremental/hashes/call_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 
 #![allow(warnings)]
diff --git a/tests/incremental/hashes/closure_expressions.rs b/tests/incremental/hashes/closure_expressions.rs
index e548e0b6ddc3..3cda661834f5 100644
--- a/tests/incremental/hashes/closure_expressions.rs
+++ b/tests/incremental/hashes/closure_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/consts.rs b/tests/incremental/hashes/consts.rs
index bd5cc89caa86..b60dce16c37e 100644
--- a/tests/incremental/hashes/consts.rs
+++ b/tests/incremental/hashes/consts.rs
@@ -8,6 +8,7 @@
 //@ build-pass (FIXME(62277): could be check-pass?)
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -O
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/enum_constructors.rs b/tests/incremental/hashes/enum_constructors.rs
index d839dabf293a..dee485681e44 100644
--- a/tests/incremental/hashes/enum_constructors.rs
+++ b/tests/incremental/hashes/enum_constructors.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/enum_defs.rs b/tests/incremental/hashes/enum_defs.rs
index bc1a2a31e97a..1f12c020116e 100644
--- a/tests/incremental/hashes/enum_defs.rs
+++ b/tests/incremental/hashes/enum_defs.rs
@@ -16,6 +16,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/exported_vs_not.rs b/tests/incremental/hashes/exported_vs_not.rs
index 1bd7756c1c6e..9ac86f39bd1a 100644
--- a/tests/incremental/hashes/exported_vs_not.rs
+++ b/tests/incremental/hashes/exported_vs_not.rs
@@ -4,6 +4,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/extern_mods.rs b/tests/incremental/hashes/extern_mods.rs
index c26fc90dd01e..0bda42fac8b8 100644
--- a/tests/incremental/hashes/extern_mods.rs
+++ b/tests/incremental/hashes/extern_mods.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/for_loops.rs b/tests/incremental/hashes/for_loops.rs
index b827ad9cc6f8..24717e0faaf5 100644
--- a/tests/incremental/hashes/for_loops.rs
+++ b/tests/incremental/hashes/for_loops.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/function_interfaces.rs b/tests/incremental/hashes/function_interfaces.rs
index 016a1813babd..e58ee1a6d4e4 100644
--- a/tests/incremental/hashes/function_interfaces.rs
+++ b/tests/incremental/hashes/function_interfaces.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(linkage)]
diff --git a/tests/incremental/hashes/if_expressions.rs b/tests/incremental/hashes/if_expressions.rs
index a21625c4f4ed..6a241692977f 100644
--- a/tests/incremental/hashes/if_expressions.rs
+++ b/tests/incremental/hashes/if_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/indexing_expressions.rs b/tests/incremental/hashes/indexing_expressions.rs
index 1b90483c50c3..bfc64c904039 100644
--- a/tests/incremental/hashes/indexing_expressions.rs
+++ b/tests/incremental/hashes/indexing_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/inherent_impls.rs b/tests/incremental/hashes/inherent_impls.rs
index 044c3d2b86c1..66d6c2b7246c 100644
--- a/tests/incremental/hashes/inherent_impls.rs
+++ b/tests/incremental/hashes/inherent_impls.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 
 #![allow(warnings)]
diff --git a/tests/incremental/hashes/inline_asm.rs b/tests/incremental/hashes/inline_asm.rs
index 105ca7611356..3453259e5b2f 100644
--- a/tests/incremental/hashes/inline_asm.rs
+++ b/tests/incremental/hashes/inline_asm.rs
@@ -12,6 +12,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/let_expressions.rs b/tests/incremental/hashes/let_expressions.rs
index 86578855699b..b2e89380acb3 100644
--- a/tests/incremental/hashes/let_expressions.rs
+++ b/tests/incremental/hashes/let_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/loop_expressions.rs b/tests/incremental/hashes/loop_expressions.rs
index 631699d3d687..a8b89217b949 100644
--- a/tests/incremental/hashes/loop_expressions.rs
+++ b/tests/incremental/hashes/loop_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/match_expressions.rs b/tests/incremental/hashes/match_expressions.rs
index 9ed5b55fb250..0c7f847932b8 100644
--- a/tests/incremental/hashes/match_expressions.rs
+++ b/tests/incremental/hashes/match_expressions.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/panic_exprs.rs b/tests/incremental/hashes/panic_exprs.rs
index 509e111edc7a..e3d6f8a211ec 100644
--- a/tests/incremental/hashes/panic_exprs.rs
+++ b/tests/incremental/hashes/panic_exprs.rs
@@ -11,6 +11,7 @@
 //@ build-pass (FIXME(62277): could be check-pass?)
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -C debug-assertions -O
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/statics.rs b/tests/incremental/hashes/statics.rs
index 9ea21d84ba15..cd394ed866e9 100644
--- a/tests/incremental/hashes/statics.rs
+++ b/tests/incremental/hashes/statics.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/struct_constructors.rs b/tests/incremental/hashes/struct_constructors.rs
index 31c549c6e7a9..da7abe049d98 100644
--- a/tests/incremental/hashes/struct_constructors.rs
+++ b/tests/incremental/hashes/struct_constructors.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/struct_defs.rs b/tests/incremental/hashes/struct_defs.rs
index 06ea08733979..c7582deb76ae 100644
--- a/tests/incremental/hashes/struct_defs.rs
+++ b/tests/incremental/hashes/struct_defs.rs
@@ -16,6 +16,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/trait_defs.rs b/tests/incremental/hashes/trait_defs.rs
index 50b56441cca5..13334a08cae8 100644
--- a/tests/incremental/hashes/trait_defs.rs
+++ b/tests/incremental/hashes/trait_defs.rs
@@ -16,6 +16,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/trait_impls.rs b/tests/incremental/hashes/trait_impls.rs
index a89b5b3e7829..03ca672af131 100644
--- a/tests/incremental/hashes/trait_impls.rs
+++ b/tests/incremental/hashes/trait_impls.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/type_defs.rs b/tests/incremental/hashes/type_defs.rs
index 137bffd9aeb6..b0cb2a6aa47d 100644
--- a/tests/incremental/hashes/type_defs.rs
+++ b/tests/incremental/hashes/type_defs.rs
@@ -13,6 +13,7 @@
 //@ build-pass (FIXME(62277): could be check-pass?)
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -O
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/unary_and_binary_exprs.rs b/tests/incremental/hashes/unary_and_binary_exprs.rs
index d0b502845071..fec167c2604c 100644
--- a/tests/incremental/hashes/unary_and_binary_exprs.rs
+++ b/tests/incremental/hashes/unary_and_binary_exprs.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/while_let_loops.rs b/tests/incremental/hashes/while_let_loops.rs
index 7a7bcb7d039d..1071ee86b1fb 100644
--- a/tests/incremental/hashes/while_let_loops.rs
+++ b/tests/incremental/hashes/while_let_loops.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hashes/while_loops.rs b/tests/incremental/hashes/while_loops.rs
index dc343c3cbfae..57287cbcdcee 100644
--- a/tests/incremental/hashes/while_loops.rs
+++ b/tests/incremental/hashes/while_loops.rs
@@ -11,6 +11,7 @@
 //@ [cfail1]compile-flags: -Zincremental-ignore-spans
 //@ [cfail2]compile-flags: -Zincremental-ignore-spans
 //@ [cfail3]compile-flags: -Zincremental-ignore-spans
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hello_world.rs b/tests/incremental/hello_world.rs
index ab1f9dbf1840..d6d9b6b598c2 100644
--- a/tests/incremental/hello_world.rs
+++ b/tests/incremental/hello_world.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/hygiene/load_cached_hygiene.rs b/tests/incremental/hygiene/load_cached_hygiene.rs
index 101d280cd492..84f91e7ecdec 100644
--- a/tests/incremental/hygiene/load_cached_hygiene.rs
+++ b/tests/incremental/hygiene/load_cached_hygiene.rs
@@ -1,6 +1,7 @@
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
 //@ aux-build:cached_hygiene.rs
+//@ ignore-backends: gcc
 
 // This tests the following scenario
 // 1. A foreign crate is compiled with incremental compilation.
diff --git a/tests/incremental/ich_method_call_trait_scope.rs b/tests/incremental/ich_method_call_trait_scope.rs
index a4c6558602f7..28234ca60d8e 100644
--- a/tests/incremental/ich_method_call_trait_scope.rs
+++ b/tests/incremental/ich_method_call_trait_scope.rs
@@ -3,6 +3,7 @@
 
 //@ revisions: rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/ich_nested_items.rs b/tests/incremental/ich_nested_items.rs
index c2a041ba268c..fa3ebcfa8e03 100644
--- a/tests/incremental/ich_nested_items.rs
+++ b/tests/incremental/ich_nested_items.rs
@@ -4,6 +4,7 @@
 //@ revisions: cfail1 cfail2
 //@ build-pass (FIXME(62277): could be check-pass?)
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/ich_resolve_results.rs b/tests/incremental/ich_resolve_results.rs
index c5b2b66fef9f..41c98bd38912 100644
--- a/tests/incremental/ich_resolve_results.rs
+++ b/tests/incremental/ich_resolve_results.rs
@@ -3,6 +3,7 @@
 
 //@ revisions: rpass1 rpass2 rpass3
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/incremental_proc_macro.rs b/tests/incremental/incremental_proc_macro.rs
index fc32aad5645c..19ae746fc272 100644
--- a/tests/incremental/incremental_proc_macro.rs
+++ b/tests/incremental/incremental_proc_macro.rs
@@ -1,6 +1,7 @@
 //@ proc-macro: incremental_proc_macro_aux.rs
 //@ revisions: cfail1 cfail2
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 // This test makes sure that we still find the proc-macro registrar function
 // when we compile proc-macros incrementally (see #47292).
diff --git a/tests/incremental/inlined_hir_34991/main.rs b/tests/incremental/inlined_hir_34991/main.rs
index 7754add917d2..1d4740bd1608 100644
--- a/tests/incremental/inlined_hir_34991/main.rs
+++ b/tests/incremental/inlined_hir_34991/main.rs
@@ -5,6 +5,7 @@
 // libstd), and we can't hash a HIR node from std.
 
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/issue-100521-change-struct-name-assocty.rs b/tests/incremental/issue-100521-change-struct-name-assocty.rs
index 0c1938beef27..80f96ca7144d 100644
--- a/tests/incremental/issue-100521-change-struct-name-assocty.rs
+++ b/tests/incremental/issue-100521-change-struct-name-assocty.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 pub fn foo() {
     bar();
diff --git a/tests/incremental/issue-108481-feed-eval-always.rs b/tests/incremental/issue-108481-feed-eval-always.rs
index ef2e2fb59c2d..e2229eeed0f1 100644
--- a/tests/incremental/issue-108481-feed-eval-always.rs
+++ b/tests/incremental/issue-108481-feed-eval-always.rs
@@ -1,4 +1,5 @@
 //@ revisions: cpass1 cpass2
+//@ ignore-backends: gcc
 
 #![crate_type = "rlib"]
 
diff --git a/tests/incremental/issue-110457-same-span-closures/main.rs b/tests/incremental/issue-110457-same-span-closures/main.rs
index 6a5e4b315ce3..b89ba9fb651a 100644
--- a/tests/incremental/issue-110457-same-span-closures/main.rs
+++ b/tests/incremental/issue-110457-same-span-closures/main.rs
@@ -1,5 +1,6 @@
 //@ proc-macro: egui_inspect_derive.rs
 //@ revisions: cpass1 cpass2
+//@ ignore-backends: gcc
 
 extern crate egui_inspect_derive;
 
diff --git a/tests/incremental/issue-35593.rs b/tests/incremental/issue-35593.rs
index 5e5235dc2c4c..522c88b9894a 100644
--- a/tests/incremental/issue-35593.rs
+++ b/tests/incremental/issue-35593.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![rustc_partition_reused(module="issue_35593", cfg="rpass2")]
diff --git a/tests/incremental/issue-38222.rs b/tests/incremental/issue-38222.rs
index e45a16f99b33..0aecd1e1f98a 100644
--- a/tests/incremental/issue-38222.rs
+++ b/tests/incremental/issue-38222.rs
@@ -12,6 +12,7 @@
 
 //@[rpass1] compile-flags: -C debuginfo=1
 //@[rpass2] compile-flags: -C debuginfo=1
+//@ ignore-backends: gcc
 
 pub fn main() {
     mod1::some_fn();
diff --git a/tests/incremental/issue-39569.rs b/tests/incremental/issue-39569.rs
index 6fd0776bd733..dd8d972a21bd 100644
--- a/tests/incremental/issue-39569.rs
+++ b/tests/incremental/issue-39569.rs
@@ -5,6 +5,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 use std::sync::Arc;
 
diff --git a/tests/incremental/issue-39828/issue-39828.rs b/tests/incremental/issue-39828/issue-39828.rs
index 703d8e8dd228..a6e62c038b84 100644
--- a/tests/incremental/issue-39828/issue-39828.rs
+++ b/tests/incremental/issue-39828/issue-39828.rs
@@ -7,6 +7,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ aux-build:generic.rs
+//@ ignore-backends: gcc
 
 extern crate generic;
 fn main() { }
diff --git a/tests/incremental/issue-42602.rs b/tests/incremental/issue-42602.rs
index d02c7f74665d..e75dd9696b79 100644
--- a/tests/incremental/issue-42602.rs
+++ b/tests/incremental/issue-42602.rs
@@ -9,6 +9,7 @@
 //@ revisions:cfail1 cfail2 cfail3
 //@ compile-flags:-Zquery-dep-graph
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/issue-49595/issue-49595.rs b/tests/incremental/issue-49595/issue-49595.rs
index 7fe843135dfe..890b1f963a2f 100644
--- a/tests/incremental/issue-49595/issue-49595.rs
+++ b/tests/incremental/issue-49595/issue-49595.rs
@@ -1,6 +1,7 @@
 //@ revisions:cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph --test
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type = "rlib"]
diff --git a/tests/incremental/issue-54242.rs b/tests/incremental/issue-54242.rs
index 17bbd619a8fb..1bc456e565b5 100644
--- a/tests/incremental/issue-54242.rs
+++ b/tests/incremental/issue-54242.rs
@@ -14,7 +14,7 @@ impl Tr for str {
     type Arr = [u8; 8];
     #[cfg(cfail)]
     type Arr = [u8; Self::C];
-    //[cfail]~^ ERROR cycle detected when caching mir
+    //[cfail]~^ ERROR cycle detected when
 }
 
 fn main() {}
diff --git a/tests/incremental/issue-59523-on-implemented-is-not-unused.rs b/tests/incremental/issue-59523-on-implemented-is-not-unused.rs
index 973757223230..8260bd350e72 100644
--- a/tests/incremental/issue-59523-on-implemented-is-not-unused.rs
+++ b/tests/incremental/issue-59523-on-implemented-is-not-unused.rs
@@ -4,6 +4,7 @@
 
 //@ revisions: cfail1 cfail2
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![deny(unused_attributes)]
diff --git a/tests/incremental/issue-59524-layout-scalar-valid-range-is-not-unused.rs b/tests/incremental/issue-59524-layout-scalar-valid-range-is-not-unused.rs
index fcd58f7a8d65..5954aaa18ff9 100644
--- a/tests/incremental/issue-59524-layout-scalar-valid-range-is-not-unused.rs
+++ b/tests/incremental/issue-59524-layout-scalar-valid-range-is-not-unused.rs
@@ -5,6 +5,7 @@
 
 //@ revisions: cfail1 cfail2
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![deny(unused_attributes)]
diff --git a/tests/incremental/issue-60629.rs b/tests/incremental/issue-60629.rs
index 117e222fc393..f21e5da92213 100644
--- a/tests/incremental/issue-60629.rs
+++ b/tests/incremental/issue-60629.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 struct A;
 
diff --git a/tests/incremental/issue-61530.rs b/tests/incremental/issue-61530.rs
index 673556a9d02f..a06fa724615b 100644
--- a/tests/incremental/issue-61530.rs
+++ b/tests/incremental/issue-61530.rs
@@ -1,6 +1,7 @@
 #![feature(repr_simd, core_intrinsics)]
 
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 use std::intrinsics::simd::simd_shuffle;
 
diff --git a/tests/incremental/issue-62649-path-collisions-happen.rs b/tests/incremental/issue-62649-path-collisions-happen.rs
index 3b04f0ac9d71..e604583b7aa7 100644
--- a/tests/incremental/issue-62649-path-collisions-happen.rs
+++ b/tests/incremental/issue-62649-path-collisions-happen.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 pub trait Something {
diff --git a/tests/incremental/issue-69596.rs b/tests/incremental/issue-69596.rs
index 00d0d2b64a4f..576cc0b82897 100644
--- a/tests/incremental/issue-69596.rs
+++ b/tests/incremental/issue-69596.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 #![allow(unused_imports)]
 
diff --git a/tests/incremental/issue-72386.rs b/tests/incremental/issue-72386.rs
index b3cd2a642081..de88dae5505f 100644
--- a/tests/incremental/issue-72386.rs
+++ b/tests/incremental/issue-72386.rs
@@ -1,6 +1,7 @@
 //@ revisions: rpass1 cfail1 rpass3
 //@ needs-asm-support
 //@ only-x86_64
+//@ ignore-backends: gcc
 // Regression test for issue #72386
 // Checks that we don't ICE when switching to an invalid register
 // and back again
diff --git a/tests/incremental/issue-79661-missing-def-path-hash.rs b/tests/incremental/issue-79661-missing-def-path-hash.rs
index c063c6a33a28..f26f680c67e8 100644
--- a/tests/incremental/issue-79661-missing-def-path-hash.rs
+++ b/tests/incremental/issue-79661-missing-def-path-hash.rs
@@ -1,5 +1,6 @@
 //@ aux-build:issue-79661.rs
 //@ revisions: rpass1 rpass2 rpass3
+//@ ignore-backends: gcc
 
 // Regression test for issue #79661
 // We were failing to copy over a DefPathHash->DefId mapping
diff --git a/tests/incremental/issue-79890-imported-crates-changed.rs b/tests/incremental/issue-79890-imported-crates-changed.rs
index 8c70d224f9e5..f29ebd650c6e 100644
--- a/tests/incremental/issue-79890-imported-crates-changed.rs
+++ b/tests/incremental/issue-79890-imported-crates-changed.rs
@@ -2,6 +2,7 @@
 //@ revisions:rpass1 rpass2 rpass3
 //@ compile-flags:--extern issue_79890 --test
 //@ edition:2018
+//@ ignore-backends: gcc
 
 // Tests that we don't ICE when the set of imported crates changes
 #[cfg(rpass2)] use issue_79890::MyTrait;
diff --git a/tests/incremental/issue-80336-invalid-span.rs b/tests/incremental/issue-80336-invalid-span.rs
index 251a59c3b57d..fe8a2688c496 100644
--- a/tests/incremental/issue-80336-invalid-span.rs
+++ b/tests/incremental/issue-80336-invalid-span.rs
@@ -4,6 +4,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ only-x86_64
+//@ ignore-backends: gcc
 
 pub fn main() {
     let _ = is_x86_feature_detected!("avx2");
diff --git a/tests/incremental/issue-80691-bad-eval-cache.rs b/tests/incremental/issue-80691-bad-eval-cache.rs
index 4f897bd64a0d..2d3dbf6fed66 100644
--- a/tests/incremental/issue-80691-bad-eval-cache.rs
+++ b/tests/incremental/issue-80691-bad-eval-cache.rs
@@ -2,6 +2,7 @@
 //@ failure-status: 101
 //@ error-pattern: not implemented
 //@ needs-unwind -Cpanic=abort causes abort instead of exit(101)
+//@ ignore-backends: gcc
 
 pub trait Interner {
     type InternedVariableKinds;
diff --git a/tests/incremental/issue-82920-predicate-order-miscompile.rs b/tests/incremental/issue-82920-predicate-order-miscompile.rs
index e1c8b1253f62..17a0601e9f4d 100644
--- a/tests/incremental/issue-82920-predicate-order-miscompile.rs
+++ b/tests/incremental/issue-82920-predicate-order-miscompile.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 trait MyTrait: One + Two {}
 impl One for T {
diff --git a/tests/incremental/issue-85197-invalid-span/invalid_span_main.rs b/tests/incremental/issue-85197-invalid-span/invalid_span_main.rs
index 6db5107cbe43..1814eb13ecc7 100644
--- a/tests/incremental/issue-85197-invalid-span/invalid_span_main.rs
+++ b/tests/incremental/issue-85197-invalid-span/invalid_span_main.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2
 //@ aux-build:invalid-span-helper-lib.rs
+//@ ignore-backends: gcc
 
 // This issue has several different parts. The high level idea is:
 // 1. We create an 'invalid' span with the help of the `respan` proc-macro,
diff --git a/tests/incremental/issue-85360-eval-obligation-ice.rs b/tests/incremental/issue-85360-eval-obligation-ice.rs
index 2148e45bb387..2b25d2a39484 100644
--- a/tests/incremental/issue-85360-eval-obligation-ice.rs
+++ b/tests/incremental/issue-85360-eval-obligation-ice.rs
@@ -3,6 +3,7 @@
 //@[cfail2] compile-flags: --crate-type=lib -Zassert-incr-state=loaded
 //@ edition: 2021
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![allow(dead_code)]
 
diff --git a/tests/incremental/issue-92163-missing-sourcefile/issue_92163_main.rs b/tests/incremental/issue-92163-missing-sourcefile/issue_92163_main.rs
index c5e73f056d70..991ac4cf257d 100644
--- a/tests/incremental/issue-92163-missing-sourcefile/issue_92163_main.rs
+++ b/tests/incremental/issue-92163-missing-sourcefile/issue_92163_main.rs
@@ -1,6 +1,7 @@
 //@ aux-build:first_crate.rs
 //@ aux-build:second_crate.rs
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 // Regression test for issue #92163
 // Under certain circumstances, we may end up trying to
diff --git a/tests/incremental/issue-92987-provisional-dep-node.rs b/tests/incremental/issue-92987-provisional-dep-node.rs
index 4daeb69189d5..4979f0956db0 100644
--- a/tests/incremental/issue-92987-provisional-dep-node.rs
+++ b/tests/incremental/issue-92987-provisional-dep-node.rs
@@ -1,4 +1,5 @@
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 // Regression test for issue #92987
 // Tests that we properly manage `DepNode`s during trait evaluation
diff --git a/tests/incremental/issue-96319-coinductive-cycle.rs b/tests/incremental/issue-96319-coinductive-cycle.rs
index cf98b325271d..f5524377256b 100644
--- a/tests/incremental/issue-96319-coinductive-cycle.rs
+++ b/tests/incremental/issue-96319-coinductive-cycle.rs
@@ -1,5 +1,6 @@
 //@ edition:2018
 //@ revisions: rpass1 rpass2
+//@ ignore-backends: gcc
 
 pub struct Stmt {
     pub stmt_type: StmtKind,
diff --git a/tests/incremental/krate-inherent.rs b/tests/incremental/krate-inherent.rs
index 0322d112560c..60ff636b6bea 100644
--- a/tests/incremental/krate-inherent.rs
+++ b/tests/incremental/krate-inherent.rs
@@ -1,6 +1,7 @@
 //@ revisions: cfail1 cfail2
 //@ compile-flags: -Z query-dep-graph
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/krate-inlined.rs b/tests/incremental/krate-inlined.rs
index 3d96b0570f15..563b14bc0263 100644
--- a/tests/incremental/krate-inlined.rs
+++ b/tests/incremental/krate-inlined.rs
@@ -4,6 +4,7 @@
 
 //@ revisions: rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/krate_reassign_34991/main.rs b/tests/incremental/krate_reassign_34991/main.rs
index 50f418a3fddb..2aa77377f5da 100644
--- a/tests/incremental/krate_reassign_34991/main.rs
+++ b/tests/incremental/krate_reassign_34991/main.rs
@@ -1,5 +1,6 @@
 //@ aux-build:a.rs
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/link_order/main.rs b/tests/incremental/link_order/main.rs
index 20931e25dd4b..6046c018ef53 100644
--- a/tests/incremental/link_order/main.rs
+++ b/tests/incremental/link_order/main.rs
@@ -1,6 +1,7 @@
 //@ aux-build:my_lib.rs
 //@ revisions:cfail1 cfail2
 //@ compile-flags:-Z query-dep-graph
+//@ ignore-backends: gcc
 
 // Tests that re-ordering the `-l` arguments used
 // when compiling an external dependency does not lead to
diff --git a/tests/incremental/lto.rs b/tests/incremental/lto.rs
index 044267a9cfdf..ddeddf95f8ae 100644
--- a/tests/incremental/lto.rs
+++ b/tests/incremental/lto.rs
@@ -1,6 +1,7 @@
 //@ no-prefer-dynamic
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -C lto
+//@ ignore-backends: gcc
 
 mod x {
     pub struct X {
diff --git a/tests/incremental/macro_export.rs b/tests/incremental/macro_export.rs
index 23e8f01cf330..4cc243ec9f0c 100644
--- a/tests/incremental/macro_export.rs
+++ b/tests/incremental/macro_export.rs
@@ -1,5 +1,6 @@
 //@ revisions: cfail1 cfail2 cfail3
 //@ build-pass (FIXME(62277): could be check-pass?)
+//@ ignore-backends: gcc
 
 // This test case makes sure that we can compile with incremental compilation
 // enabled when there are macros exported from this crate. (See #37756)
diff --git a/tests/incremental/mir-opt.rs b/tests/incremental/mir-opt.rs
index 04b82d9ea7fd..2735efd3ea59 100644
--- a/tests/incremental/mir-opt.rs
+++ b/tests/incremental/mir-opt.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph -Z mir-opt-level=3
+//@ ignore-backends: gcc
 
 fn main() {
     if std::env::var("a").is_ok() {
diff --git a/tests/incremental/remapped_paths_cc/main.rs b/tests/incremental/remapped_paths_cc/main.rs
index 35e03d754469..eee541148f0b 100644
--- a/tests/incremental/remapped_paths_cc/main.rs
+++ b/tests/incremental/remapped_paths_cc/main.rs
@@ -1,6 +1,7 @@
 //@ revisions:rpass1 rpass2 rpass3
 //@ compile-flags: -Z query-dep-graph -g
 //@ aux-build:extern_crate.rs
+//@ ignore-backends: gcc
 
 // This test case makes sure that we detect if paths emitted into debuginfo
 // are changed, even when the change happens in an external crate.
diff --git a/tests/incremental/remove-private-item-cross-crate/main.rs b/tests/incremental/remove-private-item-cross-crate/main.rs
index 209b061dfb66..762d0aac26de 100644
--- a/tests/incremental/remove-private-item-cross-crate/main.rs
+++ b/tests/incremental/remove-private-item-cross-crate/main.rs
@@ -4,6 +4,7 @@
 //@ revisions:rpass1 rpass2
 //@ aux-build:a.rs
 //@ compile-flags: -Zquery-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type = "bin"]
diff --git a/tests/incremental/remove_crate/main.rs b/tests/incremental/remove_crate/main.rs
index e3f15bd9c8cd..ee6883b6db27 100644
--- a/tests/incremental/remove_crate/main.rs
+++ b/tests/incremental/remove_crate/main.rs
@@ -2,6 +2,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ aux-build:extern_crate.rs
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 extern crate extern_crate;
diff --git a/tests/incremental/reorder_vtable.rs b/tests/incremental/reorder_vtable.rs
index 89ff14e8feee..aba2749e2f65 100644
--- a/tests/incremental/reorder_vtable.rs
+++ b/tests/incremental/reorder_vtable.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 // This test case makes sure re-order the methods in a vtable will
 // trigger recompilation of codegen units that instantiate it.
diff --git a/tests/incremental/rlib_cross_crate/b.rs b/tests/incremental/rlib_cross_crate/b.rs
index 672d2ba40684..3373f8098c4f 100644
--- a/tests/incremental/rlib_cross_crate/b.rs
+++ b/tests/incremental/rlib_cross_crate/b.rs
@@ -7,6 +7,7 @@
 //@ revisions:rpass1 rpass2 rpass3
 //@ no-prefer-dynamic
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/rustc-rust-log.rs b/tests/incremental/rustc-rust-log.rs
index 602d376bae5f..f3adbdcfa59c 100644
--- a/tests/incremental/rustc-rust-log.rs
+++ b/tests/incremental/rustc-rust-log.rs
@@ -6,6 +6,7 @@
 //@ dont-check-compiler-stderr
 //@ aux-build: rustc-rust-log-aux.rs
 //@ rustc-env:RUSTC_LOG=debug
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 fn main() {}
diff --git a/tests/incremental/slice-pattern-const-ice-83085.rs b/tests/incremental/slice-pattern-const-ice-83085.rs
index 4d318fd7ec18..e1a4407595f9 100644
--- a/tests/incremental/slice-pattern-const-ice-83085.rs
+++ b/tests/incremental/slice-pattern-const-ice-83085.rs
@@ -3,6 +3,7 @@
 // this used to fail to build straight away without needing any kind of
 // stage1/2 builds but tidy demands it
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 fn main() {
     const BOO: &[u8; 0] = &[];
diff --git a/tests/incremental/source_loc_macros.rs b/tests/incremental/source_loc_macros.rs
index dbe0a4ba9046..ba67fa2fadad 100644
--- a/tests/incremental/source_loc_macros.rs
+++ b/tests/incremental/source_loc_macros.rs
@@ -4,6 +4,7 @@
 //@ revisions:rpass1 rpass2
 
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/span_hash_stable/main.rs b/tests/incremental/span_hash_stable/main.rs
index 0dba650163d3..c524ee3ec56d 100644
--- a/tests/incremental/span_hash_stable/main.rs
+++ b/tests/incremental/span_hash_stable/main.rs
@@ -5,6 +5,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -g -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/spans_in_type_debuginfo.rs b/tests/incremental/spans_in_type_debuginfo.rs
index e8ae592c0c74..e56b782e92f1 100644
--- a/tests/incremental/spans_in_type_debuginfo.rs
+++ b/tests/incremental/spans_in_type_debuginfo.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph -g
+//@ ignore-backends: gcc
 
 #![rustc_partition_reused(module="spans_in_type_debuginfo-structs", cfg="rpass2")]
 #![rustc_partition_reused(module="spans_in_type_debuginfo-enums", cfg="rpass2")]
diff --git a/tests/incremental/spans_significant_w_debuginfo.rs b/tests/incremental/spans_significant_w_debuginfo.rs
index 48be9cd3bc96..06baecd75e5b 100644
--- a/tests/incremental/spans_significant_w_debuginfo.rs
+++ b/tests/incremental/spans_significant_w_debuginfo.rs
@@ -4,6 +4,7 @@
 //@ revisions:rpass1 rpass2
 
 //@ compile-flags: -g -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![rustc_partition_codegened(module = "spans_significant_w_debuginfo", cfg = "rpass2")]
diff --git a/tests/incremental/spans_significant_w_panic.rs b/tests/incremental/spans_significant_w_panic.rs
index f9016725258b..34363c359dca 100644
--- a/tests/incremental/spans_significant_w_panic.rs
+++ b/tests/incremental/spans_significant_w_panic.rs
@@ -4,6 +4,7 @@
 //@ revisions:rpass1 rpass2
 
 //@ compile-flags: -C overflow-checks=on -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![rustc_partition_codegened(module = "spans_significant_w_panic", cfg = "rpass2")]
diff --git a/tests/incremental/spike.rs b/tests/incremental/spike.rs
index 5e6cd2fe3b39..da1d20fe1165 100644
--- a/tests/incremental/spike.rs
+++ b/tests/incremental/spike.rs
@@ -4,6 +4,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/split_debuginfo_cached.rs b/tests/incremental/split_debuginfo_cached.rs
index 4dea77308a71..0feb1cb6174e 100644
--- a/tests/incremental/split_debuginfo_cached.rs
+++ b/tests/incremental/split_debuginfo_cached.rs
@@ -8,6 +8,7 @@
 
 //@ [rpass1]compile-flags: -g -Zquery-dep-graph -Csplit-debuginfo=packed -Zsplit-dwarf-kind=split
 //@ [rpass2]compile-flags: -g -Zquery-dep-graph -Csplit-debuginfo=packed -Zsplit-dwarf-kind=split
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 // For `rpass2`, nothing has changed so everything should re-used.
diff --git a/tests/incremental/static_cycle/b.rs b/tests/incremental/static_cycle/b.rs
index 7054b2ea4e36..c8243cb5328c 100644
--- a/tests/incremental/static_cycle/b.rs
+++ b/tests/incremental/static_cycle/b.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #![cfg_attr(rpass2, warn(dead_code))]
 
diff --git a/tests/incremental/static_refering_to_other_static/issue-49081.rs b/tests/incremental/static_refering_to_other_static/issue-49081.rs
index b6c1c1a9c8bd..83111412fee8 100644
--- a/tests/incremental/static_refering_to_other_static/issue-49081.rs
+++ b/tests/incremental/static_refering_to_other_static/issue-49081.rs
@@ -1,6 +1,7 @@
 // https://github.com/rust-lang/rust/issues/49081
 
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 pub static A: i32 = 42;
 pub static B: &i32 = &A;
diff --git a/tests/incremental/static_refering_to_other_static2/issue.rs b/tests/incremental/static_refering_to_other_static2/issue.rs
index 8b6dc6d3e4b0..7682352f3363 100644
--- a/tests/incremental/static_refering_to_other_static2/issue.rs
+++ b/tests/incremental/static_refering_to_other_static2/issue.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 pub static A: i32 = 42;
diff --git a/tests/incremental/static_refering_to_other_static3/issue.rs b/tests/incremental/static_refering_to_other_static3/issue.rs
index 39e46745f8a2..b7ad0eacfd01 100644
--- a/tests/incremental/static_refering_to_other_static3/issue.rs
+++ b/tests/incremental/static_refering_to_other_static3/issue.rs
@@ -1,4 +1,5 @@
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 pub static A: u8 = 42;
diff --git a/tests/incremental/static_stable_hash/issue-49301.rs b/tests/incremental/static_stable_hash/issue-49301.rs
index 68dc6d214c4e..7d6b292d80d2 100644
--- a/tests/incremental/static_stable_hash/issue-49301.rs
+++ b/tests/incremental/static_stable_hash/issue-49301.rs
@@ -1,6 +1,7 @@
 // https://github.com/rust-lang/rust/issues/49081
 
 //@ revisions:rpass1 rpass2
+//@ ignore-backends: gcc
 
 #[cfg(rpass1)]
 pub static A: &str = "hello";
diff --git a/tests/incremental/struct_add_field.rs b/tests/incremental/struct_add_field.rs
index ad1b6fcfe282..2c26b388122a 100644
--- a/tests/incremental/struct_add_field.rs
+++ b/tests/incremental/struct_add_field.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/struct_change_field_type.rs b/tests/incremental/struct_change_field_type.rs
index aed6c8884ac4..0291b34e3622 100644
--- a/tests/incremental/struct_change_field_type.rs
+++ b/tests/incremental/struct_change_field_type.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/struct_change_field_type_cross_crate/b.rs b/tests/incremental/struct_change_field_type_cross_crate/b.rs
index 00bc3db0de86..5307d0af9d0a 100644
--- a/tests/incremental/struct_change_field_type_cross_crate/b.rs
+++ b/tests/incremental/struct_change_field_type_cross_crate/b.rs
@@ -1,6 +1,7 @@
 //@ aux-build:a.rs
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/struct_change_nothing.rs b/tests/incremental/struct_change_nothing.rs
index 3fc9a1c4de7f..fd8c4882e1b2 100644
--- a/tests/incremental/struct_change_nothing.rs
+++ b/tests/incremental/struct_change_nothing.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/struct_remove_field.rs b/tests/incremental/struct_remove_field.rs
index 488d7996b339..23ba27344f9c 100644
--- a/tests/incremental/struct_remove_field.rs
+++ b/tests/incremental/struct_remove_field.rs
@@ -3,6 +3,7 @@
 
 //@ revisions:rpass1 rpass2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/thinlto/cgu_invalidated_via_import.rs b/tests/incremental/thinlto/cgu_invalidated_via_import.rs
index 04a5a06384a3..09380625ff61 100644
--- a/tests/incremental/thinlto/cgu_invalidated_via_import.rs
+++ b/tests/incremental/thinlto/cgu_invalidated_via_import.rs
@@ -5,6 +5,7 @@
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -O
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type="rlib"]
diff --git a/tests/incremental/thinlto/cgu_invalidated_when_export_added.rs b/tests/incremental/thinlto/cgu_invalidated_when_export_added.rs
index a9b05467bf68..b6c9ceec2719 100644
--- a/tests/incremental/thinlto/cgu_invalidated_when_export_added.rs
+++ b/tests/incremental/thinlto/cgu_invalidated_when_export_added.rs
@@ -1,5 +1,6 @@
 //@ revisions: cfail1 cfail2
 //@ build-pass
+//@ ignore-backends: gcc
 
 // rust-lang/rust#69798:
 //
diff --git a/tests/incremental/thinlto/cgu_invalidated_when_export_removed.rs b/tests/incremental/thinlto/cgu_invalidated_when_export_removed.rs
index 3a78a290c7b2..488e8f68c510 100644
--- a/tests/incremental/thinlto/cgu_invalidated_when_export_removed.rs
+++ b/tests/incremental/thinlto/cgu_invalidated_when_export_removed.rs
@@ -1,5 +1,6 @@
 //@ revisions: cfail1 cfail2
 //@ build-pass
+//@ ignore-backends: gcc
 
 // rust-lang/rust#69798:
 //
diff --git a/tests/incremental/thinlto/cgu_invalidated_when_import_added.rs b/tests/incremental/thinlto/cgu_invalidated_when_import_added.rs
index 4198c6c273ba..e17bf5eb8e80 100644
--- a/tests/incremental/thinlto/cgu_invalidated_when_import_added.rs
+++ b/tests/incremental/thinlto/cgu_invalidated_when_import_added.rs
@@ -1,6 +1,7 @@
 //@ revisions: cfail1 cfail2
 //@ compile-flags: -O -Zhuman-readable-cgu-names -Cllvm-args=-import-instr-limit=10
 //@ build-pass
+//@ ignore-backends: gcc
 
 // rust-lang/rust#59535:
 //
diff --git a/tests/incremental/thinlto/cgu_invalidated_when_import_removed.rs b/tests/incremental/thinlto/cgu_invalidated_when_import_removed.rs
index 4880d376fcc0..b2b86bdb80c2 100644
--- a/tests/incremental/thinlto/cgu_invalidated_when_import_removed.rs
+++ b/tests/incremental/thinlto/cgu_invalidated_when_import_removed.rs
@@ -1,6 +1,7 @@
 //@ revisions: cfail1 cfail2
 //@ compile-flags: -O -Zhuman-readable-cgu-names -Cllvm-args=-import-instr-limit=10
 //@ build-pass
+//@ ignore-backends: gcc
 
 // rust-lang/rust#59535:
 //
diff --git a/tests/incremental/thinlto/cgu_keeps_identical_fn.rs b/tests/incremental/thinlto/cgu_keeps_identical_fn.rs
index d124a3498c4a..6c87130b7385 100644
--- a/tests/incremental/thinlto/cgu_keeps_identical_fn.rs
+++ b/tests/incremental/thinlto/cgu_keeps_identical_fn.rs
@@ -6,6 +6,7 @@
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -O
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type = "rlib"]
diff --git a/tests/incremental/thinlto/independent_cgus_dont_affect_each_other.rs b/tests/incremental/thinlto/independent_cgus_dont_affect_each_other.rs
index e05508fa8e30..15d2f315d16e 100644
--- a/tests/incremental/thinlto/independent_cgus_dont_affect_each_other.rs
+++ b/tests/incremental/thinlto/independent_cgus_dont_affect_each_other.rs
@@ -4,6 +4,7 @@
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Z query-dep-graph -O
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 #![crate_type="rlib"]
diff --git a/tests/incremental/type_alias_cross_crate/b.rs b/tests/incremental/type_alias_cross_crate/b.rs
index 7187336ba5ee..7fa7073e3da9 100644
--- a/tests/incremental/type_alias_cross_crate/b.rs
+++ b/tests/incremental/type_alias_cross_crate/b.rs
@@ -1,6 +1,7 @@
 //@ aux-build:a.rs
 //@ revisions:rpass1 rpass2 rpass3
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![feature(rustc_attrs)]
 
diff --git a/tests/incremental/unchecked_dirty_clean.rs b/tests/incremental/unchecked_dirty_clean.rs
index aab3333dff34..e500bdf8fad3 100644
--- a/tests/incremental/unchecked_dirty_clean.rs
+++ b/tests/incremental/unchecked_dirty_clean.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 cfail2
 //@ compile-flags: -Z query-dep-graph
+//@ ignore-backends: gcc
 
 #![allow(warnings)]
 #![feature(rustc_attrs)]
diff --git a/tests/incremental/unrecoverable_query.rs b/tests/incremental/unrecoverable_query.rs
index 798a418f7990..367339cdcc02 100644
--- a/tests/incremental/unrecoverable_query.rs
+++ b/tests/incremental/unrecoverable_query.rs
@@ -7,6 +7,7 @@
 //@ revisions:cfail1 cfail2
 //@ compile-flags: --crate-type=lib
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![allow(dead_code)]
 
diff --git a/tests/incremental/user-written-closure-synthetic-closure-conflict.rs b/tests/incremental/user-written-closure-synthetic-closure-conflict.rs
index 618604d06b17..31b94536c4ab 100644
--- a/tests/incremental/user-written-closure-synthetic-closure-conflict.rs
+++ b/tests/incremental/user-written-closure-synthetic-closure-conflict.rs
@@ -1,5 +1,6 @@
 //@ revisions: rpass1 rpass2
 //@ edition: 2024
+//@ ignore-backends: gcc
 
 #![allow(unused)]
 
diff --git a/tests/incremental/warnings-reemitted.rs b/tests/incremental/warnings-reemitted.rs
index c38c43d46df6..5eefd8608c2f 100644
--- a/tests/incremental/warnings-reemitted.rs
+++ b/tests/incremental/warnings-reemitted.rs
@@ -1,6 +1,7 @@
 //@ revisions: cfail1 cfail2 cfail3
 //@ compile-flags: -Coverflow-checks=on
 //@ build-pass
+//@ ignore-backends: gcc
 
 #![warn(arithmetic_overflow)]
 
diff --git a/tests/mir-opt/address_of.address_of_reborrow.SimplifyCfg-initial.after.mir b/tests/mir-opt/address_of.address_of_reborrow.SimplifyCfg-initial.after.mir
index 8afb6ad250e0..a1fe278c6520 100644
--- a/tests/mir-opt/address_of.address_of_reborrow.SimplifyCfg-initial.after.mir
+++ b/tests/mir-opt/address_of.address_of_reborrow.SimplifyCfg-initial.after.mir
@@ -1,30 +1,30 @@
 // MIR for `address_of_reborrow` after SimplifyCfg-initial
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:8:10: 8:18, inferred_ty: *const [i32; 10]
+| 0: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:8:10: 8:18, inferred_ty: *const [i32; 10]
 | 1: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:10:10: 10:25, inferred_ty: *const dyn std::marker::Send
-| 2: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:14:12: 14:20, inferred_ty: *const [i32; 10]
-| 3: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:14:12: 14:20, inferred_ty: *const [i32; 10]
+| 2: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:14:12: 14:20, inferred_ty: *const [i32; 10]
+| 3: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:14:12: 14:20, inferred_ty: *const [i32; 10]
 | 4: user_ty: Canonical { value: Ty(*const [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:15:12: 15:28, inferred_ty: *const [i32; 10]
 | 5: user_ty: Canonical { value: Ty(*const [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:15:12: 15:28, inferred_ty: *const [i32; 10]
 | 6: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:16:12: 16:27, inferred_ty: *const dyn std::marker::Send
 | 7: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:16:12: 16:27, inferred_ty: *const dyn std::marker::Send
 | 8: user_ty: Canonical { value: Ty(*const [i32]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:17:12: 17:24, inferred_ty: *const [i32]
 | 9: user_ty: Canonical { value: Ty(*const [i32]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:17:12: 17:24, inferred_ty: *const [i32]
-| 10: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:19:10: 19:18, inferred_ty: *const [i32; 10]
+| 10: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:19:10: 19:18, inferred_ty: *const [i32; 10]
 | 11: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:21:10: 21:25, inferred_ty: *const dyn std::marker::Send
-| 12: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:24:12: 24:20, inferred_ty: *const [i32; 10]
-| 13: user_ty: Canonical { value: Ty(*const ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:24:12: 24:20, inferred_ty: *const [i32; 10]
+| 12: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:24:12: 24:20, inferred_ty: *const [i32; 10]
+| 13: user_ty: Canonical { value: Ty(*const ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:24:12: 24:20, inferred_ty: *const [i32; 10]
 | 14: user_ty: Canonical { value: Ty(*const [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:25:12: 25:28, inferred_ty: *const [i32; 10]
 | 15: user_ty: Canonical { value: Ty(*const [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:25:12: 25:28, inferred_ty: *const [i32; 10]
 | 16: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:26:12: 26:27, inferred_ty: *const dyn std::marker::Send
 | 17: user_ty: Canonical { value: Ty(*const dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:26:12: 26:27, inferred_ty: *const dyn std::marker::Send
 | 18: user_ty: Canonical { value: Ty(*const [i32]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:27:12: 27:24, inferred_ty: *const [i32]
 | 19: user_ty: Canonical { value: Ty(*const [i32]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:27:12: 27:24, inferred_ty: *const [i32]
-| 20: user_ty: Canonical { value: Ty(*mut ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:29:10: 29:16, inferred_ty: *mut [i32; 10]
+| 20: user_ty: Canonical { value: Ty(*mut ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:29:10: 29:16, inferred_ty: *mut [i32; 10]
 | 21: user_ty: Canonical { value: Ty(*mut dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:31:10: 31:23, inferred_ty: *mut dyn std::marker::Send
-| 22: user_ty: Canonical { value: Ty(*mut ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:34:12: 34:18, inferred_ty: *mut [i32; 10]
-| 23: user_ty: Canonical { value: Ty(*mut ^0), max_universe: U0, variables: [Ty(General(U0))] }, span: $DIR/address_of.rs:34:12: 34:18, inferred_ty: *mut [i32; 10]
+| 22: user_ty: Canonical { value: Ty(*mut ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:34:12: 34:18, inferred_ty: *mut [i32; 10]
+| 23: user_ty: Canonical { value: Ty(*mut ^c_0), max_universe: U0, variables: [Ty { ui: U0, sub_root: 0 }] }, span: $DIR/address_of.rs:34:12: 34:18, inferred_ty: *mut [i32; 10]
 | 24: user_ty: Canonical { value: Ty(*mut [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:35:12: 35:26, inferred_ty: *mut [i32; 10]
 | 25: user_ty: Canonical { value: Ty(*mut [i32; 10]), max_universe: U0, variables: [] }, span: $DIR/address_of.rs:35:12: 35:26, inferred_ty: *mut [i32; 10]
 | 26: user_ty: Canonical { value: Ty(*mut dyn std::marker::Send), max_universe: U0, variables: [Region(U0)] }, span: $DIR/address_of.rs:36:12: 36:25, inferred_ty: *mut dyn std::marker::Send
diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir
index 347e4119cd0e..435c1532895c 100644
--- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir
+++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-abort.mir
@@ -1,8 +1,8 @@
 // MIR for `a::{closure#0}` 0 coroutine_drop_async
 
 fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> {
-    debug _task_context => _19;
-    debug x => ((*(_1.0: &mut {async fn body of a()})).0: T);
+    debug _task_context => _2;
+    debug x => ((*_20).0: T);
     let mut _0: std::task::Poll<()>;
     let _3: T;
     let mut _4: impl std::future::Future;
@@ -10,25 +10,26 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     let mut _6: std::pin::Pin<&mut T>;
     let mut _7: &mut T;
     let mut _8: *mut T;
-    let mut _9: ();
-    let mut _10: std::task::Poll<()>;
-    let mut _11: &mut std::task::Context<'_>;
-    let mut _12: &mut impl std::future::Future;
-    let mut _13: std::pin::Pin<&mut impl std::future::Future>;
-    let mut _14: isize;
-    let mut _15: &mut std::task::Context<'_>;
-    let mut _16: &mut impl std::future::Future;
-    let mut _17: std::pin::Pin<&mut impl std::future::Future>;
-    let mut _18: isize;
-    let mut _19: &mut std::task::Context<'_>;
-    let mut _20: u32;
+    let mut _9: std::task::Poll<()>;
+    let mut _10: &mut std::task::Context<'_>;
+    let mut _11: &mut impl std::future::Future;
+    let mut _12: std::pin::Pin<&mut impl std::future::Future>;
+    let mut _13: isize;
+    let mut _14: &mut std::task::Context<'_>;
+    let mut _15: &mut impl std::future::Future;
+    let mut _16: std::pin::Pin<&mut impl std::future::Future>;
+    let mut _17: isize;
+    let mut _18: ();
+    let mut _19: u32;
+    let mut _20: &mut {async fn body of a()};
     scope 1 {
-        debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#4).0: T);
+        debug x => (((*_20) as variant#4).0: T);
     }
 
     bb0: {
-        _20 = discriminant((*(_1.0: &mut {async fn body of a()})));
-        switchInt(move _20) -> [0: bb9, 3: bb12, 4: bb13, otherwise: bb14];
+        _20 = copy (_1.0: &mut {async fn body of a()});
+        _19 = discriminant((*_20));
+        switchInt(move _19) -> [0: bb9, 3: bb12, 4: bb13, otherwise: bb14];
     }
 
     bb1: {
@@ -44,14 +45,14 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
 
     bb3: {
         _0 = Poll::<()>::Pending;
-        discriminant((*(_1.0: &mut {async fn body of a()}))) = 4;
+        discriminant((*_20)) = 4;
         return;
     }
 
     bb4: {
-        StorageLive(_17);
-        _16 = &mut (((*(_1.0: &mut {async fn body of a()})) as variant#4).1: impl std::future::Future);
-        _17 = Pin::<&mut impl Future>::new_unchecked(move _16) -> [return: bb7, unwind unreachable];
+        StorageLive(_16);
+        _15 = &mut (((*_20) as variant#4).1: impl std::future::Future);
+        _16 = Pin::<&mut impl Future>::new_unchecked(move _15) -> [return: bb7, unwind unreachable];
     }
 
     bb5: {
@@ -59,13 +60,13 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     }
 
     bb6: {
-        StorageDead(_17);
-        _18 = discriminant(_10);
-        switchInt(move _18) -> [0: bb1, 1: bb3, otherwise: bb5];
+        StorageDead(_16);
+        _17 = discriminant(_9);
+        switchInt(move _17) -> [0: bb1, 1: bb3, otherwise: bb5];
     }
 
     bb7: {
-        _10 =  as Future>::poll(move _17, move _15) -> [return: bb6, unwind unreachable];
+        _9 =  as Future>::poll(move _16, move _14) -> [return: bb6, unwind unreachable];
     }
 
     bb8: {
@@ -82,7 +83,7 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     }
 
     bb11: {
-        drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb10, unwind unreachable];
+        drop(((*_20).0: T)) -> [return: bb10, unwind unreachable];
     }
 
     bb12: {
diff --git a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir
index b1cf5373f919..1dc1d0813629 100644
--- a/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir
+++ b/tests/mir-opt/async_drop_live_dead.a-{closure#0}.coroutine_drop_async.0.panic-unwind.mir
@@ -1,8 +1,8 @@
 // MIR for `a::{closure#0}` 0 coroutine_drop_async
 
 fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> {
-    debug _task_context => _19;
-    debug x => ((*(_1.0: &mut {async fn body of a()})).0: T);
+    debug _task_context => _2;
+    debug x => ((*_20).0: T);
     let mut _0: std::task::Poll<()>;
     let _3: T;
     let mut _4: impl std::future::Future;
@@ -10,25 +10,26 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     let mut _6: std::pin::Pin<&mut T>;
     let mut _7: &mut T;
     let mut _8: *mut T;
-    let mut _9: ();
-    let mut _10: std::task::Poll<()>;
-    let mut _11: &mut std::task::Context<'_>;
-    let mut _12: &mut impl std::future::Future;
-    let mut _13: std::pin::Pin<&mut impl std::future::Future>;
-    let mut _14: isize;
-    let mut _15: &mut std::task::Context<'_>;
-    let mut _16: &mut impl std::future::Future;
-    let mut _17: std::pin::Pin<&mut impl std::future::Future>;
-    let mut _18: isize;
-    let mut _19: &mut std::task::Context<'_>;
-    let mut _20: u32;
+    let mut _9: std::task::Poll<()>;
+    let mut _10: &mut std::task::Context<'_>;
+    let mut _11: &mut impl std::future::Future;
+    let mut _12: std::pin::Pin<&mut impl std::future::Future>;
+    let mut _13: isize;
+    let mut _14: &mut std::task::Context<'_>;
+    let mut _15: &mut impl std::future::Future;
+    let mut _16: std::pin::Pin<&mut impl std::future::Future>;
+    let mut _17: isize;
+    let mut _18: ();
+    let mut _19: u32;
+    let mut _20: &mut {async fn body of a()};
     scope 1 {
-        debug x => (((*(_1.0: &mut {async fn body of a()})) as variant#4).0: T);
+        debug x => (((*_20) as variant#4).0: T);
     }
 
     bb0: {
-        _20 = discriminant((*(_1.0: &mut {async fn body of a()})));
-        switchInt(move _20) -> [0: bb12, 2: bb18, 3: bb16, 4: bb17, otherwise: bb19];
+        _20 = copy (_1.0: &mut {async fn body of a()});
+        _19 = discriminant((*_20));
+        switchInt(move _19) -> [0: bb12, 2: bb18, 3: bb16, 4: bb17, otherwise: bb19];
     }
 
     bb1: {
@@ -58,14 +59,14 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
 
     bb6: {
         _0 = Poll::<()>::Pending;
-        discriminant((*(_1.0: &mut {async fn body of a()}))) = 4;
+        discriminant((*_20)) = 4;
         return;
     }
 
     bb7: {
-        StorageLive(_17);
-        _16 = &mut (((*(_1.0: &mut {async fn body of a()})) as variant#4).1: impl std::future::Future);
-        _17 = Pin::<&mut impl Future>::new_unchecked(move _16) -> [return: bb10, unwind: bb15];
+        StorageLive(_16);
+        _15 = &mut (((*_20) as variant#4).1: impl std::future::Future);
+        _16 = Pin::<&mut impl Future>::new_unchecked(move _15) -> [return: bb10, unwind: bb15];
     }
 
     bb8: {
@@ -73,13 +74,13 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     }
 
     bb9: {
-        StorageDead(_17);
-        _18 = discriminant(_10);
-        switchInt(move _18) -> [0: bb1, 1: bb6, otherwise: bb8];
+        StorageDead(_16);
+        _17 = discriminant(_9);
+        switchInt(move _17) -> [0: bb1, 1: bb6, otherwise: bb8];
     }
 
     bb10: {
-        _10 =  as Future>::poll(move _17, move _15) -> [return: bb9, unwind: bb3];
+        _9 =  as Future>::poll(move _16, move _14) -> [return: bb9, unwind: bb3];
     }
 
     bb11: {
@@ -96,11 +97,11 @@ fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>)
     }
 
     bb14: {
-        drop(((*(_1.0: &mut {async fn body of a()})).0: T)) -> [return: bb13, unwind: bb4];
+        drop(((*_20).0: T)) -> [return: bb13, unwind: bb4];
     }
 
     bb15 (cleanup): {
-        discriminant((*(_1.0: &mut {async fn body of a()}))) = 2;
+        discriminant((*_20)) = 2;
         resume;
     }
 
diff --git a/tests/mir-opt/box_conditional_drop_allocator.main.ElaborateDrops.diff b/tests/mir-opt/box_conditional_drop_allocator.main.ElaborateDrops.diff
new file mode 100644
index 000000000000..6f6c239d7c83
--- /dev/null
+++ b/tests/mir-opt/box_conditional_drop_allocator.main.ElaborateDrops.diff
@@ -0,0 +1,186 @@
+- // MIR for `main` before ElaborateDrops
++ // MIR for `main` after ElaborateDrops
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: std::boxed::Box;
+      let mut _2: HasDrop;
+      let mut _3: DropAllocator;
+      let mut _4: bool;
+      let _5: ();
+      let mut _6: HasDrop;
+      let _7: ();
+      let mut _8: std::boxed::Box;
++     let mut _9: bool;
++     let mut _10: &mut std::boxed::Box;
++     let mut _11: ();
++     let mut _12: &mut std::boxed::Box;
++     let mut _13: ();
++     let mut _14: *const HasDrop;
++     let mut _15: &mut std::boxed::Box;
++     let mut _16: ();
++     let mut _17: *const HasDrop;
+      scope 1 {
+          debug b => _1;
+      }
+  
+      bb0: {
++         _9 = const false;
+          StorageLive(_1);
+          StorageLive(_2);
+          _2 = HasDrop;
+          StorageLive(_3);
+          _3 = DropAllocator;
+          _1 = Box::::new_in(move _2, move _3) -> [return: bb1, unwind: bb11];
+      }
+  
+      bb1: {
++         _9 = const true;
+          StorageDead(_3);
+          StorageDead(_2);
+          StorageLive(_4);
+          _4 = const true;
+          switchInt(move _4) -> [0: bb4, otherwise: bb2];
+      }
+  
+      bb2: {
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = move (*_1);
+          _5 = std::mem::drop::(move _6) -> [return: bb3, unwind: bb9];
+      }
+  
+      bb3: {
+          StorageDead(_6);
+          StorageDead(_5);
+          _0 = const ();
+          goto -> bb6;
+      }
+  
+      bb4: {
+          StorageLive(_7);
+          StorageLive(_8);
++         _9 = const false;
+          _8 = move _1;
+          _7 = std::mem::drop::>(move _8) -> [return: bb5, unwind: bb8];
+      }
+  
+      bb5: {
+          StorageDead(_8);
+          StorageDead(_7);
+          _0 = const ();
+          goto -> bb6;
+      }
+  
+      bb6: {
+          StorageDead(_4);
+-         drop(_1) -> [return: bb7, unwind continue];
++         goto -> bb23;
+      }
+  
+      bb7: {
++         _9 = const false;
+          StorageDead(_1);
+          return;
+      }
+  
+      bb8 (cleanup): {
+-         drop(_8) -> [return: bb10, unwind terminate(cleanup)];
++         goto -> bb10;
+      }
+  
+      bb9 (cleanup): {
+-         drop(_6) -> [return: bb10, unwind terminate(cleanup)];
++         goto -> bb10;
+      }
+  
+      bb10 (cleanup): {
+-         drop(_1) -> [return: bb13, unwind terminate(cleanup)];
++         goto -> bb29;
+      }
+  
+      bb11 (cleanup): {
+-         drop(_3) -> [return: bb12, unwind terminate(cleanup)];
++         goto -> bb12;
+      }
+  
+      bb12 (cleanup): {
+-         drop(_2) -> [return: bb13, unwind terminate(cleanup)];
++         goto -> bb13;
+      }
+  
+      bb13 (cleanup): {
+          resume;
++     }
++ 
++     bb14: {
++         _9 = const false;
++         goto -> bb7;
++     }
++ 
++     bb15 (cleanup): {
++         drop((_1.1: DropAllocator)) -> [return: bb13, unwind terminate(cleanup)];
++     }
++ 
++     bb16 (cleanup): {
++         switchInt(copy _9) -> [0: bb13, otherwise: bb15];
++     }
++ 
++     bb17: {
++         drop((_1.1: DropAllocator)) -> [return: bb14, unwind: bb13];
++     }
++ 
++     bb18: {
++         switchInt(copy _9) -> [0: bb14, otherwise: bb17];
++     }
++ 
++     bb19: {
++         _10 = &mut _1;
++         _11 =  as Drop>::drop(move _10) -> [return: bb18, unwind: bb16];
++     }
++ 
++     bb20 (cleanup): {
++         _12 = &mut _1;
++         _13 =  as Drop>::drop(move _12) -> [return: bb16, unwind terminate(cleanup)];
++     }
++ 
++     bb21: {
++         goto -> bb19;
++     }
++ 
++     bb22: {
++         _14 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const HasDrop (Transmute);
++         goto -> bb21;
++     }
++ 
++     bb23: {
++         switchInt(copy _9) -> [0: bb18, otherwise: bb22];
++     }
++ 
++     bb24 (cleanup): {
++         drop((_1.1: DropAllocator)) -> [return: bb13, unwind terminate(cleanup)];
++     }
++ 
++     bb25 (cleanup): {
++         switchInt(copy _9) -> [0: bb13, otherwise: bb24];
++     }
++ 
++     bb26 (cleanup): {
++         _15 = &mut _1;
++         _16 =  as Drop>::drop(move _15) -> [return: bb25, unwind terminate(cleanup)];
++     }
++ 
++     bb27 (cleanup): {
++         goto -> bb26;
++     }
++ 
++     bb28 (cleanup): {
++         _17 = copy ((_1.0: std::ptr::Unique).0: std::ptr::NonNull) as *const HasDrop (Transmute);
++         goto -> bb27;
++     }
++ 
++     bb29 (cleanup): {
++         switchInt(copy _9) -> [0: bb25, otherwise: bb28];
+      }
+  }
+  
diff --git a/tests/mir-opt/box_conditional_drop_allocator.rs b/tests/mir-opt/box_conditional_drop_allocator.rs
new file mode 100644
index 000000000000..9471be14c877
--- /dev/null
+++ b/tests/mir-opt/box_conditional_drop_allocator.rs
@@ -0,0 +1,39 @@
+// skip-filecheck
+//@ test-mir-pass: ElaborateDrops
+//@ needs-unwind
+#![feature(allocator_api)]
+
+// Regression test for #131082.
+// Testing that the allocator of a Box is dropped in conditional drops
+
+use std::alloc::{AllocError, Allocator, Global, Layout};
+use std::ptr::NonNull;
+
+struct DropAllocator;
+
+unsafe impl Allocator for DropAllocator {
+    fn allocate(&self, layout: Layout) -> Result, AllocError> {
+        Global.allocate(layout)
+    }
+    unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) {
+        Global.deallocate(ptr, layout);
+    }
+}
+impl Drop for DropAllocator {
+    fn drop(&mut self) {}
+}
+
+struct HasDrop;
+impl Drop for HasDrop {
+    fn drop(&mut self) {}
+}
+
+// EMIT_MIR box_conditional_drop_allocator.main.ElaborateDrops.diff
+fn main() {
+    let b = Box::new_in(HasDrop, DropAllocator);
+    if true {
+        drop(*b);
+    } else {
+        drop(b);
+    }
+}
diff --git a/tests/mir-opt/box_expr.main.ElaborateDrops.diff b/tests/mir-opt/box_expr.main.ElaborateDrops.diff
index 827dc6ac7aef..4fa77cf82d81 100644
--- a/tests/mir-opt/box_expr.main.ElaborateDrops.diff
+++ b/tests/mir-opt/box_expr.main.ElaborateDrops.diff
@@ -4,49 +4,45 @@
   fn main() -> () {
       let mut _0: ();
       let _1: std::boxed::Box;
-      let mut _2: usize;
-      let mut _3: usize;
-      let mut _4: *mut u8;
+      let mut _2: *mut u8;
+      let mut _3: std::boxed::Box;
+      let _4: ();
       let mut _5: std::boxed::Box;
-      let _6: ();
-      let mut _7: std::boxed::Box;
-+     let mut _8: &mut std::boxed::Box;
-+     let mut _9: ();
-+     let mut _10: *const S;
++     let mut _6: &mut std::boxed::Box;
++     let mut _7: ();
++     let mut _8: *const S;
       scope 1 {
           debug x => _1;
       }
   
       bb0: {
           StorageLive(_1);
-          _2 = SizeOf(S);
-          _3 = AlignOf(S);
-          _4 = alloc::alloc::exchange_malloc(move _2, move _3) -> [return: bb1, unwind continue];
+          _2 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind continue];
       }
   
       bb1: {
-          StorageLive(_5);
-          _5 = ShallowInitBox(move _4, S);
-          (*_5) = S::new() -> [return: bb2, unwind: bb8];
+          StorageLive(_3);
+          _3 = ShallowInitBox(move _2, S);
+          (*_3) = S::new() -> [return: bb2, unwind: bb8];
       }
   
       bb2: {
-          _1 = move _5;
--         drop(_5) -> [return: bb3, unwind continue];
+          _1 = move _3;
+-         drop(_3) -> [return: bb3, unwind continue];
 +         goto -> bb3;
       }
   
       bb3: {
-          StorageDead(_5);
-          StorageLive(_6);
-          StorageLive(_7);
-          _7 = move _1;
-          _6 = std::mem::drop::>(move _7) -> [return: bb4, unwind: bb6];
+          StorageDead(_3);
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = move _1;
+          _4 = std::mem::drop::>(move _5) -> [return: bb4, unwind: bb6];
       }
   
       bb4: {
-          StorageDead(_7);
-          StorageDead(_6);
+          StorageDead(_5);
+          StorageDead(_4);
           _0 = const ();
 -         drop(_1) -> [return: bb5, unwind continue];
 +         goto -> bb5;
@@ -58,7 +54,7 @@
       }
   
       bb6 (cleanup): {
--         drop(_7) -> [return: bb7, unwind terminate(cleanup)];
+-         drop(_5) -> [return: bb7, unwind terminate(cleanup)];
 +         goto -> bb7;
       }
   
@@ -68,7 +64,7 @@
       }
   
       bb8 (cleanup): {
--         drop(_5) -> [return: bb9, unwind terminate(cleanup)];
+-         drop(_3) -> [return: bb9, unwind terminate(cleanup)];
 +         goto -> bb12;
       }
   
@@ -77,8 +73,8 @@
 +     }
 + 
 +     bb10 (cleanup): {
-+         _8 = &mut _5;
-+         _9 =  as Drop>::drop(move _8) -> [return: bb9, unwind terminate(cleanup)];
++         _6 = &mut _3;
++         _7 =  as Drop>::drop(move _6) -> [return: bb9, unwind terminate(cleanup)];
 +     }
 + 
 +     bb11 (cleanup): {
@@ -86,7 +82,7 @@
 +     }
 + 
 +     bb12 (cleanup): {
-+         _10 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const S (Transmute);
++         _8 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const S (Transmute);
 +         goto -> bb11;
       }
   }
diff --git a/tests/mir-opt/box_expr.rs b/tests/mir-opt/box_expr.rs
index 009a5ae54e08..6299c9871809 100644
--- a/tests/mir-opt/box_expr.rs
+++ b/tests/mir-opt/box_expr.rs
@@ -6,7 +6,10 @@
 // EMIT_MIR box_expr.main.ElaborateDrops.diff
 fn main() {
     // CHECK-LABEL: fn main(
-    // CHECK:   [[box:_.*]] = ShallowInitBox(
+    // CHECK:   [[ptr:_.*]] = move {{_.*}} as *const S (Transmute);
+    // CHECK:   [[nonnull:_.*]] = NonNull:: { pointer: move [[ptr]] };
+    // CHECK:   [[unique:_.*]] = Unique:: { pointer: move [[nonnull]], _marker: const PhantomData:: };
+    // CHECK:   [[box:_.*]] = Box::(move [[unique]], const std::alloc::Global);
     // CHECK:   [[ptr:_.*]] = copy (([[box]].0: std::ptr::Unique).0: std::ptr::NonNull) as *const S (Transmute);
     // CHECK:   (*[[ptr]]) = S::new() -> [return: [[ret:bb.*]], unwind: [[unwind:bb.*]]];
     // CHECK: [[ret]]: {
diff --git a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir
index 7480324b1779..6cad5b105d3e 100644
--- a/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir
+++ b/tests/mir-opt/building/async_await.a-{closure#0}.coroutine_resume.0.mir
@@ -10,26 +10,26 @@
 } */
 
 fn a::{closure#0}(_1: Pin<&mut {async fn body of a()}>, _2: &mut Context<'_>) -> Poll<()> {
-    debug _task_context => _4;
+    debug _task_context => _2;
     let mut _0: std::task::Poll<()>;
     let mut _3: ();
-    let mut _4: &mut std::task::Context<'_>;
-    let mut _5: u32;
+    let mut _4: u32;
+    let mut _5: &mut {async fn body of a()};
 
     bb0: {
-        _5 = discriminant((*(_1.0: &mut {async fn body of a()})));
-        switchInt(move _5) -> [0: bb1, 1: bb4, otherwise: bb5];
+        _5 = copy (_1.0: &mut {async fn body of a()});
+        _4 = discriminant((*_5));
+        switchInt(move _4) -> [0: bb1, 1: bb4, otherwise: bb5];
     }
 
     bb1: {
-        _4 = move _2;
         _3 = const ();
         goto -> bb3;
     }
 
     bb2: {
         _0 = Poll::<()>::Ready(move _3);
-        discriminant((*(_1.0: &mut {async fn body of a()}))) = 1;
+        discriminant((*_5)) = 1;
         return;
     }
 
diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir
index 109a41d1ef90..96ee37185db1 100644
--- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir
+++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir
@@ -1,7 +1,7 @@
 // MIR for `b::{closure#0}` 0 coroutine_resume
 /* coroutine_layout = CoroutineLayout {
     field_tys: {
-        _0: CoroutineSavedTy {
+        _s0: CoroutineSavedTy {
             ty: Coroutine(
                 DefId(0:5 ~ async_await[ccf8]::a::{closure#0}),
                 [
@@ -9,10 +9,6 @@
                     std::future::ResumeTy,
                     (),
                     (),
-                    CoroutineWitness(
-                        DefId(0:5 ~ async_await[ccf8]::a::{closure#0}),
-                        [],
-                    ),
                     (),
                 ],
             ),
@@ -22,7 +18,7 @@
             },
             ignore_for_traits: false,
         },
-        _1: CoroutineSavedTy {
+        _s1: CoroutineSavedTy {
             ty: Coroutine(
                 DefId(0:5 ~ async_await[ccf8]::a::{closure#0}),
                 [
@@ -30,10 +26,6 @@
                     std::future::ResumeTy,
                     (),
                     (),
-                    CoroutineWitness(
-                        DefId(0:5 ~ async_await[ccf8]::a::{closure#0}),
-                        [],
-                    ),
                     (),
                 ],
             ),
@@ -48,17 +40,17 @@
         Unresumed(0): [],
         Returned (1): [],
         Panicked (2): [],
-        Suspend0 (3): [_0],
-        Suspend1 (4): [_1],
+        Suspend0 (3): [_s0],
+        Suspend1 (4): [_s1],
     },
     storage_conflicts: BitMatrix(2x2) {
-        (_0, _0),
-        (_1, _1),
+        (_s0, _s0),
+        (_s1, _s1),
     },
 } */
 
 fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> Poll<()> {
-    debug _task_context => _38;
+    debug _task_context => _2;
     let mut _0: std::task::Poll<()>;
     let _3: ();
     let mut _4: {async fn body of a()};
@@ -93,17 +85,17 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
     let mut _35: &mut std::task::Context<'_>;
     let mut _36: ();
     let mut _37: ();
-    let mut _38: &mut std::task::Context<'_>;
-    let mut _39: u32;
+    let mut _38: u32;
+    let mut _39: &mut {async fn body of b()};
     scope 1 {
-        debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()});
+        debug __awaitee => (((*_39) as variant#3).0: {async fn body of a()});
         let _17: ();
         scope 2 {
             debug result => _17;
         }
     }
     scope 3 {
-        debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()});
+        debug __awaitee => (((*_39) as variant#4).0: {async fn body of a()});
         let _33: ();
         scope 4 {
             debug result => _33;
@@ -111,12 +103,12 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
     }
 
     bb0: {
-        _39 = discriminant((*(_1.0: &mut {async fn body of b()})));
-        switchInt(move _39) -> [0: bb1, 1: bb29, 3: bb27, 4: bb28, otherwise: bb8];
+        _39 = copy (_1.0: &mut {async fn body of b()});
+        _38 = discriminant((*_39));
+        switchInt(move _38) -> [0: bb1, 1: bb29, 3: bb27, 4: bb28, otherwise: bb8];
     }
 
     bb1: {
-        _38 = move _2;
         StorageLive(_3);
         StorageLive(_4);
         StorageLive(_5);
@@ -131,7 +123,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_5);
         PlaceMention(_4);
         nop;
-        (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _4;
+        (((*_39) as variant#3).0: {async fn body of a()}) = move _4;
         goto -> bb4;
     }
 
@@ -141,7 +133,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageLive(_10);
         StorageLive(_11);
         StorageLive(_12);
-        _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()});
+        _12 = &mut (((*_39) as variant#3).0: {async fn body of a()});
         _11 = &mut (*_12);
         _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable];
     }
@@ -151,7 +143,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageLive(_13);
         StorageLive(_14);
         StorageLive(_15);
-        _15 = copy _38;
+        _15 = copy _2;
         _14 = move _15;
         goto -> bb6;
     }
@@ -188,7 +180,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_4);
         StorageDead(_19);
         StorageDead(_20);
-        discriminant((*(_1.0: &mut {async fn body of b()}))) = 3;
+        discriminant((*_39)) = 3;
         return;
     }
 
@@ -201,12 +193,12 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_12);
         StorageDead(_9);
         StorageDead(_8);
-        drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable];
+        drop((((*_39) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable];
     }
 
     bb11: {
         StorageDead(_20);
-        _38 = move _19;
+        _2 = move _19;
         StorageDead(_19);
         _7 = const ();
         goto -> bb4;
@@ -233,7 +225,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_22);
         PlaceMention(_21);
         nop;
-        (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _21;
+        (((*_39) as variant#4).0: {async fn body of a()}) = move _21;
         goto -> bb16;
     }
 
@@ -243,7 +235,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageLive(_26);
         StorageLive(_27);
         StorageLive(_28);
-        _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()});
+        _28 = &mut (((*_39) as variant#4).0: {async fn body of a()});
         _27 = &mut (*_28);
         _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable];
     }
@@ -253,7 +245,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageLive(_29);
         StorageLive(_30);
         StorageLive(_31);
-        _31 = copy _38;
+        _31 = copy _2;
         _30 = move _31;
         goto -> bb18;
     }
@@ -285,7 +277,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_21);
         StorageDead(_35);
         StorageDead(_36);
-        discriminant((*(_1.0: &mut {async fn body of b()}))) = 4;
+        discriminant((*_39)) = 4;
         return;
     }
 
@@ -298,12 +290,12 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
         StorageDead(_28);
         StorageDead(_25);
         StorageDead(_24);
-        drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable];
+        drop((((*_39) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable];
     }
 
     bb22: {
         StorageDead(_36);
-        _38 = move _35;
+        _2 = move _35;
         StorageDead(_35);
         _7 = const ();
         goto -> bb16;
@@ -321,7 +313,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) ->
 
     bb25: {
         _0 = Poll::<()>::Ready(move _37);
-        discriminant((*(_1.0: &mut {async fn body of b()}))) = 1;
+        discriminant((*_39)) = 1;
         return;
     }
 
diff --git a/tests/mir-opt/building/async_await.rs b/tests/mir-opt/building/async_await.rs
index 6c44570d109c..2d14a1fb0c6f 100644
--- a/tests/mir-opt/building/async_await.rs
+++ b/tests/mir-opt/building/async_await.rs
@@ -4,7 +4,7 @@
 // related to `yield` are `&mut Context`, and its return type is `Poll`.
 
 //@ edition:2018
-//@ compile-flags: -C panic=abort
+//@ compile-flags: -Zmir-opt-level=0 -C panic=abort
 
 #![crate_type = "lib"]
 
diff --git a/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir
new file mode 100644
index 000000000000..b61215dc28cb
--- /dev/null
+++ b/tests/mir-opt/building/coroutine.main-{closure#0}.StateTransform.after.mir
@@ -0,0 +1,193 @@
+// MIR for `main::{closure#0}` after StateTransform
+/* coroutine_layout = CoroutineLayout {
+    field_tys: {
+        _s0: CoroutineSavedTy {
+            ty: std::string::String,
+            source_info: SourceInfo {
+                span: $DIR/coroutine.rs:18:6: 18:9 (#0),
+                scope: scope[0],
+            },
+            ignore_for_traits: false,
+        },
+    },
+    variant_fields: {
+        Unresumed(0): [],
+        Returned (1): [],
+        Panicked (2): [],
+        Suspend0 (3): [_s0],
+        Suspend1 (4): [_s0],
+    },
+    storage_conflicts: BitMatrix(1x1) {
+        (_s0, _s0),
+    },
+} */
+
+fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:18:5: 18:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> {
+    debug arg => (((*_18) as variant#4).0: std::string::String);
+    let mut _0: std::ops::CoroutineState<(&str, std::string::String, &std::panic::Location<'_>), ()>;
+    let _3: std::string::String;
+    let mut _4: (&str, std::string::String, &std::panic::Location<'_>);
+    let mut _5: std::string::String;
+    let mut _6: &std::string::String;
+    let mut _7: &std::panic::Location<'_>;
+    let _8: std::string::String;
+    let mut _9: (&str, std::string::String, &std::panic::Location<'_>);
+    let mut _10: &str;
+    let _11: &str;
+    let mut _12: std::string::String;
+    let mut _13: &std::string::String;
+    let mut _14: &std::panic::Location<'_>;
+    let _15: &std::panic::Location<'_>;
+    let mut _16: ();
+    let mut _17: u32;
+    let mut _18: &mut {coroutine@$DIR/coroutine.rs:18:5: 18:18};
+
+    bb0: {
+        _18 = copy (_1.0: &mut {coroutine@$DIR/coroutine.rs:18:5: 18:18});
+        _17 = discriminant((*_18));
+        switchInt(move _17) -> [0: bb1, 1: bb19, 3: bb17, 4: bb18, otherwise: bb20];
+    }
+
+    bb1: {
+        (((*_18) as variant#4).0: std::string::String) = move _2;
+        StorageLive(_3);
+        StorageLive(_4);
+        StorageLive(_5);
+        StorageLive(_6);
+        _6 = &(((*_18) as variant#4).0: std::string::String);
+        _5 = ::clone(move _6) -> [return: bb2, unwind unreachable];
+    }
+
+    bb2: {
+        StorageDead(_6);
+        StorageLive(_7);
+        _7 = Location::<'_>::caller() -> [return: bb3, unwind unreachable];
+    }
+
+    bb3: {
+        _4 = (const "first", move _5, move _7);
+        StorageDead(_7);
+        goto -> bb4;
+    }
+
+    bb4: {
+        StorageDead(_5);
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Yielded(move _4);
+        StorageDead(_3);
+        StorageDead(_4);
+        discriminant((*_18)) = 3;
+        return;
+    }
+
+    bb5: {
+        goto -> bb6;
+    }
+
+    bb6: {
+        StorageDead(_4);
+        drop(_3) -> [return: bb7, unwind unreachable];
+    }
+
+    bb7: {
+        StorageDead(_3);
+        StorageLive(_8);
+        StorageLive(_9);
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = const "second";
+        _10 = &(*_11);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = &(((*_18) as variant#4).0: std::string::String);
+        _12 = ::clone(move _13) -> [return: bb8, unwind unreachable];
+    }
+
+    bb8: {
+        StorageDead(_13);
+        StorageLive(_14);
+        StorageLive(_15);
+        _15 = Location::<'_>::caller() -> [return: bb9, unwind unreachable];
+    }
+
+    bb9: {
+        _14 = &(*_15);
+        _9 = (move _10, move _12, move _14);
+        StorageDead(_14);
+        goto -> bb10;
+    }
+
+    bb10: {
+        StorageDead(_12);
+        StorageDead(_10);
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Yielded(move _9);
+        StorageDead(_8);
+        StorageDead(_9);
+        StorageDead(_11);
+        StorageDead(_15);
+        discriminant((*_18)) = 4;
+        return;
+    }
+
+    bb11: {
+        goto -> bb12;
+    }
+
+    bb12: {
+        StorageDead(_9);
+        drop(_8) -> [return: bb13, unwind unreachable];
+    }
+
+    bb13: {
+        StorageDead(_15);
+        StorageDead(_11);
+        StorageDead(_8);
+        _16 = const ();
+        drop((((*_18) as variant#4).0: std::string::String)) -> [return: bb14, unwind unreachable];
+    }
+
+    bb14: {
+        goto -> bb16;
+    }
+
+    bb15: {
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Complete(move _16);
+        discriminant((*_18)) = 1;
+        return;
+    }
+
+    bb16: {
+        goto -> bb15;
+    }
+
+    bb17: {
+        StorageLive(_3);
+        StorageLive(_4);
+        _3 = move _2;
+        goto -> bb5;
+    }
+
+    bb18: {
+        StorageLive(_8);
+        StorageLive(_9);
+        StorageLive(_11);
+        StorageLive(_15);
+        _8 = move _2;
+        goto -> bb11;
+    }
+
+    bb19: {
+        assert(const false, "coroutine resumed after completion") -> [success: bb19, unwind unreachable];
+    }
+
+    bb20: {
+        unreachable;
+    }
+}
+
+ALLOC0 (size: 6, align: 1) {
+    73 65 63 6f 6e 64                               │ second
+}
+
+ALLOC1 (size: 5, align: 1) {
+    66 69 72 73 74                                  │ first
+}
diff --git a/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir
new file mode 100644
index 000000000000..aac028a9e6c0
--- /dev/null
+++ b/tests/mir-opt/building/coroutine.main-{closure#1}.StateTransform.after.mir
@@ -0,0 +1,193 @@
+// MIR for `main::{closure#1}` after StateTransform
+/* coroutine_layout = CoroutineLayout {
+    field_tys: {
+        _s0: CoroutineSavedTy {
+            ty: std::string::String,
+            source_info: SourceInfo {
+                span: $DIR/coroutine.rs:25:6: 25:9 (#0),
+                scope: scope[0],
+            },
+            ignore_for_traits: false,
+        },
+    },
+    variant_fields: {
+        Unresumed(0): [],
+        Returned (1): [],
+        Panicked (2): [],
+        Suspend0 (3): [_s0],
+        Suspend1 (4): [_s0],
+    },
+    storage_conflicts: BitMatrix(1x1) {
+        (_s0, _s0),
+    },
+} */
+
+fn main::{closure#1}(_1: Pin<&mut {coroutine@$DIR/coroutine.rs:25:5: 25:18}>, _2: String) -> CoroutineState<(&str, String, &Location<'_>), ()> {
+    debug arg => (((*_18) as variant#4).0: std::string::String);
+    let mut _0: std::ops::CoroutineState<(&str, std::string::String, &std::panic::Location<'_>), ()>;
+    let _3: std::string::String;
+    let mut _4: (&str, std::string::String, &std::panic::Location<'_>);
+    let mut _5: std::string::String;
+    let mut _6: &std::string::String;
+    let mut _7: &std::panic::Location<'_>;
+    let _8: std::string::String;
+    let mut _9: (&str, std::string::String, &std::panic::Location<'_>);
+    let mut _10: &str;
+    let _11: &str;
+    let mut _12: std::string::String;
+    let mut _13: &std::string::String;
+    let mut _14: &std::panic::Location<'_>;
+    let _15: &std::panic::Location<'_>;
+    let mut _16: ();
+    let mut _17: u32;
+    let mut _18: &mut {coroutine@$DIR/coroutine.rs:25:5: 25:18};
+
+    bb0: {
+        _18 = copy (_1.0: &mut {coroutine@$DIR/coroutine.rs:25:5: 25:18});
+        _17 = discriminant((*_18));
+        switchInt(move _17) -> [0: bb1, 1: bb19, 3: bb17, 4: bb18, otherwise: bb20];
+    }
+
+    bb1: {
+        (((*_18) as variant#4).0: std::string::String) = move _2;
+        StorageLive(_3);
+        StorageLive(_4);
+        StorageLive(_5);
+        StorageLive(_6);
+        _6 = &(((*_18) as variant#4).0: std::string::String);
+        _5 = ::clone(move _6) -> [return: bb2, unwind unreachable];
+    }
+
+    bb2: {
+        StorageDead(_6);
+        StorageLive(_7);
+        _7 = Location::<'_>::caller() -> [return: bb3, unwind unreachable];
+    }
+
+    bb3: {
+        _4 = (const "first", move _5, move _7);
+        StorageDead(_7);
+        goto -> bb4;
+    }
+
+    bb4: {
+        StorageDead(_5);
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Yielded(move _4);
+        StorageDead(_3);
+        StorageDead(_4);
+        discriminant((*_18)) = 3;
+        return;
+    }
+
+    bb5: {
+        goto -> bb6;
+    }
+
+    bb6: {
+        StorageDead(_4);
+        drop(_3) -> [return: bb7, unwind unreachable];
+    }
+
+    bb7: {
+        StorageDead(_3);
+        StorageLive(_8);
+        StorageLive(_9);
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = const "second";
+        _10 = &(*_11);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = &(((*_18) as variant#4).0: std::string::String);
+        _12 = ::clone(move _13) -> [return: bb8, unwind unreachable];
+    }
+
+    bb8: {
+        StorageDead(_13);
+        StorageLive(_14);
+        StorageLive(_15);
+        _15 = Location::<'_>::caller() -> [return: bb9, unwind unreachable];
+    }
+
+    bb9: {
+        _14 = &(*_15);
+        _9 = (move _10, move _12, move _14);
+        StorageDead(_14);
+        goto -> bb10;
+    }
+
+    bb10: {
+        StorageDead(_12);
+        StorageDead(_10);
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Yielded(move _9);
+        StorageDead(_8);
+        StorageDead(_9);
+        StorageDead(_11);
+        StorageDead(_15);
+        discriminant((*_18)) = 4;
+        return;
+    }
+
+    bb11: {
+        goto -> bb12;
+    }
+
+    bb12: {
+        StorageDead(_9);
+        drop(_8) -> [return: bb13, unwind unreachable];
+    }
+
+    bb13: {
+        StorageDead(_15);
+        StorageDead(_11);
+        StorageDead(_8);
+        _16 = const ();
+        drop((((*_18) as variant#4).0: std::string::String)) -> [return: bb14, unwind unreachable];
+    }
+
+    bb14: {
+        goto -> bb16;
+    }
+
+    bb15: {
+        _0 = CoroutineState::<(&str, String, &Location<'_>), ()>::Complete(move _16);
+        discriminant((*_18)) = 1;
+        return;
+    }
+
+    bb16: {
+        goto -> bb15;
+    }
+
+    bb17: {
+        StorageLive(_3);
+        StorageLive(_4);
+        _3 = move _2;
+        goto -> bb5;
+    }
+
+    bb18: {
+        StorageLive(_8);
+        StorageLive(_9);
+        StorageLive(_11);
+        StorageLive(_15);
+        _8 = move _2;
+        goto -> bb11;
+    }
+
+    bb19: {
+        assert(const false, "coroutine resumed after completion") -> [success: bb19, unwind unreachable];
+    }
+
+    bb20: {
+        unreachable;
+    }
+}
+
+ALLOC0 (size: 6, align: 1) {
+    73 65 63 6f 6e 64                               │ second
+}
+
+ALLOC1 (size: 5, align: 1) {
+    66 69 72 73 74                                  │ first
+}
diff --git a/tests/mir-opt/building/coroutine.rs b/tests/mir-opt/building/coroutine.rs
new file mode 100644
index 000000000000..6d50c4d90b1a
--- /dev/null
+++ b/tests/mir-opt/building/coroutine.rs
@@ -0,0 +1,29 @@
+// skip-filecheck
+//@ edition:2024
+//@ compile-flags: -Zmir-opt-level=0 -C panic=abort
+
+#![feature(stmt_expr_attributes)]
+#![feature(closure_track_caller)]
+#![feature(coroutine_trait)]
+#![feature(coroutines)]
+
+use std::ops::{Coroutine, CoroutineState};
+use std::panic::Location;
+use std::pin::Pin;
+
+// EMIT_MIR coroutine.main-{closure#0}.StateTransform.after.mir
+// EMIT_MIR coroutine.main-{closure#1}.StateTransform.after.mir
+fn main() {
+    let simple = #[coroutine]
+    |arg: String| {
+        yield ("first", arg.clone(), Location::caller());
+        yield ("second", arg.clone(), Location::caller());
+    };
+
+    let track_caller = #[track_caller]
+    #[coroutine]
+    |arg: String| {
+        yield ("first", arg.clone(), Location::caller());
+        yield ("second", arg.clone(), Location::caller());
+    };
+}
diff --git a/tests/mir-opt/building/custom/arrays.arrays.built.after.mir b/tests/mir-opt/building/custom/arrays.arrays.built.after.mir
index 30d11e31e4d4..f9f24c8eabe4 100644
--- a/tests/mir-opt/building/custom/arrays.arrays.built.after.mir
+++ b/tests/mir-opt/building/custom/arrays.arrays.built.after.mir
@@ -3,12 +3,16 @@
 fn arrays() -> usize {
     let mut _0: usize;
     let mut _1: [i32; C];
-    let mut _2: usize;
+    let mut _2: *const [i32; C];
+    let mut _3: *const [i32];
+    let mut _4: usize;
 
     bb0: {
         _1 = [const 5_i32; C];
-        _2 = Len(_1);
-        _0 = copy _2;
+        _2 = &raw const _1;
+        _3 = copy _2 as *const [i32] (PointerCoercion(Unsize, AsCast));
+        _4 = PtrMetadata(copy _3);
+        _0 = copy _4;
         return;
     }
 }
diff --git a/tests/mir-opt/building/custom/arrays.rs b/tests/mir-opt/building/custom/arrays.rs
index 4bd6f93e1134..e9cdded4a0e6 100644
--- a/tests/mir-opt/building/custom/arrays.rs
+++ b/tests/mir-opt/building/custom/arrays.rs
@@ -10,7 +10,9 @@ fn arrays() -> usize {
     mir! {
         {
             let x = [5_i32; C];
-            let c = Len(x);
+            let y = &raw const x;
+            let z = CastUnsize::<_, *const [i32]>(y);
+            let c = PtrMetadata(z);
             RET = c;
             Return()
         }
diff --git a/tests/mir-opt/building/custom/enums.rs b/tests/mir-opt/building/custom/enums.rs
index 88ec228986ab..6fad373203a6 100644
--- a/tests/mir-opt/building/custom/enums.rs
+++ b/tests/mir-opt/building/custom/enums.rs
@@ -88,7 +88,6 @@ fn switch_option_repr(option: Bool) -> bool {
 fn set_discr(option: &mut Option<()>) {
     mir! {
         {
-            Deinit(*option);
             SetDiscriminant(*option, 0);
             Return()
         }
diff --git a/tests/mir-opt/building/custom/enums.set_discr.built.after.mir b/tests/mir-opt/building/custom/enums.set_discr.built.after.mir
index 8cc66e7e50d6..d9b46dff43aa 100644
--- a/tests/mir-opt/building/custom/enums.set_discr.built.after.mir
+++ b/tests/mir-opt/building/custom/enums.set_discr.built.after.mir
@@ -4,7 +4,6 @@ fn set_discr(_1: &mut Option<()>) -> () {
     let mut _0: ();
 
     bb0: {
-        Deinit((*_1));
         discriminant((*_1)) = 0;
         return;
     }
diff --git a/tests/mir-opt/building/custom/projections.copy_for_deref.built.after.mir b/tests/mir-opt/building/custom/projections.copy_for_deref.built.after.mir
deleted file mode 100644
index b28e96f10eae..000000000000
--- a/tests/mir-opt/building/custom/projections.copy_for_deref.built.after.mir
+++ /dev/null
@@ -1,12 +0,0 @@
-// MIR for `copy_for_deref` after built
-
-fn copy_for_deref(_1: (&i32, i32)) -> i32 {
-    let mut _0: i32;
-    let mut _2: &i32;
-
-    bb0: {
-        _2 = deref_copy (_1.0: &i32);
-        _0 = copy (*_2);
-        return;
-    }
-}
diff --git a/tests/mir-opt/building/custom/projections.rs b/tests/mir-opt/building/custom/projections.rs
index 0250b9b84b62..e59eebd9952d 100644
--- a/tests/mir-opt/building/custom/projections.rs
+++ b/tests/mir-opt/building/custom/projections.rs
@@ -79,19 +79,6 @@ fn simple_index(a: [i32; 10], b: &[i32]) -> i32 {
     }
 }
 
-// EMIT_MIR projections.copy_for_deref.built.after.mir
-#[custom_mir(dialect = "runtime", phase = "initial")]
-fn copy_for_deref(x: (&i32, i32)) -> i32 {
-    mir! {
-        let temp: &i32;
-        {
-            temp = CopyForDeref(x.0);
-            RET = *temp;
-            Return()
-        }
-    }
-}
-
 fn main() {
     assert_eq!(unions(U { a: 5 }), 5);
     assert_eq!(tuples((5, 6)), (5, 6));
@@ -103,7 +90,4 @@ fn main() {
     assert_eq!(o, Some(10));
 
     assert_eq!(simple_index([0; 10], &[0; 10]), 0);
-
-    let one = 1;
-    assert_eq!(copy_for_deref((&one, one)), 1);
 }
diff --git a/tests/mir-opt/building/custom/terminators.tail_call.built.after.mir b/tests/mir-opt/building/custom/terminators.tail_call.built.after.mir
index ab3925dae1c3..feec68d3b0d1 100644
--- a/tests/mir-opt/building/custom/terminators.tail_call.built.after.mir
+++ b/tests/mir-opt/building/custom/terminators.tail_call.built.after.mir
@@ -6,6 +6,6 @@ fn tail_call(_1: i32) -> i32 {
 
     bb0: {
         _2 = Add(copy _1, const 42_i32);
-        tailcall ident::(Spanned { node: copy _2, span: $DIR/terminators.rs:32:28: 32:29 (#0) });
+        tailcall ident::(copy _2);
     }
 }
diff --git a/tests/mir-opt/building/dump_mir_cycle.rs b/tests/mir-opt/building/dump_mir_cycle.rs
index 8e13420aed79..ab0f2ea6db86 100644
--- a/tests/mir-opt/building/dump_mir_cycle.rs
+++ b/tests/mir-opt/building/dump_mir_cycle.rs
@@ -1,3 +1,5 @@
+//@ compile-flags: -Zmir-opt-level=0
+
 #[derive(Debug)]
 pub struct Thing {
     pub next: &'static Thing,
diff --git a/tests/mir-opt/building/enum_cast.rs b/tests/mir-opt/building/enum_cast.rs
index eaf5537e0ab4..07025f572ab2 100644
--- a/tests/mir-opt/building/enum_cast.rs
+++ b/tests/mir-opt/building/enum_cast.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // EMIT_MIR enum_cast.foo.built.after.mir
 // EMIT_MIR enum_cast.bar.built.after.mir
diff --git a/tests/mir-opt/building/eq_never_type.rs b/tests/mir-opt/building/eq_never_type.rs
index 90e2e6975353..486e2f9fb8df 100644
--- a/tests/mir-opt/building/eq_never_type.rs
+++ b/tests/mir-opt/building/eq_never_type.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 #![feature(never_type)]
 #![allow(unreachable_code)]
diff --git a/tests/mir-opt/building/index_array_and_slice.rs b/tests/mir-opt/building/index_array_and_slice.rs
index f91b37567f79..47d8255ef57f 100644
--- a/tests/mir-opt/building/index_array_and_slice.rs
+++ b/tests/mir-opt/building/index_array_and_slice.rs
@@ -1,36 +1,42 @@
-//@ compile-flags: -C opt-level=0
+//@ compile-flags: -Zmir-opt-level=0 -C opt-level=0
 
 // EMIT_MIR index_array_and_slice.index_array.built.after.mir
 fn index_array(array: &[i32; 7], index: usize) -> &i32 {
     // CHECK: bb0:
-    // CHECK: [[LT:_.+]] = Lt(copy _2, const 7_usize);
-    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", const 7_usize, copy _2) -> [success: bb1, unwind
+    // CHECK: _3 = copy _2;
+    // CHECK: [[LT:_.+]] = Lt(copy _3, const 7_usize);
+    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", const 7_usize, copy _3) -> [success: bb1, unwind
 
     // CHECK: bb1:
-    // CHECK: _0 = &(*_1)[_2];
+    // CHECK: _5 = &(*_1)[_3];
+    // CHECK: _0 = &(*_5);
     &array[index]
 }
 
 // EMIT_MIR index_array_and_slice.index_const_generic_array.built.after.mir
 fn index_const_generic_array(array: &[i32; N], index: usize) -> &i32 {
     // CHECK: bb0:
-    // CHECK: [[LT:_.+]] = Lt(copy _2, const N);
-    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", const N, copy _2) -> [success: bb1, unwind
+    // CHECK: _3 = copy _2;
+    // CHECK: [[LT:_.+]] = Lt(copy _3, const N);
+    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", const N, copy _3) -> [success: bb1, unwind
 
     // CHECK: bb1:
-    // CHECK: _0 = &(*_1)[_2];
+    // CHECK: _5 = &(*_1)[_3];
+    // CHECK: _0 = &(*_5);
     &array[index]
 }
 
 // EMIT_MIR index_array_and_slice.index_slice.built.after.mir
 fn index_slice(slice: &[i32], index: usize) -> &i32 {
     // CHECK: bb0:
+    // CHECK: _3 = copy _2;
     // CHECK: [[LEN:_.+]] = PtrMetadata(copy _1);
-    // CHECK: [[LT:_.+]] = Lt(copy _2, copy [[LEN]]);
-    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _2) -> [success: bb1,
+    // CHECK: [[LT:_.+]] = Lt(copy _3, copy [[LEN]]);
+    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _3) -> [success: bb1,
 
     // CHECK: bb1:
-    // CHECK: _0 = &(*_1)[_2];
+    // CHECK: _6 = &(*_1)[_3];
+    // CHECK: _0 = &(*_6);
     &slice[index]
 }
 
@@ -40,12 +46,15 @@ fn index_mut_slice(slice: &mut [i32], index: usize) -> &i32 {
     // This cannot `copy _1` in the *built* MIR, only in the *runtime* MIR.
 
     // CHECK: bb0:
-    // CHECK: [[LEN:_.+]] = PtrMetadata(copy _1);
-    // CHECK: [[LT:_.+]] = Lt(copy _2, copy [[LEN]]);
-    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _2) -> [success: bb1,
+    // CHECK: _3 = copy _2;
+    // CHECK: _4 = &raw const (fake) (*_1);
+    // CHECK: [[LEN:_.+]] = PtrMetadata(move _4);
+    // CHECK: [[LT:_.+]] = Lt(copy _3, copy [[LEN]]);
+    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _3) -> [success: bb1,
 
     // CHECK: bb1:
-    // CHECK: _0 = &(*_1)[_2];
+    // CHECK: _7 = &(*_1)[_3];
+    // CHECK: _0 = &(*_7);
     &slice[index]
 }
 
@@ -54,13 +63,15 @@ struct WithSliceTail(f64, [i32]);
 // EMIT_MIR index_array_and_slice.index_custom.built.after.mir
 fn index_custom(custom: &WithSliceTail, index: usize) -> &i32 {
     // CHECK: bb0:
+    // CHECK: _3 = copy _2;
     // CHECK: [[PTR:_.+]] = &raw const (fake) ((*_1).1: [i32]);
     // CHECK: [[LEN:_.+]] = PtrMetadata(move [[PTR]]);
-    // CHECK: [[LT:_.+]] = Lt(copy _2, copy [[LEN]]);
-    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _2) -> [success: bb1,
+    // CHECK: [[LT:_.+]] = Lt(copy _3, copy [[LEN]]);
+    // CHECK: assert(move [[LT]], "index out of bounds{{.+}}", move [[LEN]], copy _3) -> [success: bb1,
 
     // CHECK: bb1:
-    // CHECK: _0 = &((*_1).1: [i32])[_2];
+    // CHECK: _7 = &((*_1).1: [i32])[_3];
+    // CHECK: _0 = &(*_7);
     &custom.1[index]
 }
 
diff --git a/tests/mir-opt/building/issue_101867.main.built.after.mir b/tests/mir-opt/building/issue_101867.main.built.after.mir
index e59b23fdd20b..fa35658a16d4 100644
--- a/tests/mir-opt/building/issue_101867.main.built.after.mir
+++ b/tests/mir-opt/building/issue_101867.main.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `main` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty(std::option::Option), max_universe: U0, variables: [] }, span: $DIR/issue_101867.rs:4:12: 4:22, inferred_ty: std::option::Option
-| 1: user_ty: Canonical { value: Ty(std::option::Option), max_universe: U0, variables: [] }, span: $DIR/issue_101867.rs:4:12: 4:22, inferred_ty: std::option::Option
+| 0: user_ty: Canonical { value: Ty(std::option::Option), max_universe: U0, variables: [] }, span: $DIR/issue_101867.rs:5:12: 5:22, inferred_ty: std::option::Option
+| 1: user_ty: Canonical { value: Ty(std::option::Option), max_universe: U0, variables: [] }, span: $DIR/issue_101867.rs:5:12: 5:22, inferred_ty: std::option::Option
 |
 fn main() -> () {
     let mut _0: ();
@@ -24,7 +24,6 @@ fn main() -> () {
         _1 = Option::::Some(const 1_u8);
         FakeRead(ForLet(None), _1);
         AscribeUserType(_1, o, UserTypeProjection { base: UserType(1), projs: [] });
-        StorageLive(_5);
         PlaceMention(_1);
         _6 = discriminant(_1);
         switchInt(move _6) -> [1: bb4, otherwise: bb3];
@@ -55,6 +54,7 @@ fn main() -> () {
     }
 
     bb6: {
+        StorageLive(_5);
         _5 = copy ((_1 as Some).0: u8);
         _0 = const ();
         StorageDead(_5);
@@ -63,7 +63,6 @@ fn main() -> () {
     }
 
     bb7: {
-        StorageDead(_5);
         goto -> bb1;
     }
 
diff --git a/tests/mir-opt/building/issue_101867.rs b/tests/mir-opt/building/issue_101867.rs
index f8a531e8982d..a00bc13b3fe5 100644
--- a/tests/mir-opt/building/issue_101867.rs
+++ b/tests/mir-opt/building/issue_101867.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // EMIT_MIR issue_101867.main.built.after.mir
 fn main() {
diff --git a/tests/mir-opt/building/issue_110508.rs b/tests/mir-opt/building/issue_110508.rs
index e597cd5d06b3..aa8a7b1d08f4 100644
--- a/tests/mir-opt/building/issue_110508.rs
+++ b/tests/mir-opt/building/issue_110508.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // EMIT_MIR issue_110508.{impl#0}-BAR.built.after.mir
 // EMIT_MIR issue_110508.{impl#0}-SELF_BAR.built.after.mir
diff --git a/tests/mir-opt/building/issue_110508.{impl#0}-BAR.built.after.mir b/tests/mir-opt/building/issue_110508.{impl#0}-BAR.built.after.mir
index c3d28fae5183..0cefb9c8fd1b 100644
--- a/tests/mir-opt/building/issue_110508.{impl#0}-BAR.built.after.mir
+++ b/tests/mir-opt/building/issue_110508.{impl#0}-BAR.built.after.mir
@@ -1,6 +1,6 @@
-// MIR for `::BAR` after built
+// MIR for `::BAR` after built
 
-const ::BAR: Foo = {
+const ::BAR: Foo = {
     let mut _0: Foo;
     let mut _1: ();
 
diff --git a/tests/mir-opt/building/issue_110508.{impl#0}-SELF_BAR.built.after.mir b/tests/mir-opt/building/issue_110508.{impl#0}-SELF_BAR.built.after.mir
index 177518c30af7..0e0e91493851 100644
--- a/tests/mir-opt/building/issue_110508.{impl#0}-SELF_BAR.built.after.mir
+++ b/tests/mir-opt/building/issue_110508.{impl#0}-SELF_BAR.built.after.mir
@@ -1,6 +1,6 @@
-// MIR for `::SELF_BAR` after built
+// MIR for `::SELF_BAR` after built
 
-const ::SELF_BAR: Foo = {
+const ::SELF_BAR: Foo = {
     let mut _0: Foo;
     let mut _1: ();
 
diff --git a/tests/mir-opt/building/issue_49232.rs b/tests/mir-opt/building/issue_49232.rs
index ac06e02778f0..f17e3e955a41 100644
--- a/tests/mir-opt/building/issue_49232.rs
+++ b/tests/mir-opt/building/issue_49232.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // We must mark a variable whose initialization fails due to an
 // abort statement as StorageDead.
diff --git a/tests/mir-opt/building/logical_or_in_conditional.rs b/tests/mir-opt/building/logical_or_in_conditional.rs
index 249ccf728044..d1c1ea2896aa 100644
--- a/tests/mir-opt/building/logical_or_in_conditional.rs
+++ b/tests/mir-opt/building/logical_or_in_conditional.rs
@@ -1,5 +1,5 @@
 // skip-filecheck
-//@ compile-flags: -Z validate-mir
+//@ compile-flags: -Zmir-opt-level=0 -Z validate-mir
 //@ edition: 2024
 struct Droppy(u8);
 impl Drop for Droppy {
diff --git a/tests/mir-opt/building/loop_match_diverges.break_to_block_unit.built.after.mir b/tests/mir-opt/building/loop_match_diverges.break_to_block_unit.built.after.mir
new file mode 100644
index 000000000000..6d779e46146c
--- /dev/null
+++ b/tests/mir-opt/building/loop_match_diverges.break_to_block_unit.built.after.mir
@@ -0,0 +1,63 @@
+// MIR for `break_to_block_unit` after built
+
+fn break_to_block_unit() -> u8 {
+    let mut _0: u8;
+    let mut _1: i32;
+    let mut _2: !;
+    scope 1 {
+        debug state => _1;
+    }
+
+    bb0: {
+        StorageLive(_1);
+        _1 = const 0_i32;
+        FakeRead(ForLet(None), _1);
+        StorageLive(_2);
+        goto -> bb1;
+    }
+
+    bb1: {
+        falseUnwind -> [real: bb2, unwind: bb10];
+    }
+
+    bb2: {
+        PlaceMention(_1);
+        _1 = const 2_i32;
+        goto -> bb5;
+    }
+
+    bb3: {
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
+    }
+
+    bb4: {
+        goto -> bb6;
+    }
+
+    bb5: {
+        goto -> bb6;
+    }
+
+    bb6: {
+        goto -> bb7;
+    }
+
+    bb7: {
+        goto -> bb1;
+    }
+
+    bb8: {
+        unreachable;
+    }
+
+    bb9: {
+        StorageDead(_2);
+        StorageDead(_1);
+        return;
+    }
+
+    bb10 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/loop_match_diverges.infinite_a.built.after.mir b/tests/mir-opt/building/loop_match_diverges.infinite_a.built.after.mir
new file mode 100644
index 000000000000..e3766744790b
--- /dev/null
+++ b/tests/mir-opt/building/loop_match_diverges.infinite_a.built.after.mir
@@ -0,0 +1,51 @@
+// MIR for `infinite_a` after built
+
+fn infinite_a(_1: u8) -> () {
+    debug state => _1;
+    let mut _0: ();
+    let mut _2: !;
+    let _3: u8;
+    scope 1 {
+        debug a => _3;
+    }
+
+    bb0: {
+        StorageLive(_2);
+        goto -> bb1;
+    }
+
+    bb1: {
+        falseUnwind -> [real: bb2, unwind: bb7];
+    }
+
+    bb2: {
+        PlaceMention(_1);
+        StorageLive(_3);
+        _3 = copy _1;
+        _1 = copy _3;
+        StorageDead(_3);
+        goto -> bb4;
+    }
+
+    bb3: {
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
+    }
+
+    bb4: {
+        goto -> bb1;
+    }
+
+    bb5: {
+        unreachable;
+    }
+
+    bb6: {
+        StorageDead(_2);
+        return;
+    }
+
+    bb7 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/loop_match_diverges.rs b/tests/mir-opt/building/loop_match_diverges.rs
new file mode 100644
index 000000000000..774e195c33c9
--- /dev/null
+++ b/tests/mir-opt/building/loop_match_diverges.rs
@@ -0,0 +1,68 @@
+// skip-filecheck
+#![allow(incomplete_features)]
+#![feature(loop_match)]
+#![crate_type = "lib"]
+
+// Test that a #[loop_match] without an explicit break from the loop generates valid MIR.
+
+enum State {
+    A,
+    B,
+    C,
+}
+
+// EMIT_MIR loop_match_diverges.simple.built.after.mir
+fn simple(mut state: State) -> State {
+    #[loop_match]
+    'a: loop {
+        state = 'blk: {
+            match state {
+                State::A => {
+                    #[const_continue]
+                    break 'blk State::B;
+                }
+                State::B => {
+                    if true {
+                        #[const_continue]
+                        break 'blk State::C;
+                    } else {
+                        #[const_continue]
+                        break 'blk State::A;
+                    }
+                }
+                State::C => break 'a,
+            }
+        };
+    }
+
+    state
+}
+
+// EMIT_MIR loop_match_diverges.break_to_block_unit.built.after.mir
+#[unsafe(no_mangle)]
+fn break_to_block_unit() -> u8 {
+    let mut state = 0;
+    #[loop_match]
+    loop {
+        state = 'blk: {
+            match state {
+                _ => 'b: {
+                    break 'b 2;
+                }
+            }
+        }
+    }
+}
+
+// EMIT_MIR loop_match_diverges.infinite_a.built.after.mir
+#[unsafe(no_mangle)]
+fn infinite_a(mut state: u8) {
+    #[loop_match]
+    loop {
+        state = 'blk: {
+            match state {
+                a => a,
+            }
+        }
+    }
+}
diff --git a/tests/mir-opt/building/loop_match_diverges.simple.built.after.mir b/tests/mir-opt/building/loop_match_diverges.simple.built.after.mir
new file mode 100644
index 000000000000..26476ad77627
--- /dev/null
+++ b/tests/mir-opt/building/loop_match_diverges.simple.built.after.mir
@@ -0,0 +1,187 @@
+// MIR for `simple` after built
+
+fn simple(_1: State) -> State {
+    debug state => _1;
+    let mut _0: State;
+    let _2: ();
+    let mut _3: isize;
+    let mut _4: !;
+    let mut _5: isize;
+    let mut _6: bool;
+    let mut _7: !;
+    let mut _8: isize;
+    let mut _9: !;
+    let mut _10: isize;
+    let mut _11: !;
+
+    bb0: {
+        StorageLive(_2);
+        goto -> bb1;
+    }
+
+    bb1: {
+        falseUnwind -> [real: bb2, unwind: bb37];
+    }
+
+    bb2: {
+        PlaceMention(_1);
+        _3 = discriminant(_1);
+        switchInt(move _3) -> [0: bb4, 1: bb6, 2: bb8, otherwise: bb3];
+    }
+
+    bb3: {
+        FakeRead(ForMatchedPlace(None), _1);
+        unreachable;
+    }
+
+    bb4: {
+        falseEdge -> [real: bb11, imaginary: bb6];
+    }
+
+    bb5: {
+        goto -> bb3;
+    }
+
+    bb6: {
+        falseEdge -> [real: bb10, imaginary: bb8];
+    }
+
+    bb7: {
+        goto -> bb3;
+    }
+
+    bb8: {
+        _2 = const ();
+        goto -> bb36;
+    }
+
+    bb9: {
+        goto -> bb3;
+    }
+
+    bb10: {
+        StorageLive(_6);
+        _6 = const true;
+        switchInt(move _6) -> [0: bb17, otherwise: bb16];
+    }
+
+    bb11: {
+        _1 = State::B;
+        _5 = discriminant(_1);
+        falseEdge -> [real: bb12, imaginary: bb13];
+    }
+
+    bb12: {
+        goto -> bb10;
+    }
+
+    bb13: {
+        goto -> bb34;
+    }
+
+    bb14: {
+        unreachable;
+    }
+
+    bb15: {
+        goto -> bb32;
+    }
+
+    bb16: {
+        _1 = State::C;
+        _8 = discriminant(_1);
+        falseEdge -> [real: bb18, imaginary: bb19];
+    }
+
+    bb17: {
+        goto -> bb23;
+    }
+
+    bb18: {
+        goto -> bb20;
+    }
+
+    bb19: {
+        goto -> bb33;
+    }
+
+    bb20: {
+        StorageDead(_6);
+        goto -> bb8;
+    }
+
+    bb21: {
+        unreachable;
+    }
+
+    bb22: {
+        goto -> bb29;
+    }
+
+    bb23: {
+        _1 = State::A;
+        _10 = discriminant(_1);
+        falseEdge -> [real: bb24, imaginary: bb25];
+    }
+
+    bb24: {
+        goto -> bb26;
+    }
+
+    bb25: {
+        goto -> bb33;
+    }
+
+    bb26: {
+        StorageDead(_6);
+        goto -> bb11;
+    }
+
+    bb27: {
+        unreachable;
+    }
+
+    bb28: {
+        goto -> bb29;
+    }
+
+    bb29: {
+        StorageDead(_6);
+        goto -> bb32;
+    }
+
+    bb30: {
+        unreachable;
+    }
+
+    bb31: {
+        goto -> bb32;
+    }
+
+    bb32: {
+        goto -> bb35;
+    }
+
+    bb33: {
+        StorageDead(_6);
+        goto -> bb34;
+    }
+
+    bb34: {
+        goto -> bb35;
+    }
+
+    bb35: {
+        goto -> bb1;
+    }
+
+    bb36: {
+        StorageDead(_2);
+        _0 = move _1;
+        return;
+    }
+
+    bb37 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-abort.mir b/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-abort.mir
new file mode 100644
index 000000000000..8d16f074b1e3
--- /dev/null
+++ b/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-abort.mir
@@ -0,0 +1,155 @@
+// MIR for `const_array_len` after built
+
+fn const_array_len(_1: [T; 5]) -> () {
+    debug x => _1;
+    let mut _0: ();
+    let _6: ();
+    let mut _7: T;
+    let _8: ();
+    let mut _9: T;
+    let _10: ();
+    let mut _11: [T; 2];
+    let _12: ();
+    let mut _13: T;
+    scope 1 {
+        debug a => _2;
+        debug b => _3;
+        debug rest => _4;
+        debug e => _5;
+        let _2: T;
+        let _3: T;
+        let _4: [T; 2];
+        let _5: T;
+    }
+
+    bb0: {
+        PlaceMention(_1);
+        falseEdge -> [real: bb2, imaginary: bb1];
+    }
+
+    bb1: {
+        goto -> bb7;
+    }
+
+    bb2: {
+        StorageLive(_2);
+        _2 = move _1[0 of 5];
+        StorageLive(_3);
+        _3 = move _1[1 of 5];
+        StorageLive(_4);
+        _4 = move _1[2..4];
+        StorageLive(_5);
+        _5 = move _1[4 of 5];
+        StorageLive(_6);
+        StorageLive(_7);
+        _7 = move _2;
+        _6 = opaque::(move _7) -> [return: bb3, unwind: bb17];
+    }
+
+    bb3: {
+        StorageDead(_7);
+        StorageDead(_6);
+        StorageLive(_8);
+        StorageLive(_9);
+        _9 = move _3;
+        _8 = opaque::(move _9) -> [return: bb4, unwind: bb16];
+    }
+
+    bb4: {
+        StorageDead(_9);
+        StorageDead(_8);
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = move _4;
+        _10 = opaque::<[T; 2]>(move _11) -> [return: bb5, unwind: bb15];
+    }
+
+    bb5: {
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = move _5;
+        _12 = opaque::(move _13) -> [return: bb6, unwind: bb14];
+    }
+
+    bb6: {
+        StorageDead(_13);
+        StorageDead(_12);
+        _0 = const ();
+        drop(_5) -> [return: bb8, unwind: bb19];
+    }
+
+    bb7: {
+        _0 = const ();
+        goto -> bb12;
+    }
+
+    bb8: {
+        StorageDead(_5);
+        drop(_4) -> [return: bb9, unwind: bb20];
+    }
+
+    bb9: {
+        StorageDead(_4);
+        drop(_3) -> [return: bb10, unwind: bb21];
+    }
+
+    bb10: {
+        StorageDead(_3);
+        drop(_2) -> [return: bb11, unwind: bb22];
+    }
+
+    bb11: {
+        StorageDead(_2);
+        goto -> bb12;
+    }
+
+    bb12: {
+        drop(_1) -> [return: bb13, unwind: bb23];
+    }
+
+    bb13: {
+        return;
+    }
+
+    bb14 (cleanup): {
+        drop(_13) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb15 (cleanup): {
+        drop(_11) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb16 (cleanup): {
+        drop(_9) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb17 (cleanup): {
+        drop(_7) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb18 (cleanup): {
+        drop(_5) -> [return: bb19, unwind terminate(cleanup)];
+    }
+
+    bb19 (cleanup): {
+        drop(_4) -> [return: bb20, unwind terminate(cleanup)];
+    }
+
+    bb20 (cleanup): {
+        drop(_3) -> [return: bb21, unwind terminate(cleanup)];
+    }
+
+    bb21 (cleanup): {
+        drop(_2) -> [return: bb22, unwind terminate(cleanup)];
+    }
+
+    bb22 (cleanup): {
+        drop(_1) -> [return: bb23, unwind terminate(cleanup)];
+    }
+
+    bb23 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-unwind.mir b/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-unwind.mir
new file mode 100644
index 000000000000..8d16f074b1e3
--- /dev/null
+++ b/tests/mir-opt/building/match/array_len.const_array_len.built.after.panic-unwind.mir
@@ -0,0 +1,155 @@
+// MIR for `const_array_len` after built
+
+fn const_array_len(_1: [T; 5]) -> () {
+    debug x => _1;
+    let mut _0: ();
+    let _6: ();
+    let mut _7: T;
+    let _8: ();
+    let mut _9: T;
+    let _10: ();
+    let mut _11: [T; 2];
+    let _12: ();
+    let mut _13: T;
+    scope 1 {
+        debug a => _2;
+        debug b => _3;
+        debug rest => _4;
+        debug e => _5;
+        let _2: T;
+        let _3: T;
+        let _4: [T; 2];
+        let _5: T;
+    }
+
+    bb0: {
+        PlaceMention(_1);
+        falseEdge -> [real: bb2, imaginary: bb1];
+    }
+
+    bb1: {
+        goto -> bb7;
+    }
+
+    bb2: {
+        StorageLive(_2);
+        _2 = move _1[0 of 5];
+        StorageLive(_3);
+        _3 = move _1[1 of 5];
+        StorageLive(_4);
+        _4 = move _1[2..4];
+        StorageLive(_5);
+        _5 = move _1[4 of 5];
+        StorageLive(_6);
+        StorageLive(_7);
+        _7 = move _2;
+        _6 = opaque::(move _7) -> [return: bb3, unwind: bb17];
+    }
+
+    bb3: {
+        StorageDead(_7);
+        StorageDead(_6);
+        StorageLive(_8);
+        StorageLive(_9);
+        _9 = move _3;
+        _8 = opaque::(move _9) -> [return: bb4, unwind: bb16];
+    }
+
+    bb4: {
+        StorageDead(_9);
+        StorageDead(_8);
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = move _4;
+        _10 = opaque::<[T; 2]>(move _11) -> [return: bb5, unwind: bb15];
+    }
+
+    bb5: {
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = move _5;
+        _12 = opaque::(move _13) -> [return: bb6, unwind: bb14];
+    }
+
+    bb6: {
+        StorageDead(_13);
+        StorageDead(_12);
+        _0 = const ();
+        drop(_5) -> [return: bb8, unwind: bb19];
+    }
+
+    bb7: {
+        _0 = const ();
+        goto -> bb12;
+    }
+
+    bb8: {
+        StorageDead(_5);
+        drop(_4) -> [return: bb9, unwind: bb20];
+    }
+
+    bb9: {
+        StorageDead(_4);
+        drop(_3) -> [return: bb10, unwind: bb21];
+    }
+
+    bb10: {
+        StorageDead(_3);
+        drop(_2) -> [return: bb11, unwind: bb22];
+    }
+
+    bb11: {
+        StorageDead(_2);
+        goto -> bb12;
+    }
+
+    bb12: {
+        drop(_1) -> [return: bb13, unwind: bb23];
+    }
+
+    bb13: {
+        return;
+    }
+
+    bb14 (cleanup): {
+        drop(_13) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb15 (cleanup): {
+        drop(_11) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb16 (cleanup): {
+        drop(_9) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb17 (cleanup): {
+        drop(_7) -> [return: bb18, unwind terminate(cleanup)];
+    }
+
+    bb18 (cleanup): {
+        drop(_5) -> [return: bb19, unwind terminate(cleanup)];
+    }
+
+    bb19 (cleanup): {
+        drop(_4) -> [return: bb20, unwind terminate(cleanup)];
+    }
+
+    bb20 (cleanup): {
+        drop(_3) -> [return: bb21, unwind terminate(cleanup)];
+    }
+
+    bb21 (cleanup): {
+        drop(_2) -> [return: bb22, unwind terminate(cleanup)];
+    }
+
+    bb22 (cleanup): {
+        drop(_1) -> [return: bb23, unwind terminate(cleanup)];
+    }
+
+    bb23 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/match/array_len.rs b/tests/mir-opt/building/match/array_len.rs
new file mode 100644
index 000000000000..0d889ada9b60
--- /dev/null
+++ b/tests/mir-opt/building/match/array_len.rs
@@ -0,0 +1,31 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Zmir-opt-level=0
+
+fn opaque(x: T) {}
+
+// EMIT_MIR array_len.const_array_len.built.after.mir
+fn const_array_len(x: [T; 5]) {
+    // CHECK-LABEL: fn const_array_len(
+    // CHECK-NOT: Len
+    // CHECK-NOT: PtrMetadata
+    // CHECK: = const 5_usize;
+    if let [a, b, rest @ .., e] = x {
+        opaque(a);
+        opaque(b);
+        opaque(rest);
+        opaque(e);
+    }
+}
+
+// EMIT_MIR array_len.slice_len.built.after.mir
+fn slice_len(x: &[T]) {
+    // CHECK-LABEL: fn slice_len(
+    // CHECK-NOT: Len
+    // CHECK: = PtrMetadata(copy _1);
+    if let [a, b, rest @ .., e] = x {
+        opaque(a);
+        opaque(b);
+        opaque(rest);
+        opaque(e);
+    }
+}
diff --git a/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-abort.mir b/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-abort.mir
new file mode 100644
index 000000000000..d73f5a1b6f71
--- /dev/null
+++ b/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-abort.mir
@@ -0,0 +1,115 @@
+// MIR for `slice_len` after built
+
+fn slice_len(_1: &[T]) -> () {
+    debug x => _1;
+    let mut _0: ();
+    let mut _2: usize;
+    let mut _3: usize;
+    let mut _4: usize;
+    let mut _5: bool;
+    let _10: ();
+    let mut _11: &T;
+    let _12: ();
+    let mut _13: &T;
+    let _14: ();
+    let mut _15: &[T];
+    let _16: ();
+    let mut _17: &T;
+    scope 1 {
+        debug a => _6;
+        debug b => _7;
+        debug rest => _8;
+        debug e => _9;
+        let _6: &T;
+        let _7: &T;
+        let _8: &[T];
+        let _9: &T;
+    }
+
+    bb0: {
+        PlaceMention(_1);
+        _3 = PtrMetadata(copy _1);
+        _2 = move _3;
+        _4 = const 3_usize;
+        _5 = Ge(move _2, move _4);
+        switchInt(move _5) -> [0: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        goto -> bb9;
+    }
+
+    bb2: {
+        falseEdge -> [real: bb4, imaginary: bb1];
+    }
+
+    bb3: {
+        goto -> bb1;
+    }
+
+    bb4: {
+        StorageLive(_6);
+        _6 = &(*_1)[0 of 3];
+        StorageLive(_7);
+        _7 = &(*_1)[1 of 3];
+        StorageLive(_8);
+        _8 = &(*_1)[2:-1];
+        StorageLive(_9);
+        _9 = &(*_1)[-1 of 3];
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = copy _6;
+        _10 = opaque::<&T>(move _11) -> [return: bb5, unwind: bb11];
+    }
+
+    bb5: {
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = copy _7;
+        _12 = opaque::<&T>(move _13) -> [return: bb6, unwind: bb11];
+    }
+
+    bb6: {
+        StorageDead(_13);
+        StorageDead(_12);
+        StorageLive(_14);
+        StorageLive(_15);
+        _15 = copy _8;
+        _14 = opaque::<&[T]>(move _15) -> [return: bb7, unwind: bb11];
+    }
+
+    bb7: {
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_16);
+        StorageLive(_17);
+        _17 = copy _9;
+        _16 = opaque::<&T>(move _17) -> [return: bb8, unwind: bb11];
+    }
+
+    bb8: {
+        StorageDead(_17);
+        StorageDead(_16);
+        _0 = const ();
+        StorageDead(_9);
+        StorageDead(_8);
+        StorageDead(_7);
+        StorageDead(_6);
+        goto -> bb10;
+    }
+
+    bb9: {
+        _0 = const ();
+        goto -> bb10;
+    }
+
+    bb10: {
+        return;
+    }
+
+    bb11 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-unwind.mir b/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-unwind.mir
new file mode 100644
index 000000000000..d73f5a1b6f71
--- /dev/null
+++ b/tests/mir-opt/building/match/array_len.slice_len.built.after.panic-unwind.mir
@@ -0,0 +1,115 @@
+// MIR for `slice_len` after built
+
+fn slice_len(_1: &[T]) -> () {
+    debug x => _1;
+    let mut _0: ();
+    let mut _2: usize;
+    let mut _3: usize;
+    let mut _4: usize;
+    let mut _5: bool;
+    let _10: ();
+    let mut _11: &T;
+    let _12: ();
+    let mut _13: &T;
+    let _14: ();
+    let mut _15: &[T];
+    let _16: ();
+    let mut _17: &T;
+    scope 1 {
+        debug a => _6;
+        debug b => _7;
+        debug rest => _8;
+        debug e => _9;
+        let _6: &T;
+        let _7: &T;
+        let _8: &[T];
+        let _9: &T;
+    }
+
+    bb0: {
+        PlaceMention(_1);
+        _3 = PtrMetadata(copy _1);
+        _2 = move _3;
+        _4 = const 3_usize;
+        _5 = Ge(move _2, move _4);
+        switchInt(move _5) -> [0: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        goto -> bb9;
+    }
+
+    bb2: {
+        falseEdge -> [real: bb4, imaginary: bb1];
+    }
+
+    bb3: {
+        goto -> bb1;
+    }
+
+    bb4: {
+        StorageLive(_6);
+        _6 = &(*_1)[0 of 3];
+        StorageLive(_7);
+        _7 = &(*_1)[1 of 3];
+        StorageLive(_8);
+        _8 = &(*_1)[2:-1];
+        StorageLive(_9);
+        _9 = &(*_1)[-1 of 3];
+        StorageLive(_10);
+        StorageLive(_11);
+        _11 = copy _6;
+        _10 = opaque::<&T>(move _11) -> [return: bb5, unwind: bb11];
+    }
+
+    bb5: {
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageLive(_12);
+        StorageLive(_13);
+        _13 = copy _7;
+        _12 = opaque::<&T>(move _13) -> [return: bb6, unwind: bb11];
+    }
+
+    bb6: {
+        StorageDead(_13);
+        StorageDead(_12);
+        StorageLive(_14);
+        StorageLive(_15);
+        _15 = copy _8;
+        _14 = opaque::<&[T]>(move _15) -> [return: bb7, unwind: bb11];
+    }
+
+    bb7: {
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageLive(_16);
+        StorageLive(_17);
+        _17 = copy _9;
+        _16 = opaque::<&T>(move _17) -> [return: bb8, unwind: bb11];
+    }
+
+    bb8: {
+        StorageDead(_17);
+        StorageDead(_16);
+        _0 = const ();
+        StorageDead(_9);
+        StorageDead(_8);
+        StorageDead(_7);
+        StorageDead(_6);
+        goto -> bb10;
+    }
+
+    bb9: {
+        _0 = const ();
+        goto -> bb10;
+    }
+
+    bb10: {
+        return;
+    }
+
+    bb11 (cleanup): {
+        resume;
+    }
+}
diff --git a/tests/mir-opt/building/receiver_ptr_mutability.main.built.after.mir b/tests/mir-opt/building/receiver_ptr_mutability.main.built.after.mir
index 744553c7cd26..5cf182c21c31 100644
--- a/tests/mir-opt/building/receiver_ptr_mutability.main.built.after.mir
+++ b/tests/mir-opt/building/receiver_ptr_mutability.main.built.after.mir
@@ -1,10 +1,10 @@
 // MIR for `main` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty(*mut Test), max_universe: U0, variables: [] }, span: $DIR/receiver_ptr_mutability.rs:15:14: 15:23, inferred_ty: *mut Test
-| 1: user_ty: Canonical { value: Ty(*mut Test), max_universe: U0, variables: [] }, span: $DIR/receiver_ptr_mutability.rs:15:14: 15:23, inferred_ty: *mut Test
-| 2: user_ty: Canonical { value: Ty(&&&&*mut Test), max_universe: U0, variables: [Region(U0), Region(U0), Region(U0), Region(U0)] }, span: $DIR/receiver_ptr_mutability.rs:19:18: 19:31, inferred_ty: &&&&*mut Test
-| 3: user_ty: Canonical { value: Ty(&&&&*mut Test), max_universe: U0, variables: [Region(U0), Region(U0), Region(U0), Region(U0)] }, span: $DIR/receiver_ptr_mutability.rs:19:18: 19:31, inferred_ty: &&&&*mut Test
+| 0: user_ty: Canonical { value: Ty(*mut Test), max_universe: U0, variables: [] }, span: $DIR/receiver_ptr_mutability.rs:16:14: 16:23, inferred_ty: *mut Test
+| 1: user_ty: Canonical { value: Ty(*mut Test), max_universe: U0, variables: [] }, span: $DIR/receiver_ptr_mutability.rs:16:14: 16:23, inferred_ty: *mut Test
+| 2: user_ty: Canonical { value: Ty(&&&&*mut Test), max_universe: U0, variables: [Region(U0), Region(U0), Region(U0), Region(U0)] }, span: $DIR/receiver_ptr_mutability.rs:20:18: 20:31, inferred_ty: &&&&*mut Test
+| 3: user_ty: Canonical { value: Ty(&&&&*mut Test), max_universe: U0, variables: [Region(U0), Region(U0), Region(U0), Region(U0)] }, span: $DIR/receiver_ptr_mutability.rs:20:18: 20:31, inferred_ty: &&&&*mut Test
 |
 fn main() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/receiver_ptr_mutability.rs b/tests/mir-opt/building/receiver_ptr_mutability.rs
index 1ddb8b71a5a3..37b142f1d8ad 100644
--- a/tests/mir-opt/building/receiver_ptr_mutability.rs
+++ b/tests/mir-opt/building/receiver_ptr_mutability.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // EMIT_MIR receiver_ptr_mutability.main.built.after.mir
 
diff --git a/tests/mir-opt/building/shifts.rs b/tests/mir-opt/building/shifts.rs
index d7747bb2f784..165b388bc18b 100644
--- a/tests/mir-opt/building/shifts.rs
+++ b/tests/mir-opt/building/shifts.rs
@@ -1,5 +1,5 @@
 // skip-filecheck
-//@ compile-flags: -C debug-assertions=yes
+//@ compile-flags: -Zmir-opt-level=0 -C debug-assertions=yes
 
 // EMIT_MIR shifts.shift_signed.built.after.mir
 fn shift_signed(small: i8, big: u128, a: i8, b: i32, c: i128) -> ([i8; 3], [u128; 3]) {
diff --git a/tests/mir-opt/building/storage_live_dead_in_statics.rs b/tests/mir-opt/building/storage_live_dead_in_statics.rs
index 7cb74acbf060..868f4a8f5597 100644
--- a/tests/mir-opt/building/storage_live_dead_in_statics.rs
+++ b/tests/mir-opt/building/storage_live_dead_in_statics.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 // Check that when we compile the static `XXX` into MIR, we do not
 // generate `StorageStart` or `StorageEnd` statements.
diff --git a/tests/mir-opt/building/uniform_array_move_out.move_out_by_subslice.built.after.mir b/tests/mir-opt/building/uniform_array_move_out.move_out_by_subslice.built.after.mir
index 6d3b2cf29103..839bdeca86a8 100644
--- a/tests/mir-opt/building/uniform_array_move_out.move_out_by_subslice.built.after.mir
+++ b/tests/mir-opt/building/uniform_array_move_out.move_out_by_subslice.built.after.mir
@@ -4,63 +4,55 @@ fn move_out_by_subslice() -> () {
     let mut _0: ();
     let _1: [std::boxed::Box; 2];
     let mut _2: std::boxed::Box;
-    let mut _3: usize;
-    let mut _4: usize;
-    let mut _5: *mut u8;
-    let mut _6: std::boxed::Box;
+    let mut _3: *mut u8;
+    let mut _4: std::boxed::Box;
+    let mut _5: std::boxed::Box;
+    let mut _6: *mut u8;
     let mut _7: std::boxed::Box;
-    let mut _8: usize;
-    let mut _9: usize;
-    let mut _10: *mut u8;
-    let mut _11: std::boxed::Box;
     scope 1 {
         debug a => _1;
-        let _12: [std::boxed::Box; 2];
+        let _8: [std::boxed::Box; 2];
         scope 2 {
-            debug _y => _12;
+            debug _y => _8;
         }
     }
 
     bb0: {
         StorageLive(_1);
         StorageLive(_2);
-        _3 = SizeOf(i32);
-        _4 = AlignOf(i32);
-        _5 = alloc::alloc::exchange_malloc(move _3, move _4) -> [return: bb1, unwind: bb13];
+        _3 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind: bb13];
     }
 
     bb1: {
-        StorageLive(_6);
-        _6 = ShallowInitBox(move _5, i32);
-        (*_6) = const 1_i32;
-        _2 = move _6;
-        drop(_6) -> [return: bb2, unwind: bb12];
+        StorageLive(_4);
+        _4 = ShallowInitBox(move _3, i32);
+        (*_4) = const 1_i32;
+        _2 = move _4;
+        drop(_4) -> [return: bb2, unwind: bb12];
     }
 
     bb2: {
-        StorageDead(_6);
-        StorageLive(_7);
-        _8 = SizeOf(i32);
-        _9 = AlignOf(i32);
-        _10 = alloc::alloc::exchange_malloc(move _8, move _9) -> [return: bb3, unwind: bb12];
+        StorageDead(_4);
+        StorageLive(_5);
+        _6 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb3, unwind: bb12];
     }
 
     bb3: {
-        StorageLive(_11);
-        _11 = ShallowInitBox(move _10, i32);
-        (*_11) = const 2_i32;
-        _7 = move _11;
-        drop(_11) -> [return: bb4, unwind: bb11];
+        StorageLive(_7);
+        _7 = ShallowInitBox(move _6, i32);
+        (*_7) = const 2_i32;
+        _5 = move _7;
+        drop(_7) -> [return: bb4, unwind: bb11];
     }
 
     bb4: {
-        StorageDead(_11);
-        _1 = [move _2, move _7];
-        drop(_7) -> [return: bb5, unwind: bb12];
+        StorageDead(_7);
+        _1 = [move _2, move _5];
+        drop(_5) -> [return: bb5, unwind: bb12];
     }
 
     bb5: {
-        StorageDead(_7);
+        StorageDead(_5);
         drop(_2) -> [return: bb6, unwind: bb13];
     }
 
@@ -68,10 +60,10 @@ fn move_out_by_subslice() -> () {
         StorageDead(_2);
         FakeRead(ForLet(None), _1);
         PlaceMention(_1);
-        StorageLive(_12);
-        _12 = move _1[0..2];
+        StorageLive(_8);
+        _8 = move _1[0..2];
         _0 = const ();
-        drop(_12) -> [return: bb8, unwind: bb10];
+        drop(_8) -> [return: bb8, unwind: bb10];
     }
 
     bb7: {
@@ -80,7 +72,7 @@ fn move_out_by_subslice() -> () {
     }
 
     bb8: {
-        StorageDead(_12);
+        StorageDead(_8);
         drop(_1) -> [return: bb9, unwind: bb13];
     }
 
@@ -94,7 +86,7 @@ fn move_out_by_subslice() -> () {
     }
 
     bb11 (cleanup): {
-        drop(_7) -> [return: bb12, unwind terminate(cleanup)];
+        drop(_5) -> [return: bb12, unwind terminate(cleanup)];
     }
 
     bb12 (cleanup): {
diff --git a/tests/mir-opt/building/uniform_array_move_out.move_out_from_end.built.after.mir b/tests/mir-opt/building/uniform_array_move_out.move_out_from_end.built.after.mir
index 003b90a912d2..7fda69c7500a 100644
--- a/tests/mir-opt/building/uniform_array_move_out.move_out_from_end.built.after.mir
+++ b/tests/mir-opt/building/uniform_array_move_out.move_out_from_end.built.after.mir
@@ -4,63 +4,55 @@ fn move_out_from_end() -> () {
     let mut _0: ();
     let _1: [std::boxed::Box; 2];
     let mut _2: std::boxed::Box;
-    let mut _3: usize;
-    let mut _4: usize;
-    let mut _5: *mut u8;
-    let mut _6: std::boxed::Box;
+    let mut _3: *mut u8;
+    let mut _4: std::boxed::Box;
+    let mut _5: std::boxed::Box;
+    let mut _6: *mut u8;
     let mut _7: std::boxed::Box;
-    let mut _8: usize;
-    let mut _9: usize;
-    let mut _10: *mut u8;
-    let mut _11: std::boxed::Box;
     scope 1 {
         debug a => _1;
-        let _12: std::boxed::Box;
+        let _8: std::boxed::Box;
         scope 2 {
-            debug _y => _12;
+            debug _y => _8;
         }
     }
 
     bb0: {
         StorageLive(_1);
         StorageLive(_2);
-        _3 = SizeOf(i32);
-        _4 = AlignOf(i32);
-        _5 = alloc::alloc::exchange_malloc(move _3, move _4) -> [return: bb1, unwind: bb13];
+        _3 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind: bb13];
     }
 
     bb1: {
-        StorageLive(_6);
-        _6 = ShallowInitBox(move _5, i32);
-        (*_6) = const 1_i32;
-        _2 = move _6;
-        drop(_6) -> [return: bb2, unwind: bb12];
+        StorageLive(_4);
+        _4 = ShallowInitBox(move _3, i32);
+        (*_4) = const 1_i32;
+        _2 = move _4;
+        drop(_4) -> [return: bb2, unwind: bb12];
     }
 
     bb2: {
-        StorageDead(_6);
-        StorageLive(_7);
-        _8 = SizeOf(i32);
-        _9 = AlignOf(i32);
-        _10 = alloc::alloc::exchange_malloc(move _8, move _9) -> [return: bb3, unwind: bb12];
+        StorageDead(_4);
+        StorageLive(_5);
+        _6 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb3, unwind: bb12];
     }
 
     bb3: {
-        StorageLive(_11);
-        _11 = ShallowInitBox(move _10, i32);
-        (*_11) = const 2_i32;
-        _7 = move _11;
-        drop(_11) -> [return: bb4, unwind: bb11];
+        StorageLive(_7);
+        _7 = ShallowInitBox(move _6, i32);
+        (*_7) = const 2_i32;
+        _5 = move _7;
+        drop(_7) -> [return: bb4, unwind: bb11];
     }
 
     bb4: {
-        StorageDead(_11);
-        _1 = [move _2, move _7];
-        drop(_7) -> [return: bb5, unwind: bb12];
+        StorageDead(_7);
+        _1 = [move _2, move _5];
+        drop(_5) -> [return: bb5, unwind: bb12];
     }
 
     bb5: {
-        StorageDead(_7);
+        StorageDead(_5);
         drop(_2) -> [return: bb6, unwind: bb13];
     }
 
@@ -68,10 +60,10 @@ fn move_out_from_end() -> () {
         StorageDead(_2);
         FakeRead(ForLet(None), _1);
         PlaceMention(_1);
-        StorageLive(_12);
-        _12 = move _1[1 of 2];
+        StorageLive(_8);
+        _8 = move _1[1 of 2];
         _0 = const ();
-        drop(_12) -> [return: bb8, unwind: bb10];
+        drop(_8) -> [return: bb8, unwind: bb10];
     }
 
     bb7: {
@@ -80,7 +72,7 @@ fn move_out_from_end() -> () {
     }
 
     bb8: {
-        StorageDead(_12);
+        StorageDead(_8);
         drop(_1) -> [return: bb9, unwind: bb13];
     }
 
@@ -94,7 +86,7 @@ fn move_out_from_end() -> () {
     }
 
     bb11 (cleanup): {
-        drop(_7) -> [return: bb12, unwind terminate(cleanup)];
+        drop(_5) -> [return: bb12, unwind terminate(cleanup)];
     }
 
     bb12 (cleanup): {
diff --git a/tests/mir-opt/building/uniform_array_move_out.rs b/tests/mir-opt/building/uniform_array_move_out.rs
index aff5996d0b64..36245273fe1c 100644
--- a/tests/mir-opt/building/uniform_array_move_out.rs
+++ b/tests/mir-opt/building/uniform_array_move_out.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 // skip-filecheck
 #![feature(liballoc_internals, rustc_attrs)]
 
diff --git a/tests/mir-opt/building/user_type_annotations.let_else.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_else.built.after.mir
index 6369dbec7506..bbf504d311f6 100644
--- a/tests/mir-opt/building/user_type_annotations.let_else.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_else.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `let_else` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:35:20: 35:45, inferred_ty: (u32, u64, &char)
-| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:35:20: 35:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:36:20: 36:45, inferred_ty: (u32, u64, &char)
+| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:36:20: 36:45, inferred_ty: (u32, u64, &char)
 |
 fn let_else() -> () {
     let mut _0: ();
@@ -21,9 +21,6 @@ fn let_else() -> () {
     }
 
     bb0: {
-        StorageLive(_2);
-        StorageLive(_3);
-        StorageLive(_4);
         StorageLive(_5);
         StorageLive(_6);
         StorageLive(_7);
@@ -51,16 +48,19 @@ fn let_else() -> () {
 
     bb4: {
         AscribeUserType(_5, +, UserTypeProjection { base: UserType(1), projs: [] });
+        StorageLive(_2);
         _2 = copy (_5.0: u32);
+        StorageLive(_3);
         _3 = copy (_5.1: u64);
+        StorageLive(_4);
         _4 = copy (_5.2: &char);
         StorageDead(_7);
         StorageDead(_5);
         _0 = const ();
-        StorageDead(_8);
         StorageDead(_4);
         StorageDead(_3);
         StorageDead(_2);
+        StorageDead(_8);
         return;
     }
 
@@ -68,9 +68,6 @@ fn let_else() -> () {
         StorageDead(_7);
         StorageDead(_5);
         StorageDead(_8);
-        StorageDead(_4);
-        StorageDead(_3);
-        StorageDead(_2);
         goto -> bb1;
     }
 
diff --git a/tests/mir-opt/building/user_type_annotations.let_else_bindless.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_else_bindless.built.after.mir
index b2a06ae53a8a..7bf2551e99f2 100644
--- a/tests/mir-opt/building/user_type_annotations.let_else_bindless.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_else_bindless.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `let_else_bindless` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:40:20: 40:45, inferred_ty: (u32, u64, &char)
-| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:40:20: 40:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:41:20: 41:45, inferred_ty: (u32, u64, &char)
+| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:41:20: 41:45, inferred_ty: (u32, u64, &char)
 |
 fn let_else_bindless() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.let_init.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_init.built.after.mir
index d1b8f823e9bc..0cf681d8ab2c 100644
--- a/tests/mir-opt/building/user_type_annotations.let_init.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_init.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `let_init` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:25:20: 25:45, inferred_ty: (u32, u64, &char)
-| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:25:20: 25:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:26:20: 26:45, inferred_ty: (u32, u64, &char)
+| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:26:20: 26:45, inferred_ty: (u32, u64, &char)
 |
 fn let_init() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.let_init_bindless.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_init_bindless.built.after.mir
index 6702f9300607..968813c826e4 100644
--- a/tests/mir-opt/building/user_type_annotations.let_init_bindless.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_init_bindless.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `let_init_bindless` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:30:20: 30:45, inferred_ty: (u32, u64, &char)
-| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:30:20: 30:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:31:20: 31:45, inferred_ty: (u32, u64, &char)
+| 1: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:31:20: 31:45, inferred_ty: (u32, u64, &char)
 |
 fn let_init_bindless() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.let_uninit.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_uninit.built.after.mir
index 76b5938b87d2..b6fdc4ff46dc 100644
--- a/tests/mir-opt/building/user_type_annotations.let_uninit.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_uninit.built.after.mir
@@ -1,7 +1,7 @@
 // MIR for `let_uninit` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:15:20: 15:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:16:20: 16:45, inferred_ty: (u32, u64, &char)
 |
 fn let_uninit() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.let_uninit_bindless.built.after.mir b/tests/mir-opt/building/user_type_annotations.let_uninit_bindless.built.after.mir
index 0cd125587714..472dbfb63043 100644
--- a/tests/mir-opt/building/user_type_annotations.let_uninit_bindless.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.let_uninit_bindless.built.after.mir
@@ -1,7 +1,7 @@
 // MIR for `let_uninit_bindless` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:20:20: 20:45, inferred_ty: (u32, u64, &char)
+| 0: user_ty: Canonical { value: Ty((u32, u64, &'static char)), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:21:20: 21:45, inferred_ty: (u32, u64, &char)
 |
 fn let_uninit_bindless() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.match_assoc_const.built.after.mir b/tests/mir-opt/building/user_type_annotations.match_assoc_const.built.after.mir
index c0ce6f1d06b5..ff4b0bf7600d 100644
--- a/tests/mir-opt/building/user_type_annotations.match_assoc_const.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.match_assoc_const.built.after.mir
@@ -1,8 +1,8 @@
 // MIR for `match_assoc_const` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:54:9: 54:44, inferred_ty: u32
-| 1: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:54:9: 54:44, inferred_ty: u32
+| 0: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:55:9: 55:44, inferred_ty: u32
+| 1: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:55:9: 55:44, inferred_ty: u32
 |
 fn match_assoc_const() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.match_assoc_const_range.built.after.mir b/tests/mir-opt/building/user_type_annotations.match_assoc_const_range.built.after.mir
index 3a6aa5b7c2ca..4cc433f475f6 100644
--- a/tests/mir-opt/building/user_type_annotations.match_assoc_const_range.built.after.mir
+++ b/tests/mir-opt/building/user_type_annotations.match_assoc_const_range.built.after.mir
@@ -1,10 +1,10 @@
 // MIR for `match_assoc_const_range` after built
 
 | User Type Annotations
-| 0: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:62:11: 62:46, inferred_ty: u32
-| 1: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:62:11: 62:46, inferred_ty: u32
-| 2: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:63:9: 63:44, inferred_ty: u32
-| 3: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:63:9: 63:44, inferred_ty: u32
+| 0: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:63:11: 63:46, inferred_ty: u32
+| 1: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:63:11: 63:46, inferred_ty: u32
+| 2: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:64:9: 64:44, inferred_ty: u32
+| 3: user_ty: Canonical { value: TypeOf(DefId(0:11 ~ user_type_annotations[ee8e]::MyTrait::FOO), UserArgs { args: [MyStruct, 'static], user_self_ty: None }), max_universe: U0, variables: [] }, span: $DIR/user_type_annotations.rs:64:9: 64:44, inferred_ty: u32
 |
 fn match_assoc_const_range() -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/building/user_type_annotations.rs b/tests/mir-opt/building/user_type_annotations.rs
index d55c678d5ae3..527bf62f7dc0 100644
--- a/tests/mir-opt/building/user_type_annotations.rs
+++ b/tests/mir-opt/building/user_type_annotations.rs
@@ -1,3 +1,4 @@
+//@ compile-flags: -Zmir-opt-level=0
 //@ edition: 2024
 // skip-filecheck
 
diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
index f43c0cca9ad2..95eaf18b4703 100644
--- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
+++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-abort.diff
@@ -6,12 +6,13 @@
       let _1: i32;
       let mut _2: i32;
       let mut _3: std::boxed::Box;
-      let mut _4: usize;
-      let mut _5: usize;
-      let mut _6: *mut u8;
-      let mut _7: std::boxed::Box;
-      let mut _8: *const i32;
+      let mut _4: *mut u8;
+      let mut _5: std::boxed::Box;
+      let mut _6: *const i32;
+      let mut _7: std::ptr::NonNull;
+      let mut _8: std::ptr::Unique;
       let mut _9: *const i32;
+      let mut _10: *const i32;
       scope 1 {
           debug x => _1;
       }
@@ -21,23 +22,26 @@
 -         StorageLive(_2);
 +         nop;
           StorageLive(_3);
--         _4 = SizeOf(i32);
--         _5 = AlignOf(i32);
--         _6 = alloc::alloc::exchange_malloc(move _4, move _5) -> [return: bb1, unwind unreachable];
-+         _4 = const 4_usize;
-+         _5 = const 4_usize;
-+         _6 = alloc::alloc::exchange_malloc(const 4_usize, const 4_usize) -> [return: bb1, unwind unreachable];
+          _4 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
-          StorageLive(_7);
-          _7 = ShallowInitBox(move _6, i32);
-          _8 = copy ((_7.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
-          (*_8) = const 42_i32;
-          _3 = move _7;
-          StorageDead(_7);
-          _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
-          _2 = copy (*_9);
+          StorageLive(_5);
+-         _6 = move _4 as *const i32 (Transmute);
+-         _7 = NonNull:: { pointer: move _6 };
+-         _8 = Unique:: { pointer: move _7, _marker: const PhantomData:: };
++         _6 = copy _4 as *const i32 (PtrToPtr);
++         _7 = NonNull:: { pointer: copy _6 };
++         _8 = Unique:: { pointer: copy _7, _marker: const PhantomData:: };
+          _5 = Box::(move _8, const std::alloc::Global);
+-         _9 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
+-         (*_9) = const 42_i32;
++         _9 = copy _6;
++         (*_6) = const 42_i32;
+          _3 = move _5;
+          StorageDead(_5);
+          _10 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
+          _2 = copy (*_10);
 -         _1 = Add(move _2, const 0_i32);
 -         StorageDead(_2);
 +         _1 = copy _2;
diff --git a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
index 2c903b6d8534..6d8d3a0dcfe2 100644
--- a/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/boxes.main.GVN.panic-unwind.diff
@@ -6,12 +6,13 @@
       let _1: i32;
       let mut _2: i32;
       let mut _3: std::boxed::Box;
-      let mut _4: usize;
-      let mut _5: usize;
-      let mut _6: *mut u8;
-      let mut _7: std::boxed::Box;
-      let mut _8: *const i32;
+      let mut _4: *mut u8;
+      let mut _5: std::boxed::Box;
+      let mut _6: *const i32;
+      let mut _7: std::ptr::NonNull;
+      let mut _8: std::ptr::Unique;
       let mut _9: *const i32;
+      let mut _10: *const i32;
       scope 1 {
           debug x => _1;
       }
@@ -21,23 +22,26 @@
 -         StorageLive(_2);
 +         nop;
           StorageLive(_3);
--         _4 = SizeOf(i32);
--         _5 = AlignOf(i32);
--         _6 = alloc::alloc::exchange_malloc(move _4, move _5) -> [return: bb1, unwind continue];
-+         _4 = const 4_usize;
-+         _5 = const 4_usize;
-+         _6 = alloc::alloc::exchange_malloc(const 4_usize, const 4_usize) -> [return: bb1, unwind continue];
+          _4 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind continue];
       }
   
       bb1: {
-          StorageLive(_7);
-          _7 = ShallowInitBox(move _6, i32);
-          _8 = copy ((_7.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
-          (*_8) = const 42_i32;
-          _3 = move _7;
-          StorageDead(_7);
-          _9 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
-          _2 = copy (*_9);
+          StorageLive(_5);
+-         _6 = move _4 as *const i32 (Transmute);
+-         _7 = NonNull:: { pointer: move _6 };
+-         _8 = Unique:: { pointer: move _7, _marker: const PhantomData:: };
++         _6 = copy _4 as *const i32 (PtrToPtr);
++         _7 = NonNull:: { pointer: copy _6 };
++         _8 = Unique:: { pointer: copy _7, _marker: const PhantomData:: };
+          _5 = Box::(move _8, const std::alloc::Global);
+-         _9 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
+-         (*_9) = const 42_i32;
++         _9 = copy _6;
++         (*_6) = const 42_i32;
+          _3 = move _5;
+          StorageDead(_5);
+          _10 = copy ((_3.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
+          _2 = copy (*_10);
 -         _1 = Add(move _2, const 0_i32);
 -         StorageDead(_2);
 +         _1 = copy _2;
diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
index 24b10217865b..5df2232053fe 100644
--- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
+++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff
@@ -3,23 +3,17 @@
   
   fn hello() -> () {
       let mut _0: ();
-      let mut _1: bool;
-      let mut _2: !;
+      let mut _1: !;
   
       bb0: {
-          StorageLive(_1);
--         _1 = const ::NEEDS;
--         switchInt(move _1) -> [0: bb2, otherwise: bb1];
-+         _1 = const false;
-+         switchInt(const false) -> [0: bb2, otherwise: bb1];
+          goto -> bb2;
       }
   
       bb1: {
-          _2 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+          _1 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
       }
   
       bb2: {
-          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
index a73485e7944d..788a4424943e 100644
--- a/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff
@@ -3,23 +3,17 @@
   
   fn hello() -> () {
       let mut _0: ();
-      let mut _1: bool;
-      let mut _2: !;
+      let mut _1: !;
   
       bb0: {
-          StorageLive(_1);
--         _1 = const ::NEEDS;
--         switchInt(move _1) -> [0: bb2, otherwise: bb1];
-+         _1 = const false;
-+         switchInt(const false) -> [0: bb2, otherwise: bb1];
+          goto -> bb2;
       }
   
       bb1: {
-          _2 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
+          _1 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
       }
   
       bb2: {
-          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/const_prop/control_flow_simplification.rs b/tests/mir-opt/const_prop/control_flow_simplification.rs
index 64605ca11c2c..8ec630d6f48d 100644
--- a/tests/mir-opt/const_prop/control_flow_simplification.rs
+++ b/tests/mir-opt/const_prop/control_flow_simplification.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 //@ test-mir-pass: GVN
 //@ compile-flags: -Zmir-opt-level=1
@@ -12,6 +11,9 @@ impl NeedsDrop for This {}
 // EMIT_MIR control_flow_simplification.hello.GVN.diff
 // EMIT_MIR control_flow_simplification.hello.PreCodegen.before.mir
 fn hello() {
+    // CHECK-LABEL: fn hello(
+    // CHECK: bb0:
+    // CHECK-NEXT: return;
     if ::NEEDS {
         panic!()
     }
diff --git a/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff b/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff
index 5e843da86792..a4900a1ac72e 100644
--- a/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff
+++ b/tests/mir-opt/const_prop/invalid_constant.main.GVN.diff
@@ -28,21 +28,27 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = InvalidChar { int: const 1114113_u32 };
-          _1 = copy (_2.1: char);
+-         _2 = InvalidChar { int: const 1114113_u32 };
+-         _1 = copy (_2.1: char);
++         _2 = const InvalidChar {{ int: 1114113_u32, chr: {transmute(0x00110001): char} }};
++         _1 = const {transmute(0x00110001): char};
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
           StorageLive(_5);
-          _5 = InvalidTag { int: const 4_u32 };
-          _4 = copy (_5.1: E);
-          _3 = [move _4];
+-         _5 = InvalidTag { int: const 4_u32 };
+-         _4 = copy (_5.1: E);
+-         _3 = [move _4];
++         _5 = const InvalidTag {{ int: 4_u32, e: Scalar(0x00000004): E }};
++         _4 = const Scalar(0x00000004): E;
++         _3 = [const Scalar(0x00000004): E];
           StorageDead(_4);
           StorageDead(_5);
           nop;
           nop;
           StorageLive(_8);
-          _8 = NoVariants { int: const 0_u32 };
+-         _8 = NoVariants { int: const 0_u32 };
++         _8 = const NoVariants {{ int: 0_u32, empty: ZeroSized: Empty }};
           nop;
           nop;
           nop;
@@ -55,5 +61,17 @@
           StorageDead(_1);
           return;
       }
++ }
++ 
++ ALLOC0 (size: 4, align: 4) {
++     00 00 00 00                                     │ ....
++ }
++ 
++ ALLOC1 (size: 4, align: 4) {
++     04 00 00 00                                     │ ....
++ }
++ 
++ ALLOC2 (size: 4, align: 4) {
++     01 00 11 00                                     │ ....
   }
   
diff --git a/tests/mir-opt/const_prop/transmute.rs b/tests/mir-opt/const_prop/transmute.rs
index 33cbefbf053f..ece6331ade27 100644
--- a/tests/mir-opt/const_prop/transmute.rs
+++ b/tests/mir-opt/const_prop/transmute.rs
@@ -43,8 +43,8 @@ pub unsafe fn invalid_bool() -> bool {
 // EMIT_MIR transmute.undef_union_as_integer.GVN.diff
 pub unsafe fn undef_union_as_integer() -> u32 {
     // CHECK-LABEL: fn undef_union_as_integer(
-    // CHECK: _1 = Union32 {
-    // CHECK: _0 = move _1 as u32 (Transmute);
+    // CHECK: _1 = const Union32
+    // CHECK: _0 = const {{.*}}: u32;
     union Union32 {
         value: u32,
         unit: (),
diff --git a/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.32bit.diff b/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.32bit.diff
index 2ac9769a0e77..be0450114b1c 100644
--- a/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.32bit.diff
+++ b/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.32bit.diff
@@ -12,11 +12,16 @@
 -         _2 = ();
 -         _1 = Union32 { value: move _2 };
 +         _2 = const ();
-+         _1 = Union32 { value: const () };
++         _1 = const Union32 {{ value: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: u32, unit: () }};
           StorageDead(_2);
-          _0 = move _1 as u32 (Transmute);
+-         _0 = move _1 as u32 (Transmute);
++         _0 = const Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: u32;
           StorageDead(_1);
           return;
       }
++ }
++ 
++ ALLOC0 (size: 4, align: 4) {
++     __ __ __ __                                     │ ░░░░
   }
   
diff --git a/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.64bit.diff b/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.64bit.diff
index 2ac9769a0e77..be0450114b1c 100644
--- a/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.64bit.diff
+++ b/tests/mir-opt/const_prop/transmute.undef_union_as_integer.GVN.64bit.diff
@@ -12,11 +12,16 @@
 -         _2 = ();
 -         _1 = Union32 { value: move _2 };
 +         _2 = const ();
-+         _1 = Union32 { value: const () };
++         _1 = const Union32 {{ value: Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: u32, unit: () }};
           StorageDead(_2);
-          _0 = move _1 as u32 (Transmute);
+-         _0 = move _1 as u32 (Transmute);
++         _0 = const Indirect { alloc_id: ALLOC0, offset: Size(0 bytes) }: u32;
           StorageDead(_1);
           return;
       }
++ }
++ 
++ ALLOC0 (size: 4, align: 4) {
++     __ __ __ __                                     │ ░░░░
   }
   
diff --git a/tests/mir-opt/const_prop/trivial_const.rs b/tests/mir-opt/const_prop/trivial_const.rs
new file mode 100644
index 000000000000..e9134c1ebdef
--- /dev/null
+++ b/tests/mir-opt/const_prop/trivial_const.rs
@@ -0,0 +1,15 @@
+//@ test-mir-pass: SimplifyConstCondition-after-inst-simplify
+//@ compile-flags: -Zmir-enable-passes=+InstSimplify-after-simplifycfg -Zub_checks=false -Zinline-mir
+
+#![crate_type = "lib"]
+
+// EMIT_MIR trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff
+pub fn unwrap_unchecked(v: &Option) -> i32 {
+    // CHECK-LABEL: fn unwrap_unchecked(
+    // CHECK: bb0: {
+    // CHECK: switchInt({{.*}}) -> [0: [[AssumeFalseBB:bb.*]], 1:
+    // CHECK: [[AssumeFalseBB]]: {
+    // CHECK-NEXT: unreachable;
+    let v = unsafe { v.unwrap_unchecked() };
+    v
+}
diff --git a/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff b/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff
new file mode 100644
index 000000000000..d14c42a330eb
--- /dev/null
+++ b/tests/mir-opt/const_prop/trivial_const.unwrap_unchecked.SimplifyConstCondition-after-inst-simplify.diff
@@ -0,0 +1,58 @@
+- // MIR for `unwrap_unchecked` before SimplifyConstCondition-after-inst-simplify
++ // MIR for `unwrap_unchecked` after SimplifyConstCondition-after-inst-simplify
+  
+  fn unwrap_unchecked(_1: &Option) -> i32 {
+      debug v => _1;
+      let mut _0: i32;
+      let _2: i32;
+      let mut _3: std::option::Option;
+      scope 1 {
+          debug v => _2;
+      }
+      scope 2 (inlined #[track_caller] Option::::unwrap_unchecked) {
+          let mut _4: isize;
+          scope 3 {
+          }
+          scope 4 (inlined #[track_caller] unreachable_unchecked) {
+              let _5: ();
+              scope 5 (inlined core::ub_checks::check_language_ub) {
+                  let mut _6: bool;
+                  scope 6 (inlined core::ub_checks::check_language_ub::runtime) {
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = copy (*_1);
+          StorageLive(_4);
+          StorageLive(_5);
+          _4 = discriminant(_3);
+          switchInt(move _4) -> [0: bb2, 1: bb3, otherwise: bb1];
+      }
+  
+      bb1: {
+          unreachable;
+      }
+  
+      bb2: {
+-         StorageLive(_6);
+-         _6 = const false;
+-         assume(copy _6);
+-         _5 = unreachable_unchecked::precondition_check() -> [return: bb1, unwind unreachable];
++         unreachable;
+      }
+  
+      bb3: {
+          _2 = move ((_3 as Some).0: i32);
+          StorageDead(_5);
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = copy _2;
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/const_prop/union.main.GVN.diff b/tests/mir-opt/const_prop/union.main.GVN.diff
new file mode 100644
index 000000000000..16a0432ab901
--- /dev/null
+++ b/tests/mir-opt/const_prop/union.main.GVN.diff
@@ -0,0 +1,42 @@
+- // MIR for `main` before GVN
++ // MIR for `main` after GVN
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: main::Un;
+      let mut _2: u32;
+      let _3: ();
+      let mut _4: u32;
+      scope 1 {
+          debug un => _1;
+          scope 3 (inlined std::mem::drop::) {
+          }
+      }
+      scope 2 (inlined val) {
+      }
+  
+      bb0: {
+          StorageLive(_1);
+-         StorageLive(_2);
++         nop;
+          _2 = const 1_u32;
+-         _1 = Un { us: move _2 };
+-         StorageDead(_2);
++         _1 = const Un {{ us: 1_u32 }};
++         nop;
+          StorageLive(_3);
+          StorageLive(_4);
+-         _4 = copy (_1.0: u32);
++         _4 = const 1_u32;
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = const ();
+          StorageDead(_1);
+          return;
+      }
++ }
++ 
++ ALLOC0 (size: 4, align: 4) {
++     01 00 00 00                                     │ ....
+  }
+  
diff --git a/tests/mir-opt/const_prop/union.rs b/tests/mir-opt/const_prop/union.rs
new file mode 100644
index 000000000000..9f197a1a5833
--- /dev/null
+++ b/tests/mir-opt/const_prop/union.rs
@@ -0,0 +1,22 @@
+//! Tests that we can propagate into places that are projections into unions
+//@ test-mir-pass: GVN
+//@ compile-flags: -Zinline-mir
+
+fn val() -> u32 {
+    1
+}
+
+// EMIT_MIR union.main.GVN.diff
+fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: debug un => [[un:_.*]];
+    // CHECK: bb0: {
+    // CHECK: [[un]] = const Un {{{{ us: 1_u32 }}}};
+    union Un {
+        us: u32,
+    }
+
+    let un = Un { us: val() };
+
+    drop(unsafe { un.us });
+}
diff --git a/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir b/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir
index 7e033916fd34..33fbca7f77ed 100644
--- a/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir
+++ b/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-abort.mir
@@ -7,15 +7,14 @@ fn main::{closure#0}(_1: *mut {coroutine@$DIR/coroutine_drop_cleanup.rs:12:5: 12
     let _4: ();
     let mut _5: ();
     let mut _6: ();
-    let mut _7: ();
-    let mut _8: u32;
+    let mut _7: u32;
     scope 1 {
         debug _s => (((*_1) as variant#3).0: std::string::String);
     }
 
     bb0: {
-        _8 = discriminant((*_1));
-        switchInt(move _8) -> [0: bb5, 3: bb8, otherwise: bb9];
+        _7 = discriminant((*_1));
+        switchInt(move _7) -> [0: bb5, 3: bb8, otherwise: bb9];
     }
 
     bb1: {
diff --git a/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir b/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir
index 613ef2909b5e..69e7219af9ff 100644
--- a/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir
+++ b/tests/mir-opt/coroutine_drop_cleanup.main-{closure#0}.coroutine_drop.0.panic-unwind.mir
@@ -7,15 +7,14 @@ fn main::{closure#0}(_1: *mut {coroutine@$DIR/coroutine_drop_cleanup.rs:12:5: 12
     let _4: ();
     let mut _5: ();
     let mut _6: ();
-    let mut _7: ();
-    let mut _8: u32;
+    let mut _7: u32;
     scope 1 {
         debug _s => (((*_1) as variant#3).0: std::string::String);
     }
 
     bb0: {
-        _8 = discriminant((*_1));
-        switchInt(move _8) -> [0: bb7, 3: bb10, otherwise: bb11];
+        _7 = discriminant((*_1));
+        switchInt(move _7) -> [0: bb7, 3: bb10, otherwise: bb11];
     }
 
     bb1: {
diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir
index f8b3f68d21e6..222c7144ef07 100644
--- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir
+++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir
@@ -1,7 +1,7 @@
 // MIR for `main::{closure#0}` 0 coroutine_resume
 /* coroutine_layout = CoroutineLayout {
     field_tys: {
-        _0: CoroutineSavedTy {
+        _s0: CoroutineSavedTy {
             ty: HasDrop,
             source_info: SourceInfo {
                 span: $DIR/coroutine_tiny.rs:22:13: 22:15 (#0),
@@ -14,15 +14,15 @@
         Unresumed(0): [],
         Returned (1): [],
         Panicked (2): [],
-        Suspend0 (3): [_0],
+        Suspend0 (3): [_s0],
     },
     storage_conflicts: BitMatrix(1x1) {
-        (_0, _0),
+        (_s0, _s0),
     },
 } */
 
 fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}>, _2: u8) -> CoroutineState<(), ()> {
-    debug _x => _10;
+    debug _x => _2;
     let mut _0: std::ops::CoroutineState<(), ()>;
     let _3: HasDrop;
     let mut _4: !;
@@ -31,21 +31,21 @@ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}
     let mut _7: ();
     let _8: ();
     let mut _9: ();
-    let _10: u8;
-    let mut _11: u32;
+    let mut _10: u32;
+    let mut _11: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13};
     scope 1 {
-        debug _d => (((*(_1.0: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13})) as variant#3).0: HasDrop);
+        debug _d => (((*_11) as variant#3).0: HasDrop);
     }
 
     bb0: {
-        _11 = discriminant((*(_1.0: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13})));
-        switchInt(move _11) -> [0: bb1, 3: bb5, otherwise: bb6];
+        _11 = copy (_1.0: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13});
+        _10 = discriminant((*_11));
+        switchInt(move _10) -> [0: bb1, 3: bb5, otherwise: bb6];
     }
 
     bb1: {
-        _10 = move _2;
         nop;
-        (((*(_1.0: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13})) as variant#3).0: HasDrop) = HasDrop;
+        (((*_11) as variant#3).0: HasDrop) = HasDrop;
         StorageLive(_4);
         goto -> bb2;
     }
@@ -58,7 +58,7 @@ fn main::{closure#0}(_1: Pin<&mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}
         StorageDead(_4);
         StorageDead(_6);
         StorageDead(_7);
-        discriminant((*(_1.0: &mut {coroutine@$DIR/coroutine_tiny.rs:21:5: 21:13}))) = 3;
+        discriminant((*_11)) = 3;
         return;
     }
 
diff --git a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff
index d465b8bded22..fa88211383a0 100644
--- a/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff
+++ b/tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff
@@ -40,7 +40,7 @@
 +     coverage Code { bcb: bcb5 } => $DIR/branch_match_arms.rs:19:17: 19:18 (#0);
 +     coverage Code { bcb: bcb5 } => $DIR/branch_match_arms.rs:19:23: 19:30 (#0);
 +     coverage Code { bcb: bcb5 } => $DIR/branch_match_arms.rs:19:31: 19:32 (#0);
-+     coverage Code { bcb: bcb2 } => $DIR/branch_match_arms.rs:21:2: 21:2 (#0);
++     coverage Code { bcb: bcb2 } => $DIR/branch_match_arms.rs:21:1: 21:2 (#0);
 + 
       bb0: {
 +         Coverage::VirtualCounter(bcb0);
diff --git a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff
index cf6d85abd80e..9b6d2b22087b 100644
--- a/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff
+++ b/tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff
@@ -6,7 +6,7 @@
   
 +     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:27:1: 27:17 (#0);
 +     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:28:5: 28:9 (#0);
-+     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:29:2: 29:2 (#0);
++     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:29:1: 29:2 (#0);
 + 
       bb0: {
 +         Coverage::VirtualCounter(bcb0);
diff --git a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff
index 980c5e202ffd..b2bb2375aee6 100644
--- a/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff
+++ b/tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff
@@ -10,8 +10,8 @@
 +     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:13:1: 13:10 (#0);
 +     coverage Code { bcb: bcb1 } => $DIR/instrument_coverage.rs:15:12: 15:15 (#0);
 +     coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:16:13: 16:18 (#0);
-+     coverage Code { bcb: bcb3 } => $DIR/instrument_coverage.rs:17:10: 17:10 (#0);
-+     coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:19:2: 19:2 (#0);
++     coverage Code { bcb: bcb3 } => $DIR/instrument_coverage.rs:17:9: 17:10 (#0);
++     coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:19:1: 19:2 (#0);
 + 
       bb0: {
 +         Coverage::VirtualCounter(bcb0);
diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff
index b707cd41788a..2eb78c08ee80 100644
--- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff
+++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff
@@ -10,8 +10,8 @@
       coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 13:10 (#0);
       coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
       coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0);
-      coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0);
-      coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0);
+      coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:38: 14:39 (#0);
+      coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:1: 15:2 (#0);
       coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
   
       bb0: {
diff --git a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff
index 239b845c2311..0c1bc24b6dc1 100644
--- a/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff
+++ b/tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff
@@ -10,8 +10,8 @@
 +     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 13:10 (#0);
 +     coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
 +     coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0);
-+     coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0);
-+     coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:2: 15:2 (#0);
++     coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:38: 14:39 (#0);
++     coverage Code { bcb: bcb2 } => $DIR/instrument_coverage_cleanup.rs:15:1: 15:2 (#0);
 +     coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
 + 
       bb0: {
diff --git a/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-abort.diff
new file mode 100644
index 000000000000..fd9ef54fe775
--- /dev/null
+++ b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-abort.diff
@@ -0,0 +1,77 @@
+- // MIR for `main` before DataflowConstProp
++ // MIR for `main` after DataflowConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: u32;
+      let mut _2: &[u32];
+      let mut _3: &[u32; 3];
+      let _4: &[u32; 3];
+      let _5: [u32; 3];
+      let _6: usize;
+      let mut _7: usize;
+      let mut _8: bool;
+      let mut _10: &[u32];
+      let _11: usize;
+      let mut _12: usize;
+      let mut _13: bool;
+      let mut _14: &[u32; 3];
+      scope 1 {
+          debug local => _1;
+          let _9: u32;
+          scope 2 {
+              debug constant => _9;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+          _14 = const main::promoted[0];
+          _4 = copy _14;
+          _3 = copy _4;
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize, AsCast));
+          StorageDead(_3);
+          StorageLive(_6);
+          _6 = const 1_usize;
+-         _7 = PtrMetadata(copy _2);
+-         _8 = Lt(copy _6, copy _7);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, copy _6) -> [success: bb1, unwind unreachable];
++         _7 = const 3_usize;
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+-         _1 = copy (*_2)[_6];
++         _1 = copy (*_2)[1 of 2];
+          StorageDead(_6);
+          StorageDead(_4);
+          StorageDead(_2);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = const main::SLICE;
+          StorageLive(_11);
+          _11 = const 1_usize;
+-         _12 = PtrMetadata(copy _10);
+-         _13 = Lt(copy _11, copy _12);
+-         assert(move _13, "index out of bounds: the length is {} but the index is {}", move _12, copy _11) -> [success: bb2, unwind unreachable];
++         _12 = const 3_usize;
++         _13 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+-         _9 = copy (*_10)[_11];
++         _9 = copy (*_10)[1 of 2];
+          StorageDead(_11);
+          StorageDead(_10);
+          _0 = const ();
+          StorageDead(_9);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-unwind.diff
new file mode 100644
index 000000000000..1fec08c25620
--- /dev/null
+++ b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.32bit.panic-unwind.diff
@@ -0,0 +1,77 @@
+- // MIR for `main` before DataflowConstProp
++ // MIR for `main` after DataflowConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: u32;
+      let mut _2: &[u32];
+      let mut _3: &[u32; 3];
+      let _4: &[u32; 3];
+      let _5: [u32; 3];
+      let _6: usize;
+      let mut _7: usize;
+      let mut _8: bool;
+      let mut _10: &[u32];
+      let _11: usize;
+      let mut _12: usize;
+      let mut _13: bool;
+      let mut _14: &[u32; 3];
+      scope 1 {
+          debug local => _1;
+          let _9: u32;
+          scope 2 {
+              debug constant => _9;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+          _14 = const main::promoted[0];
+          _4 = copy _14;
+          _3 = copy _4;
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize, AsCast));
+          StorageDead(_3);
+          StorageLive(_6);
+          _6 = const 1_usize;
+-         _7 = PtrMetadata(copy _2);
+-         _8 = Lt(copy _6, copy _7);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, copy _6) -> [success: bb1, unwind continue];
++         _7 = const 3_usize;
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+-         _1 = copy (*_2)[_6];
++         _1 = copy (*_2)[1 of 2];
+          StorageDead(_6);
+          StorageDead(_4);
+          StorageDead(_2);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = const main::SLICE;
+          StorageLive(_11);
+          _11 = const 1_usize;
+-         _12 = PtrMetadata(copy _10);
+-         _13 = Lt(copy _11, copy _12);
+-         assert(move _13, "index out of bounds: the length is {} but the index is {}", move _12, copy _11) -> [success: bb2, unwind continue];
++         _12 = const 3_usize;
++         _13 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb2, unwind continue];
+      }
+  
+      bb2: {
+-         _9 = copy (*_10)[_11];
++         _9 = copy (*_10)[1 of 2];
+          StorageDead(_11);
+          StorageDead(_10);
+          _0 = const ();
+          StorageDead(_9);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-abort.diff
new file mode 100644
index 000000000000..fd9ef54fe775
--- /dev/null
+++ b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-abort.diff
@@ -0,0 +1,77 @@
+- // MIR for `main` before DataflowConstProp
++ // MIR for `main` after DataflowConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: u32;
+      let mut _2: &[u32];
+      let mut _3: &[u32; 3];
+      let _4: &[u32; 3];
+      let _5: [u32; 3];
+      let _6: usize;
+      let mut _7: usize;
+      let mut _8: bool;
+      let mut _10: &[u32];
+      let _11: usize;
+      let mut _12: usize;
+      let mut _13: bool;
+      let mut _14: &[u32; 3];
+      scope 1 {
+          debug local => _1;
+          let _9: u32;
+          scope 2 {
+              debug constant => _9;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+          _14 = const main::promoted[0];
+          _4 = copy _14;
+          _3 = copy _4;
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize, AsCast));
+          StorageDead(_3);
+          StorageLive(_6);
+          _6 = const 1_usize;
+-         _7 = PtrMetadata(copy _2);
+-         _8 = Lt(copy _6, copy _7);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, copy _6) -> [success: bb1, unwind unreachable];
++         _7 = const 3_usize;
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+-         _1 = copy (*_2)[_6];
++         _1 = copy (*_2)[1 of 2];
+          StorageDead(_6);
+          StorageDead(_4);
+          StorageDead(_2);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = const main::SLICE;
+          StorageLive(_11);
+          _11 = const 1_usize;
+-         _12 = PtrMetadata(copy _10);
+-         _13 = Lt(copy _11, copy _12);
+-         assert(move _13, "index out of bounds: the length is {} but the index is {}", move _12, copy _11) -> [success: bb2, unwind unreachable];
++         _12 = const 3_usize;
++         _13 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+-         _9 = copy (*_10)[_11];
++         _9 = copy (*_10)[1 of 2];
+          StorageDead(_11);
+          StorageDead(_10);
+          _0 = const ();
+          StorageDead(_9);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-unwind.diff
new file mode 100644
index 000000000000..1fec08c25620
--- /dev/null
+++ b/tests/mir-opt/dataflow-const-prop/slice_len.main.DataflowConstProp.64bit.panic-unwind.diff
@@ -0,0 +1,77 @@
+- // MIR for `main` before DataflowConstProp
++ // MIR for `main` after DataflowConstProp
+  
+  fn main() -> () {
+      let mut _0: ();
+      let _1: u32;
+      let mut _2: &[u32];
+      let mut _3: &[u32; 3];
+      let _4: &[u32; 3];
+      let _5: [u32; 3];
+      let _6: usize;
+      let mut _7: usize;
+      let mut _8: bool;
+      let mut _10: &[u32];
+      let _11: usize;
+      let mut _12: usize;
+      let mut _13: bool;
+      let mut _14: &[u32; 3];
+      scope 1 {
+          debug local => _1;
+          let _9: u32;
+          scope 2 {
+              debug constant => _9;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+          _14 = const main::promoted[0];
+          _4 = copy _14;
+          _3 = copy _4;
+          _2 = move _3 as &[u32] (PointerCoercion(Unsize, AsCast));
+          StorageDead(_3);
+          StorageLive(_6);
+          _6 = const 1_usize;
+-         _7 = PtrMetadata(copy _2);
+-         _8 = Lt(copy _6, copy _7);
+-         assert(move _8, "index out of bounds: the length is {} but the index is {}", move _7, copy _6) -> [success: bb1, unwind continue];
++         _7 = const 3_usize;
++         _8 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+-         _1 = copy (*_2)[_6];
++         _1 = copy (*_2)[1 of 2];
+          StorageDead(_6);
+          StorageDead(_4);
+          StorageDead(_2);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = const main::SLICE;
+          StorageLive(_11);
+          _11 = const 1_usize;
+-         _12 = PtrMetadata(copy _10);
+-         _13 = Lt(copy _11, copy _12);
+-         assert(move _13, "index out of bounds: the length is {} but the index is {}", move _12, copy _11) -> [success: bb2, unwind continue];
++         _12 = const 3_usize;
++         _13 = const true;
++         assert(const true, "index out of bounds: the length is {} but the index is {}", const 3_usize, const 1_usize) -> [success: bb2, unwind continue];
+      }
+  
+      bb2: {
+-         _9 = copy (*_10)[_11];
++         _9 = copy (*_10)[1 of 2];
+          StorageDead(_11);
+          StorageDead(_10);
+          _0 = const ();
+          StorageDead(_9);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dataflow-const-prop/slice_len.rs b/tests/mir-opt/dataflow-const-prop/slice_len.rs
new file mode 100644
index 000000000000..e0e68f9fde54
--- /dev/null
+++ b/tests/mir-opt/dataflow-const-prop/slice_len.rs
@@ -0,0 +1,34 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ test-mir-pass: DataflowConstProp
+//@ compile-flags: -Zmir-enable-passes=+InstSimplify-after-simplifycfg
+// EMIT_MIR_FOR_EACH_BIT_WIDTH
+
+// EMIT_MIR slice_len.main.DataflowConstProp.diff
+
+// CHECK-LABEL: fn main(
+fn main() {
+    // CHECK: debug local => [[local:_.*]];
+    // CHECK: debug constant => [[constant:_.*]];
+
+    // CHECK-NOT: {{_.*}} = Len(
+    // CHECK-NOT: {{_.*}} = Lt(
+    // CHECK-NOT: assert(move _
+    // CHECK: {{_.*}} = const 3_usize;
+    // CHECK: {{_.*}} = const true;
+    // CHECK: assert(const true,
+
+    // CHECK: [[local]] = copy (*{{_.*}})[1 of 2];
+    let local = (&[1u32, 2, 3] as &[u32])[1];
+
+    // CHECK-NOT: {{_.*}} = Len(
+    // CHECK-NOT: {{_.*}} = Lt(
+    // CHECK-NOT: assert(move _
+    const SLICE: &[u32] = &[1, 2, 3];
+    // CHECK: {{_.*}} = const 3_usize;
+    // CHECK: {{_.*}} = const true;
+    // CHECK: assert(const true,
+
+    // CHECK-NOT: [[constant]] = {{copy|move}} (*{{_.*}})[_
+    // CHECK: [[constant]] = copy (*{{_.*}})[1 of 2];
+    let constant = SLICE[1];
+}
diff --git a/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.32bit.diff b/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.32bit.diff
index 3ea49265cee3..132dfd3d8178 100644
--- a/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.32bit.diff
+++ b/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.32bit.diff
@@ -112,16 +112,16 @@
           StorageDead(_10);
           StorageLive(_14);
           _14 = const {ALLOC0: &&SmallStruct};
-          _31 = deref_copy (*_14);
+          _31 = copy (*_14);
           StorageLive(_11);
-          _32 = deref_copy (*_14);
+          _32 = copy (*_14);
 -         _11 = copy ((*_32).0: f32);
 +         _11 = const 9f32;
           StorageLive(_12);
-          _33 = deref_copy (*_14);
+          _33 = copy (*_14);
           _12 = copy ((*_33).1: std::option::Option);
           StorageLive(_13);
-          _34 = deref_copy (*_14);
+          _34 = copy (*_14);
           _13 = copy ((*_34).2: &[f32]);
           StorageDead(_14);
           StorageLive(_15);
@@ -149,16 +149,16 @@
           StorageDead(_22);
           StorageLive(_26);
           _26 = const {ALLOC1: &&BigStruct};
-          _35 = deref_copy (*_26);
+          _35 = copy (*_26);
           StorageLive(_23);
-          _36 = deref_copy (*_26);
+          _36 = copy (*_26);
 -         _23 = copy ((*_36).0: f32);
 +         _23 = const 82f32;
           StorageLive(_24);
-          _37 = deref_copy (*_26);
+          _37 = copy (*_26);
           _24 = copy ((*_37).1: std::option::Option);
           StorageLive(_25);
-          _38 = deref_copy (*_26);
+          _38 = copy (*_26);
           _25 = copy ((*_38).2: &[f32]);
           StorageDead(_26);
           StorageLive(_27);
diff --git a/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.64bit.diff b/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.64bit.diff
index 78a0944975c8..0f3e6f0cb17f 100644
--- a/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.64bit.diff
+++ b/tests/mir-opt/dataflow-const-prop/struct.main.DataflowConstProp.64bit.diff
@@ -112,16 +112,16 @@
           StorageDead(_10);
           StorageLive(_14);
           _14 = const {ALLOC0: &&SmallStruct};
-          _31 = deref_copy (*_14);
+          _31 = copy (*_14);
           StorageLive(_11);
-          _32 = deref_copy (*_14);
+          _32 = copy (*_14);
 -         _11 = copy ((*_32).0: f32);
 +         _11 = const 9f32;
           StorageLive(_12);
-          _33 = deref_copy (*_14);
+          _33 = copy (*_14);
           _12 = copy ((*_33).1: std::option::Option);
           StorageLive(_13);
-          _34 = deref_copy (*_14);
+          _34 = copy (*_14);
           _13 = copy ((*_34).2: &[f32]);
           StorageDead(_14);
           StorageLive(_15);
@@ -149,16 +149,16 @@
           StorageDead(_22);
           StorageLive(_26);
           _26 = const {ALLOC1: &&BigStruct};
-          _35 = deref_copy (*_26);
+          _35 = copy (*_26);
           StorageLive(_23);
-          _36 = deref_copy (*_26);
+          _36 = copy (*_26);
 -         _23 = copy ((*_36).0: f32);
 +         _23 = const 82f32;
           StorageLive(_24);
-          _37 = deref_copy (*_26);
+          _37 = copy (*_26);
           _24 = copy ((*_37).1: std::option::Option);
           StorageLive(_25);
-          _38 = deref_copy (*_26);
+          _38 = copy (*_26);
           _25 = copy ((*_38).2: &[f32]);
           StorageDead(_26);
           StorageLive(_27);
diff --git a/tests/mir-opt/dead-store-elimination/cycle.cycle.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/cycle.cycle.DeadStoreElimination-initial.diff
index ff18df1efcfc..d584de6861c0 100644
--- a/tests/mir-opt/dead-store-elimination/cycle.cycle.DeadStoreElimination-initial.diff
+++ b/tests/mir-opt/dead-store-elimination/cycle.cycle.DeadStoreElimination-initial.diff
@@ -19,10 +19,6 @@
 -         _3 = copy _2;
 -         _2 = copy _1;
 -         _1 = copy _5;
-+         nop;
-+         nop;
-+         nop;
-+         nop;
           _4 = cond() -> [return: bb1, unwind continue];
       }
   
diff --git a/tests/mir-opt/dead-store-elimination/ref.dead_first.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/ref.dead_first.DeadStoreElimination-initial.diff
new file mode 100644
index 000000000000..2a793e24990a
--- /dev/null
+++ b/tests/mir-opt/dead-store-elimination/ref.dead_first.DeadStoreElimination-initial.diff
@@ -0,0 +1,30 @@
+- // MIR for `dead_first` before DeadStoreElimination-initial
++ // MIR for `dead_first` after DeadStoreElimination-initial
+  
+  fn dead_first(_1: &Foo) -> &i32 {
+      debug v => _1;
+      let mut _0: &i32;
+      let mut _2: &i32;
+      let mut _3: &i32;
+      let _4: &i32;
+      scope 1 {
+          debug a => _2;
+      }
+  
+      bb0: {
+          StorageLive(_2);
+-         _2 = &((*_1).2: i32);
++         // DBG: _2 = &((*_1).2: i32);
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = &((*_1).0: i32);
+          _3 = &(*_4);
+          _2 = move _3;
+          StorageDead(_3);
+          StorageDead(_4);
+          _0 = &(*_2);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dead-store-elimination/ref.rs b/tests/mir-opt/dead-store-elimination/ref.rs
new file mode 100644
index 000000000000..2d3200edab9f
--- /dev/null
+++ b/tests/mir-opt/dead-store-elimination/ref.rs
@@ -0,0 +1,31 @@
+//@ test-mir-pass: DeadStoreElimination-initial
+
+pub struct Foo {
+    a: i32,
+    b: i64,
+    c: i32,
+}
+
+// EMIT_MIR ref.tuple.DeadStoreElimination-initial.diff
+pub fn tuple(v: (i32, &Foo)) -> i32 {
+    // CHECK-LABEL: fn tuple
+    // CHECK: debug _dead => [[dead:_[0-9]+]];
+    // CHECK: bb0:
+    // CHECK: DBG: [[dead]] = &((*_3).2: i32)
+    let _dead = &v.1.c;
+    v.1.a
+}
+
+// EMIT_MIR ref.dead_first.DeadStoreElimination-initial.diff
+pub fn dead_first(v: &Foo) -> &i32 {
+    // CHECK-LABEL: fn dead_first
+    // CHECK: debug a => [[var_a:_[0-9]+]];
+    // CHECK: bb0:
+    // CHECK: DBG: [[var_a]] = &((*_1).2: i32)
+    // CHECK: [[tmp_4:_[0-9]+]] = &((*_1).0: i32)
+    // CHECK: [[tmp_3:_[0-9]+]] = &(*[[tmp_4]])
+    // CHECK: [[var_a]] = move [[tmp_3]]
+    let mut a = &v.c;
+    a = &v.a;
+    a
+}
diff --git a/tests/mir-opt/dead-store-elimination/ref.tuple.DeadStoreElimination-initial.diff b/tests/mir-opt/dead-store-elimination/ref.tuple.DeadStoreElimination-initial.diff
new file mode 100644
index 000000000000..3e677d13b89c
--- /dev/null
+++ b/tests/mir-opt/dead-store-elimination/ref.tuple.DeadStoreElimination-initial.diff
@@ -0,0 +1,25 @@
+- // MIR for `tuple` before DeadStoreElimination-initial
++ // MIR for `tuple` after DeadStoreElimination-initial
+  
+  fn tuple(_1: (i32, &Foo)) -> i32 {
+      debug v => _1;
+      let mut _0: i32;
+      let _2: &i32;
+      let mut _3: &Foo;
+      let mut _4: &Foo;
+      scope 1 {
+          debug _dead => _2;
+      }
+  
+      bb0: {
+-         StorageLive(_2);
+-         _3 = copy (_1.1: &Foo);
+-         _2 = &((*_3).2: i32);
++         // DBG: _2 = &((*_3).2: i32);
+          _4 = copy (_1.1: &Foo);
+          _0 = copy ((*_4).0: i32);
+-         StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/dest_prop.remap_debuginfo_locals.DestinationPropagation.diff b/tests/mir-opt/debuginfo/dest_prop.remap_debuginfo_locals.DestinationPropagation.diff
new file mode 100644
index 000000000000..fe00da67e8bb
--- /dev/null
+++ b/tests/mir-opt/debuginfo/dest_prop.remap_debuginfo_locals.DestinationPropagation.diff
@@ -0,0 +1,36 @@
+- // MIR for `remap_debuginfo_locals` before DestinationPropagation
++ // MIR for `remap_debuginfo_locals` after DestinationPropagation
+  
+  fn remap_debuginfo_locals(_1: bool, _2: &bool) -> &bool {
+-     debug c => _3;
++     debug c => _2;
+      let mut _0: &bool;
+      let mut _3: &bool;
+      let mut _4: bool;
+  
+      bb0: {
+-         // DBG: _3 = &_1;
+-         StorageLive(_4);
+-         _4 = copy _1;
+-         _3 = copy _2;
+-         switchInt(copy _4) -> [1: bb1, otherwise: bb2];
++         // DBG: _2 = &_1;
++         nop;
++         nop;
++         nop;
++         switchInt(copy _1) -> [1: bb1, otherwise: bb2];
+      }
+  
+      bb1: {
+          goto -> bb2;
+      }
+  
+      bb2: {
+-         StorageDead(_4);
+-         _0 = copy _3;
++         nop;
++         _0 = copy _2;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/dest_prop.rs b/tests/mir-opt/debuginfo/dest_prop.rs
new file mode 100644
index 000000000000..a734cacb4d29
--- /dev/null
+++ b/tests/mir-opt/debuginfo/dest_prop.rs
@@ -0,0 +1,39 @@
+//@ test-mir-pass: DestinationPropagation
+//@ compile-flags: -g -Zmir-enable-passes=+DeadStoreElimination-initial
+
+#![feature(core_intrinsics, custom_mir)]
+#![crate_type = "lib"]
+
+use std::intrinsics::mir::*;
+
+// EMIT_MIR dest_prop.remap_debuginfo_locals.DestinationPropagation.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn remap_debuginfo_locals(a: bool, b: &bool) -> &bool {
+    // CHECK-LABEL: fn remap_debuginfo_locals(
+    // CHECK: debug c => [[c:_.*]];
+    // CHECK: bb0:
+    // CHECK-NEXT: DBG: [[c]] = &_1;
+    mir! {
+        let _3: &bool;
+        let _4: bool;
+        debug c => _3;
+        {
+            _3 = &a;
+            StorageLive(_4);
+            _4 = a;
+            _3 = b;
+            match _4 {
+                true => bb1,
+                _ => bb2,
+            }
+        }
+        bb1 = {
+            Goto(bb2)
+        }
+        bb2 = {
+            StorageDead(_4);
+            RET = _3;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff b/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff
new file mode 100644
index 000000000000..2d93b1d842fa
--- /dev/null
+++ b/tests/mir-opt/debuginfo/ref_prop.remap_debuginfo_locals.ReferencePropagation.diff
@@ -0,0 +1,33 @@
+- // MIR for `remap_debuginfo_locals` before ReferencePropagation
++ // MIR for `remap_debuginfo_locals` after ReferencePropagation
+  
+  fn remap_debuginfo_locals() -> () {
+      let mut _0: ();
+      let _1: &usize;
+      let mut _2: *const usize;
+      let _3: &usize;
+      let _4: usize;
+      let mut _5: &usize;
+      scope 1 (inlined foo) {
+-         debug a => _1;
++         debug a => _5;
+      }
+  
+      bb0: {
+-         StorageLive(_1);
+-         StorageLive(_2);
+-         StorageLive(_3);
+          _5 = const remap_debuginfo_locals::promoted[0];
+-         _3 = &(*_5);
+-         _2 = &raw const (*_3);
+-         // DBG: _1 = &(*_2);
+-         _1 = &(*_2);
+-         StorageDead(_2);
+-         StorageDead(_3);
+-         StorageDead(_1);
++         // DBG: _5 = &(*_5);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/ref_prop.rs b/tests/mir-opt/debuginfo/ref_prop.rs
new file mode 100644
index 000000000000..60d68ba178a7
--- /dev/null
+++ b/tests/mir-opt/debuginfo/ref_prop.rs
@@ -0,0 +1,30 @@
+//@ test-mir-pass: ReferencePropagation
+//@ compile-flags: -g -Zub_checks=false -Zinline-mir -Zmir-enable-passes=+DeadStoreElimination-initial
+
+#![feature(core_intrinsics, custom_mir)]
+#![crate_type = "lib"]
+
+use std::intrinsics::mir::*;
+
+// EMIT_MIR ref_prop.remap_debuginfo_locals.ReferencePropagation.diff
+pub fn remap_debuginfo_locals() {
+    // CHECK-LABEL: fn remap_debuginfo_locals()
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: bb0:
+    // CHECK-NEXT: [[a]] = const
+    // CHECK-NEXT: DBG: [[a]] = &(*[[a]]);
+    foo(&0);
+}
+
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+#[inline]
+fn foo(x: *const usize) -> &'static usize {
+    mir! {
+        debug a => RET;
+        {
+            RET = &*x;
+            RET = &*x;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/debuginfo/simplifycfg.drop_debuginfo.SimplifyCfg-final.diff b/tests/mir-opt/debuginfo/simplifycfg.drop_debuginfo.SimplifyCfg-final.diff
new file mode 100644
index 000000000000..d4a73351ee4a
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.drop_debuginfo.SimplifyCfg-final.diff
@@ -0,0 +1,26 @@
+- // MIR for `drop_debuginfo` before SimplifyCfg-final
++ // MIR for `drop_debuginfo` after SimplifyCfg-final
+  
+  fn drop_debuginfo(_1: &Foo, _2: bool) -> i32 {
+      debug foo_a => _3;
+      debug foo_b => _4;
+      let mut _0: i32;
+      let mut _3: &i32;
+      let mut _4: &i64;
+  
+      bb0: {
+-         switchInt(copy _2) -> [1: bb1, otherwise: bb2];
+-     }
+- 
+-     bb1: {
+-         // DBG: _3 = &((*_1).0: i32);
+-         goto -> bb2;
+-     }
+- 
+-     bb2: {
+          // DBG: _4 = &((*_1).1: i64);
+          _0 = copy ((*_1).2: i32);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_1.SimplifyCfg-final.diff b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_1.SimplifyCfg-final.diff
new file mode 100644
index 000000000000..1c12358ad893
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_1.SimplifyCfg-final.diff
@@ -0,0 +1,30 @@
+- // MIR for `preserve_debuginfo_1` before SimplifyCfg-final
++ // MIR for `preserve_debuginfo_1` after SimplifyCfg-final
+  
+  fn preserve_debuginfo_1(_1: &Foo, _2: &mut bool) -> i32 {
+      debug foo_a => _3;
+      debug foo_b => _4;
+      debug foo_c => _5;
+      let mut _0: i32;
+      let mut _3: &i32;
+      let mut _4: &i64;
+      let mut _5: &i32;
+  
+      bb0: {
+-         goto -> bb1;
+-     }
+- 
+-     bb1: {
+          (*_2) = const true;
+          // DBG: _3 = &((*_1).0: i32);
+-         goto -> bb2;
+-     }
+- 
+-     bb2: {
+          // DBG: _4 = &((*_1).1: i64);
+          _0 = copy ((*_1).2: i32);
+          // DBG: _5 = &((*_1).2: i32);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_2.SimplifyCfg-final.diff b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_2.SimplifyCfg-final.diff
new file mode 100644
index 000000000000..de8e5612c878
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_2.SimplifyCfg-final.diff
@@ -0,0 +1,29 @@
+- // MIR for `preserve_debuginfo_2` before SimplifyCfg-final
++ // MIR for `preserve_debuginfo_2` after SimplifyCfg-final
+  
+  fn preserve_debuginfo_2(_1: &Foo) -> i32 {
+      debug foo_a => _2;
+      debug foo_b => _3;
+      debug foo_c => _4;
+      let mut _0: i32;
+      let mut _2: &i32;
+      let mut _3: &i64;
+      let mut _4: &i32;
+  
+      bb0: {
+-         goto -> bb1;
+-     }
+- 
+-     bb1: {
+          // DBG: _2 = &((*_1).0: i32);
+-         goto -> bb2;
+-     }
+- 
+-     bb2: {
+          // DBG: _3 = &((*_1).1: i64);
+          _0 = copy ((*_1).2: i32);
+          // DBG: _4 = &((*_1).2: i32);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_3.SimplifyCfg-final.diff b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_3.SimplifyCfg-final.diff
new file mode 100644
index 000000000000..11372a262a76
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_3.SimplifyCfg-final.diff
@@ -0,0 +1,37 @@
+- // MIR for `preserve_debuginfo_3` before SimplifyCfg-final
++ // MIR for `preserve_debuginfo_3` after SimplifyCfg-final
+  
+  fn preserve_debuginfo_3(_1: &Foo, _2: bool) -> i32 {
+      debug foo_a => _3;
+      debug foo_b => _4;
+      debug foo_c => _5;
+      let mut _0: i32;
+      let mut _3: &i32;
+      let mut _4: &i64;
+      let mut _5: &i32;
+  
+      bb0: {
+-         switchInt(copy _2) -> [1: bb1, otherwise: bb2];
++         switchInt(copy _2) -> [1: bb2, otherwise: bb1];
+      }
+  
+      bb1: {
+-         // DBG: _3 = &((*_1).0: i32);
+-         goto -> bb3;
+-     }
+- 
+-     bb2: {
+          // DBG: _4 = &((*_1).1: i64);
+          _0 = copy ((*_1).2: i32);
+          return;
+      }
+  
+-     bb3: {
++     bb2: {
++         // DBG: _3 = &((*_1).0: i32);
+          // DBG: _5 = &((*_1).2: i32);
+          _0 = copy ((*_1).0: i32);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_identical_succs.SimplifyCfg-final.diff b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_identical_succs.SimplifyCfg-final.diff
new file mode 100644
index 000000000000..0c6a75237d8b
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.preserve_debuginfo_identical_succs.SimplifyCfg-final.diff
@@ -0,0 +1,29 @@
+- // MIR for `preserve_debuginfo_identical_succs` before SimplifyCfg-final
++ // MIR for `preserve_debuginfo_identical_succs` after SimplifyCfg-final
+  
+  fn preserve_debuginfo_identical_succs(_1: &Foo, _2: bool) -> i32 {
+      debug foo_a => _3;
+      debug foo_b => _4;
+      debug foo_c => _5;
+      let mut _0: i32;
+      let mut _3: &i32;
+      let mut _4: &i64;
+      let mut _5: &i32;
+  
+      bb0: {
+-         switchInt(copy _2) -> [1: bb1, otherwise: bb1];
+-     }
+- 
+-     bb1: {
+          // DBG: _3 = &((*_1).0: i32);
+-         goto -> bb2;
+-     }
+- 
+-     bb2: {
+          // DBG: _4 = &((*_1).1: i64);
+          _0 = copy ((*_1).2: i32);
+          // DBG: _5 = &((*_1).2: i32);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/debuginfo/simplifycfg.rs b/tests/mir-opt/debuginfo/simplifycfg.rs
new file mode 100644
index 000000000000..2bd510fd3b9d
--- /dev/null
+++ b/tests/mir-opt/debuginfo/simplifycfg.rs
@@ -0,0 +1,207 @@
+//@ test-mir-pass: SimplifyCfg-final
+//@ compile-flags: -Zmir-enable-passes=+DeadStoreElimination-initial
+
+#![feature(core_intrinsics, custom_mir)]
+#![crate_type = "lib"]
+
+use std::intrinsics::mir::*;
+
+pub struct Foo {
+    a: i32,
+    b: i64,
+    c: i32,
+}
+
+// EMIT_MIR simplifycfg.drop_debuginfo.SimplifyCfg-final.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn drop_debuginfo(foo: &Foo, c: bool) -> i32 {
+    // CHECK-LABEL: fn drop_debuginfo
+    // CHECK: debug foo_b => [[foo_b:_[0-9]+]];
+    // CHECK: bb0: {
+    // CHECK-NEXT: DBG: [[foo_b]] = &((*_1).1: i64)
+    // CHECK-NEXT: _0 = copy ((*_1).2: i32);
+    // CHECK-NEXT: return;
+    mir! {
+        let _foo_a: &i32;
+        let _foo_b: &i64;
+        debug foo_a => _foo_a;
+        debug foo_b => _foo_b;
+        {
+            match c {
+                true => tmp,
+                _ => ret,
+            }
+        }
+        tmp = {
+            // Because we don't know if `c` is always true, we must drop this debuginfo.
+            _foo_a = &(*foo).a;
+            Goto(ret)
+        }
+        ret = {
+            _foo_b = &(*foo).b;
+            RET = (*foo).c;
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR simplifycfg.preserve_debuginfo_1.SimplifyCfg-final.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn preserve_debuginfo_1(foo: &Foo, v: &mut bool) -> i32 {
+    // CHECK-LABEL: fn preserve_debuginfo_1
+    // CHECK: debug foo_a => [[foo_a:_[0-9]+]];
+    // CHECK: debug foo_b => [[foo_b:_[0-9]+]];
+    // CHECK: debug foo_c => [[foo_c:_[0-9]+]];
+    // CHECK: bb0: {
+    // CHECK-NEXT: (*_2) = const true;
+    // CHECK-NEXT: DBG: [[foo_a]] = &((*_1).0: i32)
+    // CHECK-NEXT: DBG: [[foo_b]] = &((*_1).1: i64)
+    // CHECK-NEXT: _0 = copy ((*_1).2: i32);
+    // CHECK-NEXT: DBG: [[foo_c]] = &((*_1).2: i32)
+    // CHECK-NEXT: return;
+    mir! {
+        let _foo_a: &i32;
+        let _foo_b: &i64;
+        let _foo_c: &i32;
+        debug foo_a => _foo_a;
+        debug foo_b => _foo_b;
+        debug foo_c => _foo_c;
+        {
+            Goto(tmp)
+        }
+        tmp = {
+            *v = true;
+            _foo_a = &(*foo).a;
+            Goto(ret)
+        }
+        ret = {
+            _foo_b = &(*foo).b;
+            RET = (*foo).c;
+            _foo_c = &(*foo).c;
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR simplifycfg.preserve_debuginfo_2.SimplifyCfg-final.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn preserve_debuginfo_2(foo: &Foo) -> i32 {
+    // CHECK-LABEL: fn preserve_debuginfo_2
+    // CHECK: debug foo_a => [[foo_a:_[0-9]+]];
+    // CHECK: debug foo_b => [[foo_b:_[0-9]+]];
+    // CHECK: debug foo_c => [[foo_c:_[0-9]+]];
+    // CHECK: bb0: {
+    // CHECK-NEXT: DBG: [[foo_a]] = &((*_1).0: i32)
+    // CHECK-NEXT: DBG: [[foo_b]] = &((*_1).1: i64)
+    // CHECK-NEXT: _0 = copy ((*_1).2: i32);
+    // CHECK-NEXT: DBG: [[foo_c]] = &((*_1).2: i32)
+    // CHECK-NEXT: return;
+    mir! {
+        let _foo_a: &i32;
+        let _foo_b: &i64;
+        let _foo_c: &i32;
+        debug foo_a => _foo_a;
+        debug foo_b => _foo_b;
+        debug foo_c => _foo_c;
+        {
+            Goto(tmp)
+        }
+        tmp = {
+            _foo_a = &(*foo).a;
+            Goto(ret)
+        }
+        ret = {
+            _foo_b = &(*foo).b;
+            RET = (*foo).c;
+            _foo_c = &(*foo).c;
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR simplifycfg.preserve_debuginfo_3.SimplifyCfg-final.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn preserve_debuginfo_3(foo: &Foo, c: bool) -> i32 {
+    // CHECK-LABEL: fn preserve_debuginfo_3
+    // CHECK: debug foo_a => [[foo_a:_[0-9]+]];
+    // CHECK: debug foo_b => [[foo_b:_[0-9]+]];
+    // CHECK: debug foo_c => [[foo_c:_[0-9]+]];
+    // CHECK: bb0: {
+    // CHECK-NEXT: switchInt(copy _2) -> [1: bb2, otherwise: bb1];
+    // CHECK: bb1: {
+    // CHECK-NEXT: DBG: [[foo_b]] = &((*_1).1: i64)
+    // CHECK-NEXT: _0 = copy ((*_1).2: i32);
+    // CHECK-NEXT: return;
+    // CHECK: bb2: {
+    // CHECK-NEXT: DBG: [[foo_a]] = &((*_1).0: i32)
+    // CHECK-NEXT: DBG: [[foo_c]] = &((*_1).2: i32)
+    // CHECK-NEXT: _0 = copy ((*_1).0: i32);
+    // CHECK-NEXT: return;
+    mir! {
+        let _foo_a: &i32;
+        let _foo_b: &i64;
+        let _foo_c: &i32;
+        debug foo_a => _foo_a;
+        debug foo_b => _foo_b;
+        debug foo_c => _foo_c;
+        {
+            match c {
+                true => tmp,
+                _ => ret,
+            }
+        }
+        tmp = {
+            _foo_a = &(*foo).a;
+            Goto(ret_1)
+        }
+        ret = {
+            _foo_b = &(*foo).b;
+            RET = (*foo).c;
+            Return()
+        }
+        ret_1 = {
+            _foo_c = &(*foo).c;
+            RET = (*foo).a;
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR simplifycfg.preserve_debuginfo_identical_succs.SimplifyCfg-final.diff
+#[custom_mir(dialect = "runtime", phase = "post-cleanup")]
+pub fn preserve_debuginfo_identical_succs(foo: &Foo, c: bool) -> i32 {
+    // CHECK-LABEL: fn preserve_debuginfo_identical_succs
+    // CHECK: debug foo_a => [[foo_a:_[0-9]+]];
+    // CHECK: debug foo_b => [[foo_b:_[0-9]+]];
+    // CHECK: debug foo_c => [[foo_c:_[0-9]+]];
+    // CHECK: bb0: {
+    // CHECK-NEXT: DBG: [[foo_a]] = &((*_1).0: i32)
+    // CHECK-NEXT: DBG: [[foo_b]] = &((*_1).1: i64)
+    // CHECK-NEXT: _0 = copy ((*_1).2: i32);
+    // CHECK-NEXT: DBG: [[foo_c]] = &((*_1).2: i32)
+    // CHECK-NEXT: return;
+    mir! {
+        let _foo_a: &i32;
+        let _foo_b: &i64;
+        let _foo_c: &i32;
+        debug foo_a => _foo_a;
+        debug foo_b => _foo_b;
+        debug foo_c => _foo_c;
+        {
+            match c {
+                true => tmp,
+                _ => tmp,
+            }
+        }
+        tmp = {
+            _foo_a = &(*foo).a;
+            Goto(ret)
+        }
+        ret = {
+            _foo_b = &(*foo).b;
+            RET = (*foo).c;
+            _foo_c = &(*foo).c;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-abort.diff
new file mode 100644
index 000000000000..e80660f176b7
--- /dev/null
+++ b/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-abort.diff
@@ -0,0 +1,19 @@
+- // MIR for `rewrap` before DestinationPropagation
++ // MIR for `rewrap` after DestinationPropagation
+  
+  fn rewrap() -> (u8,) {
+      let mut _0: (u8,);
+      let mut _1: (u8,);
+      let mut _2: (u8,);
+  
+      bb0: {
+-         (_1.0: u8) = const 0_u8;
+-         _0 = copy _1;
++         (_0.0: u8) = const 0_u8;
++         nop;
+          _2 = (copy (_0.0: u8),);
+          _0 = copy _2;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-unwind.diff
new file mode 100644
index 000000000000..e80660f176b7
--- /dev/null
+++ b/tests/mir-opt/dest-prop/aggregate.rewrap.DestinationPropagation.panic-unwind.diff
@@ -0,0 +1,19 @@
+- // MIR for `rewrap` before DestinationPropagation
++ // MIR for `rewrap` after DestinationPropagation
+  
+  fn rewrap() -> (u8,) {
+      let mut _0: (u8,);
+      let mut _1: (u8,);
+      let mut _2: (u8,);
+  
+      bb0: {
+-         (_1.0: u8) = const 0_u8;
+-         _0 = copy _1;
++         (_0.0: u8) = const 0_u8;
++         nop;
+          _2 = (copy (_0.0: u8),);
+          _0 = copy _2;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/aggregate.rs b/tests/mir-opt/dest-prop/aggregate.rs
new file mode 100644
index 000000000000..636852159eb4
--- /dev/null
+++ b/tests/mir-opt/dest-prop/aggregate.rs
@@ -0,0 +1,51 @@
+//@ test-mir-pass: DestinationPropagation
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
+#![feature(custom_mir, core_intrinsics)]
+#![allow(internal_features)]
+
+use std::intrinsics::mir::*;
+use std::mem::MaybeUninit;
+
+fn dump_var(_: T) {}
+
+// EMIT_MIR aggregate.rewrap.DestinationPropagation.diff
+#[custom_mir(dialect = "runtime")]
+fn rewrap() -> (u8,) {
+    // CHECK-LABEL: fn rewrap(
+    // CHECK: (_0.0: u8) = const 0_u8;
+    // CHECK: _2 = (copy (_0.0: u8),);
+    // CHECK: _0 = copy _2;
+    mir! {
+        let _1: (u8,);
+        let _2: (u8,);
+        {
+            _1.0 = 0;
+            RET = _1;
+            _2 = (RET.0, );
+            RET = _2;
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR aggregate.swap.DestinationPropagation.diff
+#[custom_mir(dialect = "runtime")]
+fn swap() -> (MaybeUninit<[u8; 10]>, MaybeUninit<[u8; 10]>) {
+    // CHECK-LABEL: fn swap(
+    // CHECK: _0 = const
+    // CHECK: _2 = copy _0;
+    // CHECK: _0 = (copy (_2.1: {{.*}}), copy (_2.0: {{.*}}));
+    mir! {
+        let _1: (MaybeUninit<[u8; 10]>, MaybeUninit<[u8; 10]>);
+        let _2: (MaybeUninit<[u8; 10]>, MaybeUninit<[u8; 10]>);
+        let _3: ();
+        {
+            _1 = const { (MaybeUninit::new([0; 10]), MaybeUninit::new([1; 10])) };
+            _2 = _1;
+            _1 = (_2.1, _2.0);
+            RET = _1;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-abort.diff
new file mode 100644
index 000000000000..3aaad3aaf692
--- /dev/null
+++ b/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-abort.diff
@@ -0,0 +1,22 @@
+- // MIR for `swap` before DestinationPropagation
++ // MIR for `swap` after DestinationPropagation
+  
+  fn swap() -> (MaybeUninit<[u8; 10]>, MaybeUninit<[u8; 10]>) {
+      let mut _0: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _1: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _2: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _3: ();
+  
+      bb0: {
+-         _1 = const swap::{constant#6};
+-         _2 = copy _1;
+-         _1 = (copy (_2.1: std::mem::MaybeUninit<[u8; 10]>), copy (_2.0: std::mem::MaybeUninit<[u8; 10]>));
+-         _0 = copy _1;
++         _0 = const swap::{constant#6};
++         _2 = copy _0;
++         _0 = (copy (_2.1: std::mem::MaybeUninit<[u8; 10]>), copy (_2.0: std::mem::MaybeUninit<[u8; 10]>));
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-unwind.diff
new file mode 100644
index 000000000000..3aaad3aaf692
--- /dev/null
+++ b/tests/mir-opt/dest-prop/aggregate.swap.DestinationPropagation.panic-unwind.diff
@@ -0,0 +1,22 @@
+- // MIR for `swap` before DestinationPropagation
++ // MIR for `swap` after DestinationPropagation
+  
+  fn swap() -> (MaybeUninit<[u8; 10]>, MaybeUninit<[u8; 10]>) {
+      let mut _0: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _1: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _2: (std::mem::MaybeUninit<[u8; 10]>, std::mem::MaybeUninit<[u8; 10]>);
+      let mut _3: ();
+  
+      bb0: {
+-         _1 = const swap::{constant#6};
+-         _2 = copy _1;
+-         _1 = (copy (_2.1: std::mem::MaybeUninit<[u8; 10]>), copy (_2.0: std::mem::MaybeUninit<[u8; 10]>));
+-         _0 = copy _1;
++         _0 = const swap::{constant#6};
++         _2 = copy _0;
++         _0 = (copy (_2.1: std::mem::MaybeUninit<[u8; 10]>), copy (_2.0: std::mem::MaybeUninit<[u8; 10]>));
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff
index 5d8aaedae374..a2f10be31a9e 100644
--- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff
+++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-abort.diff
@@ -8,25 +8,23 @@
       let _5: ();
       let mut _6: i32;
       scope 1 {
--         debug x => _1;
-+         debug x => _6;
+          debug x => _1;
           let _2: i32;
           scope 2 {
 -             debug y => _2;
-+             debug y => _6;
++             debug y => _1;
               let _3: i32;
               scope 3 {
 -                 debug z => _3;
-+                 debug z => _6;
++                 debug z => _1;
               }
           }
       }
   
       bb0: {
 -         StorageLive(_1);
--         _1 = val() -> [return: bb1, unwind unreachable];
 +         nop;
-+         _6 = val() -> [return: bb1, unwind unreachable];
+          _1 = val() -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
@@ -49,9 +47,10 @@
           StorageLive(_5);
 -         StorageLive(_6);
 -         _6 = copy _1;
+-         _5 = std::mem::drop::(move _6) -> [return: bb2, unwind unreachable];
 +         nop;
 +         nop;
-          _5 = std::mem::drop::(move _6) -> [return: bb2, unwind unreachable];
++         _5 = std::mem::drop::(move _1) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
diff --git a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff
index 05c9bcc1d737..a08488615b15 100644
--- a/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff
+++ b/tests/mir-opt/dest-prop/cycle.main.DestinationPropagation.panic-unwind.diff
@@ -8,25 +8,23 @@
       let _5: ();
       let mut _6: i32;
       scope 1 {
--         debug x => _1;
-+         debug x => _6;
+          debug x => _1;
           let _2: i32;
           scope 2 {
 -             debug y => _2;
-+             debug y => _6;
++             debug y => _1;
               let _3: i32;
               scope 3 {
 -                 debug z => _3;
-+                 debug z => _6;
++                 debug z => _1;
               }
           }
       }
   
       bb0: {
 -         StorageLive(_1);
--         _1 = val() -> [return: bb1, unwind continue];
 +         nop;
-+         _6 = val() -> [return: bb1, unwind continue];
+          _1 = val() -> [return: bb1, unwind continue];
       }
   
       bb1: {
@@ -49,9 +47,10 @@
           StorageLive(_5);
 -         StorageLive(_6);
 -         _6 = copy _1;
+-         _5 = std::mem::drop::(move _6) -> [return: bb2, unwind continue];
 +         nop;
 +         nop;
-          _5 = std::mem::drop::(move _6) -> [return: bb2, unwind continue];
++         _5 = std::mem::drop::(move _1) -> [return: bb2, unwind continue];
       }
   
       bb2: {
diff --git a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir
index eb4209731c62..15061da81206 100644
--- a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir
+++ b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-abort.mir
@@ -7,16 +7,16 @@ fn f(_1: usize) -> usize {
     let mut _3: usize;
     let mut _4: usize;
     scope 1 {
-        debug b => _3;
+        debug b => _2;
     }
 
     bb0: {
         nop;
-        _3 = copy _1;
+        _2 = copy _1;
         _1 = const 5_usize;
         nop;
         nop;
-        _1 = move _3;
+        _1 = move _2;
         nop;
         nop;
         nop;
diff --git a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir
index fe9a7376a585..ddfe4dc5b3eb 100644
--- a/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir
+++ b/tests/mir-opt/dest-prop/dead_stores_79191.f.DestinationPropagation.after.panic-unwind.mir
@@ -7,16 +7,16 @@ fn f(_1: usize) -> usize {
     let mut _3: usize;
     let mut _4: usize;
     scope 1 {
-        debug b => _3;
+        debug b => _2;
     }
 
     bb0: {
         nop;
-        _3 = copy _1;
+        _2 = copy _1;
         _1 = const 5_usize;
         nop;
         nop;
-        _1 = move _3;
+        _1 = move _2;
         nop;
         nop;
         nop;
diff --git a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir
index eb4209731c62..15061da81206 100644
--- a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir
+++ b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-abort.mir
@@ -7,16 +7,16 @@ fn f(_1: usize) -> usize {
     let mut _3: usize;
     let mut _4: usize;
     scope 1 {
-        debug b => _3;
+        debug b => _2;
     }
 
     bb0: {
         nop;
-        _3 = copy _1;
+        _2 = copy _1;
         _1 = const 5_usize;
         nop;
         nop;
-        _1 = move _3;
+        _1 = move _2;
         nop;
         nop;
         nop;
diff --git a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir
index fe9a7376a585..ddfe4dc5b3eb 100644
--- a/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir
+++ b/tests/mir-opt/dest-prop/dead_stores_better.f.DestinationPropagation.after.panic-unwind.mir
@@ -7,16 +7,16 @@ fn f(_1: usize) -> usize {
     let mut _3: usize;
     let mut _4: usize;
     scope 1 {
-        debug b => _3;
+        debug b => _2;
     }
 
     bb0: {
         nop;
-        _3 = copy _1;
+        _2 = copy _1;
         _1 = const 5_usize;
         nop;
         nop;
-        _1 = move _3;
+        _1 = move _2;
         nop;
         nop;
         nop;
diff --git a/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff
new file mode 100644
index 000000000000..e9fbcf20a722
--- /dev/null
+++ b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-abort.diff
@@ -0,0 +1,43 @@
+- // MIR for `nrvo` before DestinationPropagation
++ // MIR for `nrvo` after DestinationPropagation
+  
+  fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] {
+      debug init => _1;
+      let mut _0: [u8; 1024];
+      let mut _2: [u8; 1024];
+      let _3: ();
+      let mut _4: for<'a> fn(&'a mut [u8; 1024]);
+      let mut _5: &mut [u8; 1024];
+      let mut _6: &mut [u8; 1024];
+      scope 1 {
+          debug buf => _2;
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = [const 0_u8; 1024];
+          StorageLive(_3);
+-         StorageLive(_4);
+-         _4 = copy _1;
++         nop;
++         nop;
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = &mut _2;
+          _5 = &mut (*_6);
+-         _3 = move _4(move _5) -> [return: bb1, unwind unreachable];
++         _3 = move _1(move _5) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_5);
+-         StorageDead(_4);
++         nop;
+          StorageDead(_6);
+          StorageDead(_3);
+          _0 = copy _2;
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff
new file mode 100644
index 000000000000..95d5fe1b9304
--- /dev/null
+++ b/tests/mir-opt/dest-prop/nrvo_borrowed.nrvo.DestinationPropagation.panic-unwind.diff
@@ -0,0 +1,43 @@
+- // MIR for `nrvo` before DestinationPropagation
++ // MIR for `nrvo` after DestinationPropagation
+  
+  fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] {
+      debug init => _1;
+      let mut _0: [u8; 1024];
+      let mut _2: [u8; 1024];
+      let _3: ();
+      let mut _4: for<'a> fn(&'a mut [u8; 1024]);
+      let mut _5: &mut [u8; 1024];
+      let mut _6: &mut [u8; 1024];
+      scope 1 {
+          debug buf => _2;
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = [const 0_u8; 1024];
+          StorageLive(_3);
+-         StorageLive(_4);
+-         _4 = copy _1;
++         nop;
++         nop;
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = &mut _2;
+          _5 = &mut (*_6);
+-         _3 = move _4(move _5) -> [return: bb1, unwind continue];
++         _3 = move _1(move _5) -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          StorageDead(_5);
+-         StorageDead(_4);
++         nop;
+          StorageDead(_6);
+          StorageDead(_3);
+          _0 = copy _2;
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/nrvo_borrowed.rs b/tests/mir-opt/dest-prop/nrvo_borrowed.rs
new file mode 100644
index 000000000000..6f3076d4c13e
--- /dev/null
+++ b/tests/mir-opt/dest-prop/nrvo_borrowed.rs
@@ -0,0 +1,16 @@
+// skip-filecheck
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ test-mir-pass: DestinationPropagation
+
+// EMIT_MIR nrvo_borrowed.nrvo.DestinationPropagation.diff
+fn nrvo(init: fn(&mut [u8; 1024])) -> [u8; 1024] {
+    let mut buf = [0; 1024];
+    init(&mut buf);
+    buf
+}
+
+fn main() {
+    let _ = nrvo(|buf| {
+        buf[4] = 4;
+    });
+}
diff --git a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs
new file mode 100644
index 000000000000..17e6450278ac
--- /dev/null
+++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.rs
@@ -0,0 +1,28 @@
+// This is a miscompilation, #111005 to track
+
+//@ test-mir-pass: DestinationPropagation
+
+#![feature(custom_mir, core_intrinsics)]
+extern crate core;
+use core::intrinsics::mir::*;
+
+// EMIT_MIR nrvo_miscompile_111005.wrong.DestinationPropagation.diff
+#[custom_mir(dialect = "runtime", phase = "initial")]
+pub fn wrong(arg: char) -> char {
+    // CHECK-LABEL: fn wrong(
+    // CHECK: _0 = copy _1;
+    // CHECK-NEXT: _1 = const 'b';
+    // CHECK-NEXT: return;
+    mir! {
+        {
+            let temp = arg;
+            RET = temp;
+            temp = 'b';
+            Return()
+        }
+    }
+}
+
+fn main() {
+    assert_eq!(wrong('a'), 'a');
+}
diff --git a/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff
new file mode 100644
index 000000000000..60cf9236f6c2
--- /dev/null
+++ b/tests/mir-opt/dest-prop/nrvo_miscompile_111005.wrong.DestinationPropagation.diff
@@ -0,0 +1,18 @@
+- // MIR for `wrong` before DestinationPropagation
++ // MIR for `wrong` after DestinationPropagation
+  
+  fn wrong(_1: char) -> char {
+      let mut _0: char;
+      let mut _2: char;
+  
+      bb0: {
+-         _2 = copy _1;
+-         _0 = copy _2;
+-         _2 = const 'b';
++         nop;
++         _0 = copy _1;
++         _1 = const 'b';
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-abort.diff b/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-abort.diff
deleted file mode 100644
index ef418798faaf..000000000000
--- a/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-abort.diff
+++ /dev/null
@@ -1,27 +0,0 @@
-- // MIR for `main` before DestinationPropagation
-+ // MIR for `main` after DestinationPropagation
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: main::Un;
-      let mut _2: u32;
-      scope 1 {
-          debug un => _1;
-          scope 3 (inlined std::mem::drop::) {
-              debug _x => _2;
-          }
-      }
-      scope 2 (inlined val) {
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          _1 = Un { us: const 1_u32 };
-          StorageLive(_2);
-          _2 = copy (_1.0: u32);
-          StorageDead(_2);
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-unwind.diff b/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-unwind.diff
deleted file mode 100644
index ef418798faaf..000000000000
--- a/tests/mir-opt/dest-prop/union.main.DestinationPropagation.panic-unwind.diff
+++ /dev/null
@@ -1,27 +0,0 @@
-- // MIR for `main` before DestinationPropagation
-+ // MIR for `main` after DestinationPropagation
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: main::Un;
-      let mut _2: u32;
-      scope 1 {
-          debug un => _1;
-          scope 3 (inlined std::mem::drop::) {
-              debug _x => _2;
-          }
-      }
-      scope 2 (inlined val) {
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          _1 = Un { us: const 1_u32 };
-          StorageLive(_2);
-          _2 = copy (_1.0: u32);
-          StorageDead(_2);
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/dest-prop/union.rs b/tests/mir-opt/dest-prop/union.rs
deleted file mode 100644
index 85eded099803..000000000000
--- a/tests/mir-opt/dest-prop/union.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
-//! Tests that we can propagate into places that are projections into unions
-//@ compile-flags: -Zunsound-mir-opts -C debuginfo=full
-fn val() -> u32 {
-    1
-}
-
-// EMIT_MIR union.main.DestinationPropagation.diff
-fn main() {
-    // CHECK-LABEL: fn main(
-    // CHECK: {{_.*}} = Un { us: const 1_u32 };
-    union Un {
-        us: u32,
-    }
-
-    let un = Un { us: val() };
-
-    drop(unsafe { un.us });
-}
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.rs b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.rs
index 4368933dc627..5534a45f19d6 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.rs
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.rs
@@ -1,10 +1,14 @@
-// skip-filecheck
-//@ compile-flags: -Zmir-enable-passes=+Inline,+GVN --crate-type lib
+//@ test-mir-pass: GVN
+//@ compile-flags: -Zinline-mir --crate-type lib
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // EMIT_MIR dont_reset_cast_kind_without_updating_operand.test.GVN.diff
 
 fn test() {
+    // CHECK-LABEL: fn test(
+    // CHECK: debug slf => [[SLF:_.*]];
+    // CHECK: debug _x => [[X:_.*]];
+    // CHECK: [[X]] = copy [[SLF]] as *mut () (PtrToPtr);
     let vp_ctx: &Box<()> = &Box::new(());
     let slf: *const () = &raw const **vp_ctx;
     let bytes = std::ptr::slice_from_raw_parts(slf, 1);
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
index 25ffff619e60..38beb81e1ead 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-abort.diff
@@ -6,24 +6,28 @@
       let _1: &std::boxed::Box<()>;
       let _2: &std::boxed::Box<()>;
       let _3: std::boxed::Box<()>;
-      let mut _6: *const ();
-      let mut _8: *const [()];
-      let mut _9: std::boxed::Box<()>;
-      let mut _10: *const ();
-      let mut _23: usize;
+      let mut _4: ();
+      let mut _7: *const ();
+      let mut _9: *const [()];
+      let mut _10: std::boxed::Box<()>;
+      let mut _11: *const ();
+      let mut _16: usize;
+      let mut _17: usize;
+      let mut _27: usize;
       scope 1 {
           debug vp_ctx => _1;
-          let _4: *const ();
+          let _5: *const ();
           scope 2 {
-              debug slf => _10;
-              let _5: *const [()];
+              debug slf => _5;
+              let _6: *const [()];
               scope 3 {
-                  debug bytes => _5;
-                  let _7: *mut ();
+                  debug bytes => _6;
+                  let _8: *mut ();
                   scope 4 {
-                      debug _x => _7;
+                      debug _x => _8;
                   }
                   scope 18 (inlined foo) {
+                      let mut _28: *const [()];
                   }
               }
               scope 16 (inlined slice_from_raw_parts::<()>) {
@@ -33,21 +37,22 @@
           }
       }
       scope 5 (inlined Box::<()>::new) {
-          let mut _11: usize;
-          let mut _12: usize;
-          let mut _13: *mut u8;
+          let mut _12: *mut u8;
+          let mut _13: *const ();
+          let mut _14: std::ptr::NonNull<()>;
+          let mut _15: std::ptr::Unique<()>;
           scope 6 (inlined alloc::alloc::exchange_malloc) {
-              let _14: std::alloc::Layout;
-              let mut _15: std::result::Result, std::alloc::AllocError>;
-              let mut _16: isize;
-              let mut _18: !;
+              let _18: std::alloc::Layout;
+              let mut _19: std::result::Result, std::alloc::AllocError>;
+              let mut _20: isize;
+              let mut _22: !;
               scope 7 {
-                  let _17: std::ptr::NonNull<[u8]>;
+                  let _21: std::ptr::NonNull<[u8]>;
                   scope 8 {
                       scope 11 (inlined NonNull::<[u8]>::as_mut_ptr) {
                           scope 12 (inlined NonNull::<[u8]>::as_non_null_ptr) {
                               scope 13 (inlined NonNull::<[u8]>::cast::) {
-                                  let mut _22: *mut [u8];
+                                  let mut _26: *mut [u8];
                                   scope 14 (inlined NonNull::<[u8]>::as_ptr) {
                                   }
                               }
@@ -60,31 +65,39 @@
                   }
               }
               scope 9 (inlined #[track_caller] Layout::from_size_align_unchecked) {
-                  let mut _19: bool;
-                  let _20: ();
-                  let mut _21: std::ptr::Alignment;
+                  let mut _23: bool;
+                  let _24: ();
+                  let mut _25: std::ptr::Alignment;
               }
           }
       }
   
       bb0: {
           StorageLive(_1);
-          StorageLive(_2);
+-         StorageLive(_2);
++         nop;
           StorageLive(_3);
-          StorageLive(_11);
+          StorageLive(_4);
+-         _4 = ();
++         _4 = const ();
           StorageLive(_12);
           StorageLive(_13);
--         _11 = SizeOf(());
--         _12 = AlignOf(());
-+         _11 = const 0_usize;
-+         _12 = const 1_usize;
           StorageLive(_14);
+          StorageLive(_15);
           StorageLive(_16);
+-         _16 = const <() as std::mem::SizedTypeProperties>::SIZE;
++         _16 = const 0_usize;
           StorageLive(_17);
-          StorageLive(_19);
-          _19 = const false;
--         switchInt(move _19) -> [0: bb6, otherwise: bb5];
-+         switchInt(const false) -> [0: bb6, otherwise: bb5];
+-         _17 = const <() as std::mem::SizedTypeProperties>::ALIGN;
++         _17 = const 1_usize;
+          StorageLive(_18);
+          StorageLive(_20);
+          StorageLive(_21);
+          StorageLive(_22);
+          StorageLive(_24);
+          StorageLive(_23);
+          _23 = UbChecks();
+          switchInt(move _23) -> [0: bb6, otherwise: bb5];
       }
   
       bb1: {
@@ -98,77 +111,98 @@
       }
   
       bb3: {
--         _18 = handle_alloc_error(move _14) -> unwind unreachable;
-+         _18 = handle_alloc_error(const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}) -> unwind unreachable;
+-         _22 = handle_alloc_error(move _18) -> unwind unreachable;
++         _22 = handle_alloc_error(const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}) -> unwind unreachable;
       }
   
       bb4: {
-          _17 = copy ((_15 as Ok).0: std::ptr::NonNull<[u8]>);
-          StorageLive(_22);
-          _22 = copy _17 as *mut [u8] (Transmute);
-          _13 = copy _22 as *mut u8 (PtrToPtr);
+          _21 = copy ((_19 as Ok).0: std::ptr::NonNull<[u8]>);
+-         StorageLive(_26);
++         nop;
+          _26 = copy _21 as *mut [u8] (Transmute);
+          _12 = copy _26 as *mut u8 (PtrToPtr);
+-         StorageDead(_26);
++         nop;
+          StorageDead(_19);
+          StorageDead(_24);
           StorageDead(_22);
-          StorageDead(_15);
+          StorageDead(_21);
+          StorageDead(_20);
+          StorageDead(_18);
           StorageDead(_17);
           StorageDead(_16);
+-         _13 = copy _12 as *const () (PtrToPtr);
++         _13 = copy _26 as *const () (PtrToPtr);
+          _14 = NonNull::<()> { pointer: copy _13 };
+          _15 = Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
+          _3 = Box::<()>(move _15, const std::alloc::Global);
+-         (*_13) = move _4;
++         (*_13) = const ();
+          StorageDead(_15);
           StorageDead(_14);
-          _3 = ShallowInitBox(move _13, ());
           StorageDead(_13);
           StorageDead(_12);
-          StorageDead(_11);
+          StorageDead(_4);
           _2 = &_3;
-          _1 = copy _2;
-          StorageDead(_2);
-          StorageLive(_4);
--         _9 = deref_copy _3;
-+         _9 = copy _3;
-          _10 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
-          _4 = copy _10;
+          _1 = &(*_2);
+-         StorageDead(_2);
 -         StorageLive(_5);
+-         _10 = copy (*_1);
++         nop;
++         nop;
++         _10 = copy (*_2);
+          _11 = copy ((_10.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
+          _5 = &raw const (*_11);
+-         StorageLive(_6);
 +         nop;
-          StorageLive(_6);
--         _6 = copy _4;
-+         _6 = copy _10;
-          StorageLive(_23);
-          _23 = const 1_usize;
--         _5 = *const [()] from (copy _6, copy _23);
-+         _5 = *const [()] from (copy _10, const 1_usize);
-          StorageDead(_23);
-          StorageDead(_6);
           StorageLive(_7);
-          StorageLive(_8);
-          _8 = copy _5;
--         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
-          StorageDead(_8);
+          _7 = copy _5;
+          StorageLive(_27);
+          _27 = const 1_usize;
+-         _6 = *const [()] from (copy _7, copy _27);
++         _6 = *const [()] from (copy _5, const 1_usize);
+          StorageDead(_27);
           StorageDead(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _6;
+          StorageLive(_28);
+-         _28 = copy _9;
+-         _8 = copy _9 as *mut () (PtrToPtr);
++         _28 = copy _6;
++         _8 = copy _5 as *mut () (PtrToPtr);
+          StorageDead(_28);
+          StorageDead(_9);
+          _0 = const ();
+          StorageDead(_8);
+-         StorageDead(_6);
 -         StorageDead(_5);
 +         nop;
-          StorageDead(_4);
++         nop;
           drop(_3) -> [return: bb1, unwind unreachable];
       }
   
       bb5: {
--         _20 = Layout::from_size_align_unchecked::precondition_check(copy _11, copy _12) -> [return: bb6, unwind unreachable];
-+         _20 = Layout::from_size_align_unchecked::precondition_check(const 0_usize, const 1_usize) -> [return: bb6, unwind unreachable];
+-         _24 = Layout::from_size_align_unchecked::precondition_check(copy _16, copy _17) -> [return: bb6, unwind unreachable];
++         _24 = Layout::from_size_align_unchecked::precondition_check(const 0_usize, const 1_usize) -> [return: bb6, unwind unreachable];
       }
   
       bb6: {
-          StorageDead(_19);
-          StorageLive(_21);
--         _21 = copy _12 as std::ptr::Alignment (Transmute);
--         _14 = Layout { size: copy _11, align: move _21 };
-+         _21 = const std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0);
-+         _14 = const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }};
-          StorageDead(_21);
-          StorageLive(_15);
--         _15 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], copy _14, const false) -> [return: bb7, unwind unreachable];
-+         _15 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}, const false) -> [return: bb7, unwind unreachable];
+          StorageDead(_23);
+          StorageLive(_25);
+-         _25 = copy _17 as std::ptr::Alignment (Transmute);
+-         _18 = Layout { size: copy _16, align: move _25 };
++         _25 = const std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0);
++         _18 = const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }};
+          StorageDead(_25);
+          StorageLive(_19);
+-         _19 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], copy _18, const false) -> [return: bb7, unwind unreachable];
++         _19 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}, const false) -> [return: bb7, unwind unreachable];
       }
   
       bb7: {
-          _16 = discriminant(_15);
-          switchInt(move _16) -> [0: bb4, 1: bb3, otherwise: bb2];
+          _20 = discriminant(_19);
+          switchInt(move _20) -> [0: bb4, 1: bb3, otherwise: bb2];
       }
 + }
 + 
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff
index b2085afb7137..82a14a8b6ec9 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.32bit.panic-unwind.diff
@@ -6,24 +6,26 @@
       let _1: &std::boxed::Box<()>;
       let _2: &std::boxed::Box<()>;
       let _3: std::boxed::Box<()>;
-      let mut _6: *const ();
-      let mut _8: *const [()];
-      let mut _9: std::boxed::Box<()>;
-      let mut _10: *const ();
-      let mut _11: usize;
+      let mut _4: ();
+      let mut _7: *const ();
+      let mut _9: *const [()];
+      let mut _10: std::boxed::Box<()>;
+      let mut _11: *const ();
+      let mut _12: usize;
       scope 1 {
           debug vp_ctx => _1;
-          let _4: *const ();
+          let _5: *const ();
           scope 2 {
-              debug slf => _10;
-              let _5: *const [()];
+              debug slf => _5;
+              let _6: *const [()];
               scope 3 {
-                  debug bytes => _5;
-                  let _7: *mut ();
+                  debug bytes => _6;
+                  let _8: *mut ();
                   scope 4 {
-                      debug _x => _7;
+                      debug _x => _8;
                   }
                   scope 7 (inlined foo) {
+                      let mut _13: *const [()];
                   }
               }
               scope 5 (inlined slice_from_raw_parts::<()>) {
@@ -35,41 +37,54 @@
   
       bb0: {
           StorageLive(_1);
-          StorageLive(_2);
+-         StorageLive(_2);
++         nop;
           StorageLive(_3);
-          _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
+          StorageLive(_4);
+-         _4 = ();
+-         _3 = Box::<()>::new(move _4) -> [return: bb1, unwind continue];
++         _4 = const ();
++         _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
       }
   
       bb1: {
+          StorageDead(_4);
           _2 = &_3;
-          _1 = copy _2;
-          StorageDead(_2);
-          StorageLive(_4);
--         _9 = deref_copy _3;
-+         _9 = copy _3;
-          _10 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
-          _4 = copy _10;
+          _1 = &(*_2);
+-         StorageDead(_2);
 -         StorageLive(_5);
+-         _10 = copy (*_1);
++         nop;
++         nop;
++         _10 = copy (*_2);
+          _11 = copy ((_10.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
+          _5 = &raw const (*_11);
+-         StorageLive(_6);
 +         nop;
-          StorageLive(_6);
--         _6 = copy _4;
-+         _6 = copy _10;
-          StorageLive(_11);
-          _11 = const 1_usize;
--         _5 = *const [()] from (copy _6, copy _11);
-+         _5 = *const [()] from (copy _10, const 1_usize);
-          StorageDead(_11);
-          StorageDead(_6);
           StorageLive(_7);
-          StorageLive(_8);
-          _8 = copy _5;
--         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
-          StorageDead(_8);
+          _7 = copy _5;
+          StorageLive(_12);
+          _12 = const 1_usize;
+-         _6 = *const [()] from (copy _7, copy _12);
++         _6 = *const [()] from (copy _5, const 1_usize);
+          StorageDead(_12);
           StorageDead(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _6;
+          StorageLive(_13);
+-         _13 = copy _9;
+-         _8 = copy _9 as *mut () (PtrToPtr);
++         _13 = copy _6;
++         _8 = copy _5 as *mut () (PtrToPtr);
+          StorageDead(_13);
+          StorageDead(_9);
+          _0 = const ();
+          StorageDead(_8);
+-         StorageDead(_6);
 -         StorageDead(_5);
 +         nop;
-          StorageDead(_4);
++         nop;
           drop(_3) -> [return: bb2, unwind: bb3];
       }
   
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
index 839b53e3b0b3..047579cdb509 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-abort.diff
@@ -6,24 +6,28 @@
       let _1: &std::boxed::Box<()>;
       let _2: &std::boxed::Box<()>;
       let _3: std::boxed::Box<()>;
-      let mut _6: *const ();
-      let mut _8: *const [()];
-      let mut _9: std::boxed::Box<()>;
-      let mut _10: *const ();
-      let mut _23: usize;
+      let mut _4: ();
+      let mut _7: *const ();
+      let mut _9: *const [()];
+      let mut _10: std::boxed::Box<()>;
+      let mut _11: *const ();
+      let mut _16: usize;
+      let mut _17: usize;
+      let mut _27: usize;
       scope 1 {
           debug vp_ctx => _1;
-          let _4: *const ();
+          let _5: *const ();
           scope 2 {
-              debug slf => _10;
-              let _5: *const [()];
+              debug slf => _5;
+              let _6: *const [()];
               scope 3 {
-                  debug bytes => _5;
-                  let _7: *mut ();
+                  debug bytes => _6;
+                  let _8: *mut ();
                   scope 4 {
-                      debug _x => _7;
+                      debug _x => _8;
                   }
                   scope 18 (inlined foo) {
+                      let mut _28: *const [()];
                   }
               }
               scope 16 (inlined slice_from_raw_parts::<()>) {
@@ -33,21 +37,22 @@
           }
       }
       scope 5 (inlined Box::<()>::new) {
-          let mut _11: usize;
-          let mut _12: usize;
-          let mut _13: *mut u8;
+          let mut _12: *mut u8;
+          let mut _13: *const ();
+          let mut _14: std::ptr::NonNull<()>;
+          let mut _15: std::ptr::Unique<()>;
           scope 6 (inlined alloc::alloc::exchange_malloc) {
-              let _14: std::alloc::Layout;
-              let mut _15: std::result::Result, std::alloc::AllocError>;
-              let mut _16: isize;
-              let mut _18: !;
+              let _18: std::alloc::Layout;
+              let mut _19: std::result::Result, std::alloc::AllocError>;
+              let mut _20: isize;
+              let mut _22: !;
               scope 7 {
-                  let _17: std::ptr::NonNull<[u8]>;
+                  let _21: std::ptr::NonNull<[u8]>;
                   scope 8 {
                       scope 11 (inlined NonNull::<[u8]>::as_mut_ptr) {
                           scope 12 (inlined NonNull::<[u8]>::as_non_null_ptr) {
                               scope 13 (inlined NonNull::<[u8]>::cast::) {
-                                  let mut _22: *mut [u8];
+                                  let mut _26: *mut [u8];
                                   scope 14 (inlined NonNull::<[u8]>::as_ptr) {
                                   }
                               }
@@ -60,31 +65,39 @@
                   }
               }
               scope 9 (inlined #[track_caller] Layout::from_size_align_unchecked) {
-                  let mut _19: bool;
-                  let _20: ();
-                  let mut _21: std::ptr::Alignment;
+                  let mut _23: bool;
+                  let _24: ();
+                  let mut _25: std::ptr::Alignment;
               }
           }
       }
   
       bb0: {
           StorageLive(_1);
-          StorageLive(_2);
+-         StorageLive(_2);
++         nop;
           StorageLive(_3);
-          StorageLive(_11);
+          StorageLive(_4);
+-         _4 = ();
++         _4 = const ();
           StorageLive(_12);
           StorageLive(_13);
--         _11 = SizeOf(());
--         _12 = AlignOf(());
-+         _11 = const 0_usize;
-+         _12 = const 1_usize;
           StorageLive(_14);
+          StorageLive(_15);
           StorageLive(_16);
+-         _16 = const <() as std::mem::SizedTypeProperties>::SIZE;
++         _16 = const 0_usize;
           StorageLive(_17);
-          StorageLive(_19);
-          _19 = const false;
--         switchInt(move _19) -> [0: bb6, otherwise: bb5];
-+         switchInt(const false) -> [0: bb6, otherwise: bb5];
+-         _17 = const <() as std::mem::SizedTypeProperties>::ALIGN;
++         _17 = const 1_usize;
+          StorageLive(_18);
+          StorageLive(_20);
+          StorageLive(_21);
+          StorageLive(_22);
+          StorageLive(_24);
+          StorageLive(_23);
+          _23 = UbChecks();
+          switchInt(move _23) -> [0: bb6, otherwise: bb5];
       }
   
       bb1: {
@@ -98,77 +111,98 @@
       }
   
       bb3: {
--         _18 = handle_alloc_error(move _14) -> unwind unreachable;
-+         _18 = handle_alloc_error(const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}) -> unwind unreachable;
+-         _22 = handle_alloc_error(move _18) -> unwind unreachable;
++         _22 = handle_alloc_error(const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}) -> unwind unreachable;
       }
   
       bb4: {
-          _17 = copy ((_15 as Ok).0: std::ptr::NonNull<[u8]>);
-          StorageLive(_22);
-          _22 = copy _17 as *mut [u8] (Transmute);
-          _13 = copy _22 as *mut u8 (PtrToPtr);
+          _21 = copy ((_19 as Ok).0: std::ptr::NonNull<[u8]>);
+-         StorageLive(_26);
++         nop;
+          _26 = copy _21 as *mut [u8] (Transmute);
+          _12 = copy _26 as *mut u8 (PtrToPtr);
+-         StorageDead(_26);
++         nop;
+          StorageDead(_19);
+          StorageDead(_24);
           StorageDead(_22);
-          StorageDead(_15);
+          StorageDead(_21);
+          StorageDead(_20);
+          StorageDead(_18);
           StorageDead(_17);
           StorageDead(_16);
+-         _13 = copy _12 as *const () (PtrToPtr);
++         _13 = copy _26 as *const () (PtrToPtr);
+          _14 = NonNull::<()> { pointer: copy _13 };
+          _15 = Unique::<()> { pointer: copy _14, _marker: const PhantomData::<()> };
+          _3 = Box::<()>(move _15, const std::alloc::Global);
+-         (*_13) = move _4;
++         (*_13) = const ();
+          StorageDead(_15);
           StorageDead(_14);
-          _3 = ShallowInitBox(move _13, ());
           StorageDead(_13);
           StorageDead(_12);
-          StorageDead(_11);
+          StorageDead(_4);
           _2 = &_3;
-          _1 = copy _2;
-          StorageDead(_2);
-          StorageLive(_4);
--         _9 = deref_copy _3;
-+         _9 = copy _3;
-          _10 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
-          _4 = copy _10;
+          _1 = &(*_2);
+-         StorageDead(_2);
 -         StorageLive(_5);
+-         _10 = copy (*_1);
++         nop;
++         nop;
++         _10 = copy (*_2);
+          _11 = copy ((_10.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
+          _5 = &raw const (*_11);
+-         StorageLive(_6);
 +         nop;
-          StorageLive(_6);
--         _6 = copy _4;
-+         _6 = copy _10;
-          StorageLive(_23);
-          _23 = const 1_usize;
--         _5 = *const [()] from (copy _6, copy _23);
-+         _5 = *const [()] from (copy _10, const 1_usize);
-          StorageDead(_23);
-          StorageDead(_6);
           StorageLive(_7);
-          StorageLive(_8);
-          _8 = copy _5;
--         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
-          StorageDead(_8);
+          _7 = copy _5;
+          StorageLive(_27);
+          _27 = const 1_usize;
+-         _6 = *const [()] from (copy _7, copy _27);
++         _6 = *const [()] from (copy _5, const 1_usize);
+          StorageDead(_27);
           StorageDead(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _6;
+          StorageLive(_28);
+-         _28 = copy _9;
+-         _8 = copy _9 as *mut () (PtrToPtr);
++         _28 = copy _6;
++         _8 = copy _5 as *mut () (PtrToPtr);
+          StorageDead(_28);
+          StorageDead(_9);
+          _0 = const ();
+          StorageDead(_8);
+-         StorageDead(_6);
 -         StorageDead(_5);
 +         nop;
-          StorageDead(_4);
++         nop;
           drop(_3) -> [return: bb1, unwind unreachable];
       }
   
       bb5: {
--         _20 = Layout::from_size_align_unchecked::precondition_check(copy _11, copy _12) -> [return: bb6, unwind unreachable];
-+         _20 = Layout::from_size_align_unchecked::precondition_check(const 0_usize, const 1_usize) -> [return: bb6, unwind unreachable];
+-         _24 = Layout::from_size_align_unchecked::precondition_check(copy _16, copy _17) -> [return: bb6, unwind unreachable];
++         _24 = Layout::from_size_align_unchecked::precondition_check(const 0_usize, const 1_usize) -> [return: bb6, unwind unreachable];
       }
   
       bb6: {
-          StorageDead(_19);
-          StorageLive(_21);
--         _21 = copy _12 as std::ptr::Alignment (Transmute);
--         _14 = Layout { size: copy _11, align: move _21 };
-+         _21 = const std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0);
-+         _14 = const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }};
-          StorageDead(_21);
-          StorageLive(_15);
--         _15 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], copy _14, const false) -> [return: bb7, unwind unreachable];
-+         _15 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}, const false) -> [return: bb7, unwind unreachable];
+          StorageDead(_23);
+          StorageLive(_25);
+-         _25 = copy _17 as std::ptr::Alignment (Transmute);
+-         _18 = Layout { size: copy _16, align: move _25 };
++         _25 = const std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0);
++         _18 = const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }};
+          StorageDead(_25);
+          StorageLive(_19);
+-         _19 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], copy _18, const false) -> [return: bb7, unwind unreachable];
++         _19 = std::alloc::Global::alloc_impl(const alloc::alloc::exchange_malloc::promoted[0], const Layout {{ size: 0_usize, align: std::ptr::Alignment(std::ptr::alignment::AlignmentEnum::_Align1Shl0) }}, const false) -> [return: bb7, unwind unreachable];
       }
   
       bb7: {
-          _16 = discriminant(_15);
-          switchInt(move _16) -> [0: bb4, 1: bb3, otherwise: bb2];
+          _20 = discriminant(_19);
+          switchInt(move _20) -> [0: bb4, 1: bb3, otherwise: bb2];
       }
 + }
 + 
diff --git a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff
index b2085afb7137..82a14a8b6ec9 100644
--- a/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff
+++ b/tests/mir-opt/dont_reset_cast_kind_without_updating_operand.test.GVN.64bit.panic-unwind.diff
@@ -6,24 +6,26 @@
       let _1: &std::boxed::Box<()>;
       let _2: &std::boxed::Box<()>;
       let _3: std::boxed::Box<()>;
-      let mut _6: *const ();
-      let mut _8: *const [()];
-      let mut _9: std::boxed::Box<()>;
-      let mut _10: *const ();
-      let mut _11: usize;
+      let mut _4: ();
+      let mut _7: *const ();
+      let mut _9: *const [()];
+      let mut _10: std::boxed::Box<()>;
+      let mut _11: *const ();
+      let mut _12: usize;
       scope 1 {
           debug vp_ctx => _1;
-          let _4: *const ();
+          let _5: *const ();
           scope 2 {
-              debug slf => _10;
-              let _5: *const [()];
+              debug slf => _5;
+              let _6: *const [()];
               scope 3 {
-                  debug bytes => _5;
-                  let _7: *mut ();
+                  debug bytes => _6;
+                  let _8: *mut ();
                   scope 4 {
-                      debug _x => _7;
+                      debug _x => _8;
                   }
                   scope 7 (inlined foo) {
+                      let mut _13: *const [()];
                   }
               }
               scope 5 (inlined slice_from_raw_parts::<()>) {
@@ -35,41 +37,54 @@
   
       bb0: {
           StorageLive(_1);
-          StorageLive(_2);
+-         StorageLive(_2);
++         nop;
           StorageLive(_3);
-          _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
+          StorageLive(_4);
+-         _4 = ();
+-         _3 = Box::<()>::new(move _4) -> [return: bb1, unwind continue];
++         _4 = const ();
++         _3 = Box::<()>::new(const ()) -> [return: bb1, unwind continue];
       }
   
       bb1: {
+          StorageDead(_4);
           _2 = &_3;
-          _1 = copy _2;
-          StorageDead(_2);
-          StorageLive(_4);
--         _9 = deref_copy _3;
-+         _9 = copy _3;
-          _10 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
-          _4 = copy _10;
+          _1 = &(*_2);
+-         StorageDead(_2);
 -         StorageLive(_5);
+-         _10 = copy (*_1);
++         nop;
++         nop;
++         _10 = copy (*_2);
+          _11 = copy ((_10.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *const () (Transmute);
+          _5 = &raw const (*_11);
+-         StorageLive(_6);
 +         nop;
-          StorageLive(_6);
--         _6 = copy _4;
-+         _6 = copy _10;
-          StorageLive(_11);
-          _11 = const 1_usize;
--         _5 = *const [()] from (copy _6, copy _11);
-+         _5 = *const [()] from (copy _10, const 1_usize);
-          StorageDead(_11);
-          StorageDead(_6);
           StorageLive(_7);
-          StorageLive(_8);
-          _8 = copy _5;
--         _7 = copy _8 as *mut () (PtrToPtr);
-+         _7 = copy ((_9.0: std::ptr::Unique<()>).0: std::ptr::NonNull<()>) as *mut () (Transmute);
-          StorageDead(_8);
+          _7 = copy _5;
+          StorageLive(_12);
+          _12 = const 1_usize;
+-         _6 = *const [()] from (copy _7, copy _12);
++         _6 = *const [()] from (copy _5, const 1_usize);
+          StorageDead(_12);
           StorageDead(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _6;
+          StorageLive(_13);
+-         _13 = copy _9;
+-         _8 = copy _9 as *mut () (PtrToPtr);
++         _13 = copy _6;
++         _8 = copy _5 as *mut () (PtrToPtr);
+          StorageDead(_13);
+          StorageDead(_9);
+          _0 = const ();
+          StorageDead(_8);
+-         StorageDead(_6);
 -         StorageDead(_5);
 +         nop;
-          StorageDead(_4);
++         nop;
           drop(_3) -> [return: bb2, unwind: bb3];
       }
   
diff --git a/tests/mir-opt/early_otherwise_branch_68867.try_sum.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch_68867.try_sum.EarlyOtherwiseBranch.diff
index fec318c1ab49..c8b516c14987 100644
--- a/tests/mir-opt/early_otherwise_branch_68867.try_sum.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch_68867.try_sum.EarlyOtherwiseBranch.diff
@@ -76,7 +76,7 @@
           _4 = (move _5, move _6);
           StorageDead(_6);
           StorageDead(_5);
-          _34 = deref_copy (_4.0: &ViewportPercentageLength);
+          _34 = copy (_4.0: &ViewportPercentageLength);
           _11 = discriminant((*_34));
           switchInt(move _11) -> [0: bb2, 1: bb3, 2: bb4, 3: bb5, otherwise: bb12];
       }
@@ -92,35 +92,35 @@
       }
   
       bb2: {
-          _35 = deref_copy (_4.1: &ViewportPercentageLength);
+          _35 = copy (_4.1: &ViewportPercentageLength);
           _7 = discriminant((*_35));
           switchInt(move _7) -> [0: bb9, otherwise: bb1];
       }
   
       bb3: {
-          _36 = deref_copy (_4.1: &ViewportPercentageLength);
+          _36 = copy (_4.1: &ViewportPercentageLength);
           _8 = discriminant((*_36));
           switchInt(move _8) -> [1: bb8, otherwise: bb1];
       }
   
       bb4: {
-          _37 = deref_copy (_4.1: &ViewportPercentageLength);
+          _37 = copy (_4.1: &ViewportPercentageLength);
           _9 = discriminant((*_37));
           switchInt(move _9) -> [2: bb7, otherwise: bb1];
       }
   
       bb5: {
-          _38 = deref_copy (_4.1: &ViewportPercentageLength);
+          _38 = copy (_4.1: &ViewportPercentageLength);
           _10 = discriminant((*_38));
           switchInt(move _10) -> [3: bb6, otherwise: bb1];
       }
   
       bb6: {
           StorageLive(_27);
-          _39 = deref_copy (_4.0: &ViewportPercentageLength);
+          _39 = copy (_4.0: &ViewportPercentageLength);
           _27 = copy (((*_39) as Vmax).0: f32);
           StorageLive(_28);
-          _40 = deref_copy (_4.1: &ViewportPercentageLength);
+          _40 = copy (_4.1: &ViewportPercentageLength);
           _28 = copy (((*_40) as Vmax).0: f32);
           StorageLive(_29);
           StorageLive(_30);
@@ -139,10 +139,10 @@
   
       bb7: {
           StorageLive(_22);
-          _41 = deref_copy (_4.0: &ViewportPercentageLength);
+          _41 = copy (_4.0: &ViewportPercentageLength);
           _22 = copy (((*_41) as Vmin).0: f32);
           StorageLive(_23);
-          _42 = deref_copy (_4.1: &ViewportPercentageLength);
+          _42 = copy (_4.1: &ViewportPercentageLength);
           _23 = copy (((*_42) as Vmin).0: f32);
           StorageLive(_24);
           StorageLive(_25);
@@ -161,10 +161,10 @@
   
       bb8: {
           StorageLive(_17);
-          _43 = deref_copy (_4.0: &ViewportPercentageLength);
+          _43 = copy (_4.0: &ViewportPercentageLength);
           _17 = copy (((*_43) as Vh).0: f32);
           StorageLive(_18);
-          _44 = deref_copy (_4.1: &ViewportPercentageLength);
+          _44 = copy (_4.1: &ViewportPercentageLength);
           _18 = copy (((*_44) as Vh).0: f32);
           StorageLive(_19);
           StorageLive(_20);
@@ -183,10 +183,10 @@
   
       bb9: {
           StorageLive(_12);
-          _45 = deref_copy (_4.0: &ViewportPercentageLength);
+          _45 = copy (_4.0: &ViewportPercentageLength);
           _12 = copy (((*_45) as Vw).0: f32);
           StorageLive(_13);
-          _46 = deref_copy (_4.1: &ViewportPercentageLength);
+          _46 = copy (_4.1: &ViewportPercentageLength);
           _13 = copy (((*_46) as Vw).0: f32);
           StorageLive(_14);
           StorageLive(_15);
diff --git a/tests/mir-opt/early_otherwise_branch_soundness.no_downcast.EarlyOtherwiseBranch.diff b/tests/mir-opt/early_otherwise_branch_soundness.no_downcast.EarlyOtherwiseBranch.diff
index 6a4c947b8826..ab81f0a67e4e 100644
--- a/tests/mir-opt/early_otherwise_branch_soundness.no_downcast.EarlyOtherwiseBranch.diff
+++ b/tests/mir-opt/early_otherwise_branch_soundness.no_downcast.EarlyOtherwiseBranch.diff
@@ -16,7 +16,7 @@
       }
   
       bb1: {
-          _4 = deref_copy (((*_1) as Some).0: &E<'_>);
+          _4 = copy (((*_1) as Some).0: &E<'_>);
           _2 = discriminant((*_4));
           switchInt(move _2) -> [1: bb2, 0: bb3, otherwise: bb5];
       }
diff --git a/tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff b/tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff
index 267a4c1cf6be..cf850351c1d6 100644
--- a/tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff
+++ b/tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff
@@ -44,7 +44,6 @@
 +         _9 = copy _8 as *mut u8 (PtrToPtr);
 +         _10 = &raw const _2;
 +         _11 = copy _10 as *const u8 (PtrToPtr);
-+         Deinit(_8);
 +         copy_nonoverlapping(dst = copy _9, src = copy _11, count = copy _7);
 +         StorageDead(_4);
 +         nop;
@@ -59,7 +58,6 @@
 +         _17 = copy _16 as *mut u8 (PtrToPtr);
 +         _18 = &raw const _1;
 +         _19 = copy _18 as *const u8 (PtrToPtr);
-+         Deinit(_16);
 +         copy_nonoverlapping(dst = copy _17, src = copy _19, count = copy _15);
 +         StorageDead(_12);
 +         nop;
diff --git a/tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff b/tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff
index 8e5c403cd7e6..dc5ea1add000 100644
--- a/tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff
+++ b/tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff
@@ -44,7 +44,6 @@
 +         _9 = copy _8 as *mut u8 (PtrToPtr);
 +         _10 = &raw const _2;
 +         _11 = copy _10 as *const u8 (PtrToPtr);
-+         Deinit(_8);
 +         copy_nonoverlapping(dst = copy _9, src = copy _11, count = copy _7);
 +         StorageDead(_4);
 +         nop;
@@ -59,7 +58,6 @@
 +         _17 = copy _16 as *mut u8 (PtrToPtr);
 +         _18 = &raw const _1;
 +         _19 = copy _18 as *const u8 (PtrToPtr);
-+         Deinit(_16);
 +         copy_nonoverlapping(dst = copy _17, src = copy _19, count = copy _15);
 +         StorageDead(_12);
 +         nop;
diff --git a/tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff b/tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff
index 96c5aadd85fd..3cd2e74a0db6 100644
--- a/tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff
+++ b/tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff
@@ -44,7 +44,6 @@
 +         _9 = copy _8 as *mut u8 (PtrToPtr);
 +         _10 = &raw const _2;
 +         _11 = copy _10 as *const u8 (PtrToPtr);
-+         Deinit(_8);
 +         copy_nonoverlapping(dst = copy _9, src = copy _11, count = copy _7);
 +         StorageDead(_4);
 +         nop;
@@ -59,7 +58,6 @@
 +         _17 = copy _16 as *mut u8 (PtrToPtr);
 +         _18 = &raw const _1;
 +         _19 = copy _18 as *const u8 (PtrToPtr);
-+         Deinit(_16);
 +         copy_nonoverlapping(dst = copy _17, src = copy _19, count = copy _15);
 +         StorageDead(_12);
 +         nop;
diff --git a/tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff b/tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff
index d20e2e08eaaf..10b0ec5a63fa 100644
--- a/tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff
+++ b/tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff
@@ -44,7 +44,6 @@
 +         _9 = copy _8 as *mut u8 (PtrToPtr);
 +         _10 = &raw const _2;
 +         _11 = copy _10 as *const u8 (PtrToPtr);
-+         Deinit(_8);
 +         copy_nonoverlapping(dst = copy _9, src = copy _11, count = copy _7);
 +         StorageDead(_4);
 +         nop;
@@ -59,7 +58,6 @@
 +         _17 = copy _16 as *mut u8 (PtrToPtr);
 +         _18 = &raw const _1;
 +         _19 = copy _18 as *const u8 (PtrToPtr);
-+         Deinit(_16);
 +         copy_nonoverlapping(dst = copy _17, src = copy _19, count = copy _15);
 +         StorageDead(_12);
 +         nop;
diff --git a/tests/mir-opt/gvn.array_len.GVN.panic-abort.diff b/tests/mir-opt/gvn.array_len.GVN.panic-abort.diff
index 0d0477fe7729..59b65a52f4ee 100644
--- a/tests/mir-opt/gvn.array_len.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.array_len.GVN.panic-abort.diff
@@ -12,8 +12,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           _3 = &(*_1);
           _2 = move _3 as &[i32] (PointerCoercion(Unsize, Implicit));
@@ -23,8 +22,7 @@
 -         _0 = PtrMetadata(move _4);
 +         _0 = const 42_usize;
           StorageDead(_4);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.array_len.GVN.panic-unwind.diff b/tests/mir-opt/gvn.array_len.GVN.panic-unwind.diff
index 0d0477fe7729..59b65a52f4ee 100644
--- a/tests/mir-opt/gvn.array_len.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.array_len.GVN.panic-unwind.diff
@@ -12,8 +12,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           _3 = &(*_1);
           _2 = move _3 as &[i32] (PointerCoercion(Unsize, Implicit));
@@ -23,8 +22,7 @@
 -         _0 = PtrMetadata(move _4);
 +         _0 = const 42_usize;
           StorageDead(_4);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.dereference_indexing.GVN.panic-abort.diff b/tests/mir-opt/gvn.dereference_indexing.GVN.panic-abort.diff
new file mode 100644
index 000000000000..cdfbef8e9633
--- /dev/null
+++ b/tests/mir-opt/gvn.dereference_indexing.GVN.panic-abort.diff
@@ -0,0 +1,60 @@
+- // MIR for `dereference_indexing` before GVN
++ // MIR for `dereference_indexing` after GVN
+  
+  fn dereference_indexing(_1: [u8; 2], _2: usize) -> () {
+      debug array => _1;
+      debug index => _2;
+      let mut _0: ();
+      let _3: &u8;
+      let _4: usize;
+      let mut _5: usize;
+      let _6: usize;
+      let mut _7: bool;
+      let _8: ();
+      let mut _9: u8;
+      scope 1 {
+          debug a => _3;
+      }
+      scope 2 {
+          debug i => _4;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+-         StorageLive(_4);
++         nop;
+          StorageLive(_5);
+          _5 = copy _2;
+-         _4 = Add(move _5, const 1_usize);
++         _4 = Add(copy _2, const 1_usize);
+          StorageDead(_5);
+          StorageLive(_6);
+          _6 = copy _4;
+-         _7 = Lt(copy _6, const 2_usize);
+-         assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _6) -> [success: bb1, unwind unreachable];
++         _7 = Lt(copy _4, const 2_usize);
++         assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _4) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+-         _3 = &_1[_6];
+-         StorageDead(_4);
++         _3 = &_1[_4];
++         nop;
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+-         _9 = copy (*_3);
++         _9 = copy _1[_4];
+          _8 = opaque::(move _9) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          StorageDead(_9);
+          StorageDead(_8);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.dereference_indexing.GVN.panic-unwind.diff b/tests/mir-opt/gvn.dereference_indexing.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..8c817f9d5cf7
--- /dev/null
+++ b/tests/mir-opt/gvn.dereference_indexing.GVN.panic-unwind.diff
@@ -0,0 +1,60 @@
+- // MIR for `dereference_indexing` before GVN
++ // MIR for `dereference_indexing` after GVN
+  
+  fn dereference_indexing(_1: [u8; 2], _2: usize) -> () {
+      debug array => _1;
+      debug index => _2;
+      let mut _0: ();
+      let _3: &u8;
+      let _4: usize;
+      let mut _5: usize;
+      let _6: usize;
+      let mut _7: bool;
+      let _8: ();
+      let mut _9: u8;
+      scope 1 {
+          debug a => _3;
+      }
+      scope 2 {
+          debug i => _4;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+-         StorageLive(_4);
++         nop;
+          StorageLive(_5);
+          _5 = copy _2;
+-         _4 = Add(move _5, const 1_usize);
++         _4 = Add(copy _2, const 1_usize);
+          StorageDead(_5);
+          StorageLive(_6);
+          _6 = copy _4;
+-         _7 = Lt(copy _6, const 2_usize);
+-         assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _6) -> [success: bb1, unwind continue];
++         _7 = Lt(copy _4, const 2_usize);
++         assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _4) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+-         _3 = &_1[_6];
+-         StorageDead(_4);
++         _3 = &_1[_4];
++         nop;
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+-         _9 = copy (*_3);
++         _9 = copy _1[_4];
+          _8 = opaque::(move _9) -> [return: bb2, unwind continue];
+      }
+  
+      bb2: {
+          StorageDead(_9);
+          StorageDead(_8);
+          _0 = const ();
+          StorageDead(_3);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 407980fd0fd4..3c3241fefe22 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -1052,6 +1052,26 @@ fn remove_casts_must_change_both_sides(mut_a: &*mut u8, mut_b: *mut u8) -> bool
     }
 }
 
+/// Verify that we do not references to non-existing locals when dereferencing projections.
+fn dereference_indexing(array: [u8; 2], index: usize) {
+    // CHECK-LABEL: fn dereference_indexing(
+    // CHECK: debug a => [[a:_.*]];
+    // CHECK: debug i => [[i:_.*]];
+
+    let a = {
+        // CHECK: [[i]] = Add(copy _2, const 1_usize);
+        let i = index + 1;
+        // CHECK: [[a]] = &_1[[[i]]];
+        &array[i]
+    };
+
+    // CHECK-NOT: StorageDead([[i]]);
+    // CHECK: [[tmp:_.*]] = copy _1[[[i]]];
+    // CHECK: opaque::(move [[tmp]])
+    opaque(*a);
+}
+
+// CHECK-LABEL: fn main(
 fn main() {
     subexpression_elimination(2, 4, 5);
     wrap_unwrap(5);
@@ -1079,6 +1099,7 @@ fn main() {
     slice_const_length(&[1]);
     meta_of_ref_to_slice(&42);
     slice_from_raw_parts_as_ptr(&123, 456);
+    dereference_indexing([129, 14], 5);
 }
 
 #[inline(never)]
@@ -1138,3 +1159,4 @@ enum Never {}
 // EMIT_MIR gvn.cast_pointer_then_transmute.GVN.diff
 // EMIT_MIR gvn.transmute_then_cast_pointer.GVN.diff
 // EMIT_MIR gvn.remove_casts_must_change_both_sides.GVN.diff
+// EMIT_MIR gvn.dereference_indexing.GVN.diff
diff --git a/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-abort.diff b/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-abort.diff
index 1b305e746f5e..0165136fda4c 100644
--- a/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-abort.diff
@@ -89,6 +89,7 @@
 -         StorageDead(_4);
 +         _3 = copy _4;
 +         nop;
+          StorageDead(_8);
           StorageDead(_7);
           StorageDead(_5);
           StorageLive(_10);
@@ -116,6 +117,7 @@
 -         StorageDead(_11);
 +         _10 = copy _11;
 +         nop;
+          StorageDead(_15);
           StorageDead(_14);
           StorageDead(_12);
           StorageLive(_17);
@@ -251,9 +253,7 @@
           StorageDead(_43);
           StorageDead(_42);
           _0 = const ();
-          StorageDead(_15);
           StorageDead(_10);
-          StorageDead(_8);
           StorageDead(_3);
           StorageDead(_1);
           return;
diff --git a/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-unwind.diff b/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-unwind.diff
index e418ecf25bd4..bbaab46b4756 100644
--- a/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.wide_ptr_same_provenance.GVN.panic-unwind.diff
@@ -89,6 +89,7 @@
 -         StorageDead(_4);
 +         _3 = copy _4;
 +         nop;
+          StorageDead(_8);
           StorageDead(_7);
           StorageDead(_5);
           StorageLive(_10);
@@ -116,6 +117,7 @@
 -         StorageDead(_11);
 +         _10 = copy _11;
 +         nop;
+          StorageDead(_15);
           StorageDead(_14);
           StorageDead(_12);
           StorageLive(_17);
@@ -251,9 +253,7 @@
           StorageDead(_43);
           StorageDead(_42);
           _0 = const ();
-          StorageDead(_15);
           StorageDead(_10);
-          StorageDead(_8);
           StorageDead(_3);
           StorageDead(_1);
           return;
diff --git a/tests/mir-opt/gvn_copy_aggregate.all_copy_2.GVN.diff b/tests/mir-opt/gvn_copy_aggregate.all_copy_2.GVN.diff
index 452d8a933203..eed8cb7d62e7 100644
--- a/tests/mir-opt/gvn_copy_aggregate.all_copy_2.GVN.diff
+++ b/tests/mir-opt/gvn_copy_aggregate.all_copy_2.GVN.diff
@@ -25,15 +25,14 @@
   
       bb0: {
 -         StorageLive(_2);
--         _8 = deref_copy (*_1);
 +         nop;
-+         _8 = copy (*_1);
+          _8 = copy (*_1);
           _2 = copy ((*_8).0: i32);
 -         StorageLive(_3);
--         _9 = deref_copy (*_1);
+-         _9 = copy (*_1);
 -         _3 = copy ((*_9).1: u64);
 -         StorageLive(_4);
--         _10 = deref_copy (*_1);
+-         _10 = copy (*_1);
 -         _4 = copy ((*_10).2: [i8; 3]);
 +         nop;
 +         _9 = copy _8;
diff --git a/tests/mir-opt/gvn_loop.loop_deref_mut.GVN.diff b/tests/mir-opt/gvn_loop.loop_deref_mut.GVN.diff
new file mode 100644
index 000000000000..92e5ccabedf9
--- /dev/null
+++ b/tests/mir-opt/gvn_loop.loop_deref_mut.GVN.diff
@@ -0,0 +1,115 @@
+- // MIR for `loop_deref_mut` before GVN
++ // MIR for `loop_deref_mut` after GVN
+  
+  fn loop_deref_mut(_1: &mut Value) -> Value {
+      debug val => _1;
+      let mut _0: Value;
+      let _2: &Value;
+      let _3: &Value;
+      let mut _4: &Value;
+      let mut _6: !;
+      let mut _8: isize;
+      let mut _9: !;
+      let mut _10: ();
+      let mut _12: i32;
+      let _13: ();
+      let mut _14: bool;
+      let mut _15: !;
+      let mut _16: Value;
+      scope 1 {
+          debug val_alias => _2;
+          let mut _5: bool;
+          scope 2 {
+              debug stop => _5;
+              let _7: i32;
+              scope 3 {
+                  debug v => _7;
+                  let _11: Value;
+                  scope 4 {
+                      debug v => _11;
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+-         StorageLive(_3);
++         nop;
+          StorageLive(_4);
+          _4 = &(*_1);
+          _3 = get::(move _4) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _2 = &(*_3);
+          StorageDead(_4);
+-         StorageDead(_3);
++         nop;
+          StorageLive(_5);
+          _5 = const false;
+-         _8 = discriminant((*_2));
++         _8 = discriminant((*_3));
+          switchInt(move _8) -> [0: bb3, otherwise: bb2];
+      }
+  
+      bb2: {
+          unreachable;
+      }
+  
+      bb3: {
+-         StorageLive(_7);
+-         _7 = copy (((*_2) as V0).0: i32);
++         nop;
++         _7 = copy (((*_3) as V0).0: i32);
+          StorageLive(_9);
+          goto -> bb4;
+      }
+  
+      bb4: {
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = copy _7;
+-         _11 = Value::V0(move _12);
++         _11 = Value::V0(copy _7);
+          StorageDead(_12);
+          StorageLive(_13);
+          StorageLive(_14);
+          _14 = copy _5;
+          switchInt(move _14) -> [0: bb6, otherwise: bb5];
+      }
+  
+      bb5: {
+          _0 = move _11;
+          StorageDead(_14);
+          StorageDead(_13);
+          StorageDead(_11);
+          StorageDead(_9);
+-         StorageDead(_7);
++         nop;
+          StorageDead(_5);
+          StorageDead(_2);
+          return;
+      }
+  
+      bb6: {
+          _13 = const ();
+          StorageDead(_14);
+          StorageDead(_13);
+          _5 = const true;
+          StorageLive(_16);
+-         _16 = Value::V1;
+-         (*_1) = move _16;
++         _16 = const Value::V1;
++         (*_1) = const Value::V1;
+          StorageDead(_16);
+          _10 = const ();
+          StorageDead(_11);
+          goto -> bb4;
+      }
++ }
++ 
++ ALLOC0 (size: 8, align: 4) {
++     01 00 00 00 __ __ __ __                         │ ....░░░░
+  }
+  
diff --git a/tests/mir-opt/gvn_loop.rs b/tests/mir-opt/gvn_loop.rs
new file mode 100644
index 000000000000..6e9df55a968d
--- /dev/null
+++ b/tests/mir-opt/gvn_loop.rs
@@ -0,0 +1,39 @@
+//@ test-mir-pass: GVN
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics, rustc_attrs)]
+
+pub enum Value {
+    V0(i32),
+    V1,
+}
+
+// Check that we do not use the dereferenced value of `val_alias` when returning.
+
+// EMIT_MIR gvn_loop.loop_deref_mut.GVN.diff
+fn loop_deref_mut(val: &mut Value) -> Value {
+    // CHECK-LABEL: fn loop_deref_mut(
+    // CHECK: [[VAL_REF:_.*]] = get::(
+    // CHECK: [[V:_.*]] = copy (((*[[VAL_REF]]) as V0).0: i32);
+    // CEHCK-NOT: copy (*[[VAL_REF]]);
+    // CHECK: [[RET:_*]] = Value::V0(copy [[V]]);
+    // CEHCK-NOT: copy (*[[VAL_REF]]);
+    // CHECK: _0 = move [[RET]]
+    let val_alias: &Value = get(val);
+    let mut stop = false;
+    let Value::V0(v) = *val_alias else { unsafe { core::intrinsics::unreachable() } };
+    loop {
+        let v = Value::V0(v);
+        if stop {
+            return v;
+        }
+        stop = true;
+        *val = Value::V1;
+    }
+}
+
+#[inline(never)]
+#[rustc_nounwind]
+fn get(v: &T) -> &T {
+    v
+}
diff --git a/tests/mir-opt/gvn_overlapping.fields.GVN.diff b/tests/mir-opt/gvn_overlapping.fields.GVN.diff
new file mode 100644
index 000000000000..0548f4e42e43
--- /dev/null
+++ b/tests/mir-opt/gvn_overlapping.fields.GVN.diff
@@ -0,0 +1,14 @@
+- // MIR for `fields` before GVN
++ // MIR for `fields` after GVN
+  
+  fn fields(_1: (Adt, Adt)) -> () {
+      let mut _0: ();
+      let mut _2: u32;
+  
+      bb0: {
+          _2 = copy (((_1.0: Adt) as variant#1).0: u32);
+          (_1.1: Adt) = Adt::Some(copy _2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn_overlapping.rs b/tests/mir-opt/gvn_overlapping.rs
index 99113445e683..f148a8435612 100644
--- a/tests/mir-opt/gvn_overlapping.rs
+++ b/tests/mir-opt/gvn_overlapping.rs
@@ -2,11 +2,10 @@
 
 #![feature(custom_mir, core_intrinsics)]
 
-// Check that we do not create overlapping assignments.
-
 use std::intrinsics::mir::*;
 
 // EMIT_MIR gvn_overlapping.overlapping.GVN.diff
+/// Check that we do not create overlapping assignments.
 #[custom_mir(dialect = "runtime")]
 fn overlapping(_17: Adt) {
     // CHECK-LABEL: fn overlapping(
@@ -26,6 +25,45 @@ fn overlapping(_17: Adt) {
     }
 }
 
+// EMIT_MIR gvn_overlapping.stable_projection.GVN.diff
+/// Check that we allow dereferences in the RHS if the LHS is a stable projection.
+#[custom_mir(dialect = "runtime")]
+fn stable_projection(_1: (Adt,)) {
+    // CHECK-LABEL: fn stable_projection(
+    // CHECK: let mut _2: *mut Adt;
+    // CHECK: let mut _4: &Adt;
+    // CHECK: (_1.0: Adt) = copy (*_4);
+    mir! {
+        let _2: *mut Adt;
+        let _3: u32;
+        let _4: &Adt;
+        {
+            _2 = core::ptr::addr_of_mut!(_1.0);
+            _4 = &(*_2);
+            _3 = Field(Variant((*_4), 1), 0);
+            _1.0 = Adt::Some(_3);
+            Return()
+        }
+    }
+}
+
+// EMIT_MIR gvn_overlapping.fields.GVN.diff
+/// Check that we do not create assignments between different fields of the same local.
+#[custom_mir(dialect = "runtime")]
+fn fields(_1: (Adt, Adt)) {
+    // CHECK-LABEL: fn fields(
+    // CHECK: _2 = copy (((_1.0: Adt) as variant#1).0: u32);
+    // CHECK-NEXT: (_1.1: Adt) = Adt::Some(copy _2);
+    mir! {
+        let _2: u32;
+        {
+            _2 = Field(Variant(_1.0, 1), 0);
+            _1.1 = Adt::Some(_2);
+            Return()
+        }
+    }
+}
+
 fn main() {
     overlapping(Adt::Some(0));
 }
diff --git a/tests/mir-opt/gvn_overlapping.stable_projection.GVN.diff b/tests/mir-opt/gvn_overlapping.stable_projection.GVN.diff
new file mode 100644
index 000000000000..088354565911
--- /dev/null
+++ b/tests/mir-opt/gvn_overlapping.stable_projection.GVN.diff
@@ -0,0 +1,19 @@
+- // MIR for `stable_projection` before GVN
++ // MIR for `stable_projection` after GVN
+  
+  fn stable_projection(_1: (Adt,)) -> () {
+      let mut _0: ();
+      let mut _2: *mut Adt;
+      let mut _3: u32;
+      let mut _4: &Adt;
+  
+      bb0: {
+          _2 = &raw mut (_1.0: Adt);
+          _4 = &(*_2);
+          _3 = copy (((*_4) as variant#1).0: u32);
+-         (_1.0: Adt) = Adt::Some(copy _3);
++         (_1.0: Adt) = copy (*_4);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn_repeat.index_place.GVN.diff b/tests/mir-opt/gvn_repeat.index_place.GVN.diff
new file mode 100644
index 000000000000..1eb7e9015cc3
--- /dev/null
+++ b/tests/mir-opt/gvn_repeat.index_place.GVN.diff
@@ -0,0 +1,15 @@
+- // MIR for `index_place` before GVN
++ // MIR for `index_place` after GVN
+  
+  fn index_place(_1: usize, _2: usize, _3: [i32; 5]) -> i32 {
+      let mut _0: i32;
+      let mut _4: &i32;
+  
+      bb0: {
+          _4 = &_3[_1];
+          _1 = copy _2;
+          _0 = copy (*_4);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff b/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff
index fd0478252811..eb3f885b8665 100644
--- a/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff
+++ b/tests/mir-opt/gvn_repeat.repeat_local.GVN.diff
@@ -10,8 +10,7 @@
           _4 = [copy _3; 5];
           _5 = &_4[_1];
           _1 = copy _2;
--         _0 = copy (*_5);
-+         _0 = copy _3;
+          _0 = copy (*_5);
           return;
       }
   }
diff --git a/tests/mir-opt/gvn_repeat.rs b/tests/mir-opt/gvn_repeat.rs
index bbbb2a7ccbaf..49364c6bfd23 100644
--- a/tests/mir-opt/gvn_repeat.rs
+++ b/tests/mir-opt/gvn_repeat.rs
@@ -6,6 +6,23 @@
 
 use std::intrinsics::mir::*;
 
+// EMIT_MIR gvn_repeat.index_place.GVN.diff
+#[custom_mir(dialect = "runtime")]
+pub fn index_place(mut idx1: usize, idx2: usize, array: [i32; 5]) -> i32 {
+    // CHECK-LABEL: fn index_place(
+    // CHECK: let mut [[ELEM:.*]]: &i32;
+    // CHECK: _0 = copy (*[[ELEM]])
+    mir! {
+        let elem;
+        {
+            elem = &array[idx1];
+            idx1 = idx2;
+            RET = *elem;
+            Return()
+        }
+    }
+}
+
 // EMIT_MIR gvn_repeat.repeat_place.GVN.diff
 #[custom_mir(dialect = "runtime")]
 pub fn repeat_place(mut idx1: usize, idx2: usize, val: &i32) -> i32 {
@@ -29,7 +46,8 @@ pub fn repeat_place(mut idx1: usize, idx2: usize, val: &i32) -> i32 {
 #[custom_mir(dialect = "runtime")]
 pub fn repeat_local(mut idx1: usize, idx2: usize, val: i32) -> i32 {
     // CHECK-LABEL: fn repeat_local(
-    // CHECK: _0 = copy _3
+    // CHECK: let mut [[ELEM:.*]]: &i32;
+    // CHECK: _0 = copy (*[[ELEM]]);
     mir! {
         let array;
         let elem;
@@ -44,6 +62,7 @@ pub fn repeat_local(mut idx1: usize, idx2: usize, val: i32) -> i32 {
 }
 
 fn main() {
+    assert_eq!(index_place(0, 5, [0; 5]), 0);
     assert_eq!(repeat_place(0, 5, &0), 0);
     assert_eq!(repeat_local(0, 5, 0), 0);
 }
diff --git a/tests/mir-opt/inline/asm_unwind.rs b/tests/mir-opt/inline/asm_unwind.rs
index 8a102b2f5613..df4dc8f0c333 100644
--- a/tests/mir-opt/inline/asm_unwind.rs
+++ b/tests/mir-opt/inline/asm_unwind.rs
@@ -4,6 +4,7 @@
 //@ needs-asm-support
 //@ needs-unwind
 //@ compile-flags: -Zinline-mir-hint-threshold=1000 -C debuginfo=full
+//@ ignore-backends: gcc
 #![feature(asm_unwind)]
 
 struct D;
diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff
index 0615f8132af0..86e8749fb0b5 100644
--- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff
@@ -9,8 +9,8 @@
       let mut _4: I;
 +     scope 1 (inlined > as FnMut>::call_mut) {
 +         let mut _5: &mut dyn std::ops::FnMut;
-+         let mut _6: std::boxed::Box>;
-+         let mut _7: *const dyn std::ops::FnMut;
++         let mut _6: *const dyn std::ops::FnMut;
++         let mut _7: std::ptr::NonNull>;
 +     }
   
       bb0: {
@@ -22,9 +22,9 @@
 +         StorageLive(_6);
 +         StorageLive(_7);
 +         StorageLive(_5);
-+         _6 = copy (*_3);
-+         _7 = copy ((_6.0: std::ptr::Unique>).0: std::ptr::NonNull>) as *const dyn std::ops::FnMut (Transmute);
-+         _5 = &mut (*_7);
++         _7 = copy (((*_3).0: std::ptr::Unique>).0: std::ptr::NonNull>);
++         _6 = copy _7 as *const dyn std::ops::FnMut (Transmute);
++         _5 = &mut (*_6);
 +         _0 =  as FnMut>::call_mut(move _5, move _4) -> [return: bb2, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff
index 21b20329d4fa..50136ae8766b 100644
--- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff
@@ -9,8 +9,8 @@
       let mut _4: I;
 +     scope 1 (inlined > as FnMut>::call_mut) {
 +         let mut _5: &mut dyn std::ops::FnMut;
-+         let mut _6: std::boxed::Box>;
-+         let mut _7: *const dyn std::ops::FnMut;
++         let mut _6: *const dyn std::ops::FnMut;
++         let mut _7: std::ptr::NonNull>;
 +     }
   
       bb0: {
@@ -22,9 +22,9 @@
 +         StorageLive(_6);
 +         StorageLive(_7);
 +         StorageLive(_5);
-+         _6 = copy (*_3);
-+         _7 = copy ((_6.0: std::ptr::Unique>).0: std::ptr::NonNull>) as *const dyn std::ops::FnMut (Transmute);
-+         _5 = &mut (*_7);
++         _7 = copy (((*_3).0: std::ptr::Unique>).0: std::ptr::NonNull>);
++         _6 = copy _7 as *const dyn std::ops::FnMut (Transmute);
++         _5 = &mut (*_6);
 +         _0 =  as FnMut>::call_mut(move _5, move _4) -> [return: bb4, unwind: bb2];
       }
   
diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff
new file mode 100644
index 000000000000..8e03432c2af2
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+  
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure_inherent.rs:14:6: 14:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_2);
+-         _2 = Foo::callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff
new file mode 100644
index 000000000000..0e41fd89dacf
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.caller-{closure#0}.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller::{closure#0}` before ForceInline
++ // MIR for `caller::{closure#0}` after ForceInline
+  
+  fn caller::{closure#0}(_1: &{closure@$DIR/forced_closure_inherent.rs:14:6: 14:8}) -> () {
+      let mut _0: ();
+      let _2: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_2);
+-         _2 = Foo::callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_closure_inherent.rs b/tests/mir-opt/inline/forced_closure_inherent.rs
new file mode 100644
index 000000000000..949c7d6ecbf2
--- /dev/null
+++ b/tests/mir-opt/inline/forced_closure_inherent.rs
@@ -0,0 +1,19 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_closure_inherent.caller-{closure#0}.ForceInline.diff
+pub fn caller() {
+    (|| {
+        Foo::callee_forced();
+        // CHECK-LABEL: fn caller::{closure#0}(
+        // CHECK: (inlined Foo::callee_forced)
+    })();
+}
diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff
new file mode 100644
index 000000000000..6ea1894af989
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 000000000000..dd91c3387723
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent.rs b/tests/mir-opt/inline/forced_inherent.rs
new file mode 100644
index 000000000000..24bf8daa6445
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent.rs
@@ -0,0 +1,17 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo;
+
+impl Foo {
+    #[rustc_force_inline]
+    fn bar() {}
+}
+
+// EMIT_MIR forced_inherent.caller.ForceInline.diff
+fn caller() {
+    Foo::bar();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::bar)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff
new file mode 100644
index 000000000000..6ea1894af989
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 000000000000..dd91c3387723
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::bar) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::bar() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_ambiguous.rs b/tests/mir-opt/inline/forced_inherent_ambiguous.rs
new file mode 100644
index 000000000000..e3c5d3e4f9e8
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_ambiguous.rs
@@ -0,0 +1,25 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+#![feature(rustc_attrs)]
+
+struct Foo;
+
+impl Foo {
+    #[rustc_force_inline]
+    fn bar() {}
+}
+
+trait Tr {
+    fn bar();
+}
+
+impl Tr for Foo {
+    fn bar() {}
+}
+
+// EMIT_MIR forced_inherent_ambiguous.caller.ForceInline.diff
+fn caller() {
+    Foo::bar();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::bar)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff
new file mode 100644
index 000000000000..6495ddbafba4
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+  
+      bb0: {
+          _0 = {coroutine@$DIR/forced_inherent_async.rs:14:19: 18:2 (#0)};
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 000000000000..6495ddbafba4
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,12 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> {async fn body of caller()} {
+      let mut _0: {async fn body of caller()};
+  
+      bb0: {
+          _0 = {coroutine@$DIR/forced_inherent_async.rs:14:19: 18:2 (#0)};
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_async.rs b/tests/mir-opt/inline/forced_inherent_async.rs
new file mode 100644
index 000000000000..ce58a0ac48f4
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_async.rs
@@ -0,0 +1,18 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 --crate-type=lib
+//@ edition: 2021
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_inherent_async.caller.ForceInline.diff
+async fn caller() {
+    Foo::callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::callee_forced)
+}
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff
new file mode 100644
index 000000000000..edaf2820d85c
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-abort.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::callee_forced() -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff
new file mode 100644
index 000000000000..22f8b14a724b
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.caller.ForceInline.panic-unwind.diff
@@ -0,0 +1,21 @@
+- // MIR for `caller` before ForceInline
++ // MIR for `caller` after ForceInline
+  
+  fn caller() -> () {
+      let mut _0: ();
+      let _1: ();
++     scope 1 (inlined Foo::callee_forced) {
++     }
+  
+      bb0: {
+          StorageLive(_1);
+-         _1 = Foo::callee_forced() -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_1);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/forced_inherent_dead_code.rs b/tests/mir-opt/inline/forced_inherent_dead_code.rs
new file mode 100644
index 000000000000..057a4cac5287
--- /dev/null
+++ b/tests/mir-opt/inline/forced_inherent_dead_code.rs
@@ -0,0 +1,21 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ compile-flags: -Copt-level=0 -Clink-dead-code
+#![feature(rustc_attrs)]
+
+struct Foo {}
+
+impl Foo {
+    #[rustc_force_inline]
+    pub fn callee_forced() {}
+}
+
+// EMIT_MIR forced_inherent_dead_code.caller.ForceInline.diff
+pub fn caller() {
+    Foo::callee_forced();
+    // CHECK-LABEL: fn caller(
+    // CHECK: (inlined Foo::callee_forced)
+}
+
+fn main() {
+    caller();
+}
diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff
index ecea7a97513d..97a8a99d9805 100644
--- a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff
@@ -9,8 +9,8 @@
       let mut _4: (i32,);
 +     scope 1 (inlined  as Fn<(i32,)>>::call) {
 +         let mut _5: &dyn std::ops::Fn(i32);
-+         let mut _6: std::boxed::Box;
-+         let mut _7: *const dyn std::ops::Fn(i32);
++         let mut _6: *const dyn std::ops::Fn(i32);
++         let mut _7: std::ptr::NonNull;
 +     }
   
       bb0: {
@@ -23,9 +23,9 @@
 +         StorageLive(_6);
 +         StorageLive(_7);
 +         StorageLive(_5);
-+         _6 = copy (*_3);
-+         _7 = copy ((_6.0: std::ptr::Unique).0: std::ptr::NonNull) as *const dyn std::ops::Fn(i32) (Transmute);
-+         _5 = &(*_7);
++         _7 = copy (((*_3).0: std::ptr::Unique).0: std::ptr::NonNull);
++         _6 = copy _7 as *const dyn std::ops::Fn(i32) (Transmute);
++         _5 = &(*_6);
 +         _2 = >::call(move _5, move _4) -> [return: bb2, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff
index 3a4a528e879f..3b18052616e2 100644
--- a/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff
@@ -9,8 +9,8 @@
       let mut _4: (i32,);
 +     scope 1 (inlined  as Fn<(i32,)>>::call) {
 +         let mut _5: &dyn std::ops::Fn(i32);
-+         let mut _6: std::boxed::Box;
-+         let mut _7: *const dyn std::ops::Fn(i32);
++         let mut _6: *const dyn std::ops::Fn(i32);
++         let mut _7: std::ptr::NonNull;
 +     }
   
       bb0: {
@@ -23,9 +23,9 @@
 +         StorageLive(_6);
 +         StorageLive(_7);
 +         StorageLive(_5);
-+         _6 = copy (*_3);
-+         _7 = copy ((_6.0: std::ptr::Unique).0: std::ptr::NonNull) as *const dyn std::ops::Fn(i32) (Transmute);
-+         _5 = &(*_7);
++         _7 = copy (((*_3).0: std::ptr::Unique).0: std::ptr::NonNull);
++         _6 = copy _7 as *const dyn std::ops::Fn(i32) (Transmute);
++         _5 = &(*_6);
 +         _2 = >::call(move _5, move _4) -> [return: bb4, unwind: bb2];
       }
   
diff --git a/tests/mir-opt/inline/inline_compatibility.rs b/tests/mir-opt/inline/inline_compatibility.rs
index 1bb102ccda58..2b8280a3cfb0 100644
--- a/tests/mir-opt/inline/inline_compatibility.rs
+++ b/tests/mir-opt/inline/inline_compatibility.rs
@@ -1,9 +1,10 @@
 // Checks that only functions with compatible attributes are inlined.
 //@ only-x86_64
 //@ compile-flags: -Cpanic=abort
+//@ ignore-backends: gcc
 
 #![crate_type = "lib"]
-#![feature(no_sanitize)]
+#![feature(sanitize)]
 #![feature(c_variadic)]
 
 #[inline]
@@ -37,22 +38,22 @@ pub unsafe fn f2() {
 }
 
 #[inline]
-#[no_sanitize(address)]
-pub unsafe fn no_sanitize() {}
+#[sanitize(address = "off")]
+pub unsafe fn sanitize_off() {}
 
-// CHECK-LABEL: fn inlined_no_sanitize()
+// CHECK-LABEL: fn inlined_sanitize_off()
 // CHECK:       bb0: {
 // CHECK-NEXT:  return;
-#[no_sanitize(address)]
-pub unsafe fn inlined_no_sanitize() {
-    no_sanitize();
+#[sanitize(address = "off")]
+pub unsafe fn inlined_sanitize_off() {
+    sanitize_off();
 }
 
-// CHECK-LABEL: fn not_inlined_no_sanitize()
+// CHECK-LABEL: fn not_inlined_sanitize_off()
 // CHECK:       bb0: {
-// CHECK-NEXT:  no_sanitize()
-pub unsafe fn not_inlined_no_sanitize() {
-    no_sanitize();
+// CHECK-NEXT:  sanitize_off()
+pub unsafe fn not_inlined_sanitize_off() {
+    sanitize_off();
 }
 
 // CHECK-LABEL: fn not_inlined_c_variadic()
diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
index f6c111a2228a..9509739413b7 100644
--- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff
@@ -84,7 +84,6 @@
           _5 = copy _2;
 -         _0 = drop_in_place::>(move _5) -> [return: bb2, unwind unreachable];
 +         StorageLive(_15);
-+         StorageLive(_16);
 +         _15 = discriminant((*_5));
 +         switchInt(move _15) -> [0: bb5, otherwise: bb6];
       }
@@ -110,7 +109,6 @@
 +     }
 + 
 +     bb5: {
-+         StorageDead(_16);
 +         StorageDead(_15);
           StorageDead(_5);
           return;
diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff
index 183242764252..34f89da19f51 100644
--- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-unwind.diff
@@ -27,13 +27,11 @@
           _5 = copy _2;
 -         _0 = drop_in_place::>(move _5) -> [return: bb2, unwind continue];
 +         StorageLive(_6);
-+         StorageLive(_7);
 +         _6 = discriminant((*_5));
 +         switchInt(move _6) -> [0: bb2, otherwise: bb3];
       }
   
       bb2: {
-+         StorageDead(_7);
 +         StorageDead(_6);
           StorageDead(_5);
           return;
diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir
index 103475b608ce..688e6967c2f8 100644
--- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir
+++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.b.Inline.after.mir
@@ -8,8 +8,8 @@ fn b(_1: &mut Box) -> &mut T {
     let mut _4: &mut std::boxed::Box;
     scope 1 (inlined  as AsMut>::as_mut) {
         debug self => _4;
-        let mut _5: std::boxed::Box;
-        let mut _6: *const T;
+        let mut _5: *const T;
+        let mut _6: std::ptr::NonNull;
     }
 
     bb0: {
@@ -19,9 +19,9 @@ fn b(_1: &mut Box) -> &mut T {
         _4 = copy _1;
         StorageLive(_5);
         StorageLive(_6);
-        _5 = copy (*_4);
-        _6 = copy ((_5.0: std::ptr::Unique).0: std::ptr::NonNull) as *const T (Transmute);
-        _3 = &mut (*_6);
+        _6 = copy (((*_4).0: std::ptr::Unique).0: std::ptr::NonNull);
+        _5 = copy _6 as *const T (Transmute);
+        _3 = &mut (*_5);
         StorageDead(_6);
         StorageDead(_5);
         _2 = copy _3;
diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir
index babb26808cec..663741c62fb8 100644
--- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir
+++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.d.Inline.after.mir
@@ -7,8 +7,8 @@ fn d(_1: &Box) -> &T {
     let mut _3: &std::boxed::Box;
     scope 1 (inlined  as AsRef>::as_ref) {
         debug self => _3;
-        let mut _4: std::boxed::Box;
-        let mut _5: *const T;
+        let mut _4: *const T;
+        let mut _5: std::ptr::NonNull;
     }
 
     bb0: {
@@ -17,9 +17,9 @@ fn d(_1: &Box) -> &T {
         _3 = copy _1;
         StorageLive(_4);
         StorageLive(_5);
-        _4 = copy (*_3);
-        _5 = copy ((_4.0: std::ptr::Unique).0: std::ptr::NonNull) as *const T (Transmute);
-        _2 = &(*_5);
+        _5 = copy (((*_3).0: std::ptr::Unique).0: std::ptr::NonNull);
+        _4 = copy _5 as *const T (Transmute);
+        _2 = &(*_4);
         StorageDead(_5);
         StorageDead(_4);
         _0 = copy _2;
diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff
index 22e6ea722dda..4c1b25c786ef 100644
--- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff
@@ -39,16 +39,8 @@
 +                 let mut _28: &mut std::task::Context<'_>;
 +                 let mut _29: ();
 +                 let mut _30: ();
-+                 let mut _31: &mut std::task::Context<'_>;
-+                 let mut _32: u32;
-+                 let mut _33: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _34: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _35: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _36: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _37: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _38: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _39: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _40: &mut {async fn body of ActionPermit<'_, T>::perform()};
++                 let mut _31: u32;
++                 let mut _32: &mut {async fn body of ActionPermit<'_, T>::perform()};
 +                 scope 7 {
 +                     let mut _15: std::future::Ready<()>;
 +                     scope 8 {
@@ -58,39 +50,37 @@
 +                         scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
 +                         }
 +                         scope 13 (inlined  as Future>::poll) {
-+                             let mut _42: ();
-+                             let mut _43: std::option::Option<()>;
-+                             let mut _44: &mut std::option::Option<()>;
-+                             let mut _45: &mut std::future::Ready<()>;
-+                             let mut _46: &mut std::pin::Pin<&mut std::future::Ready<()>>;
++                             let mut _34: ();
++                             let mut _35: std::option::Option<()>;
++                             let mut _36: &mut std::option::Option<()>;
++                             let mut _37: &mut std::future::Ready<()>;
++                             let mut _38: &mut std::pin::Pin<&mut std::future::Ready<()>>;
 +                             scope 14 (inlined > as DerefMut>::deref_mut) {
-+                                 scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) {
-+                                     let mut _47: &mut &mut std::future::Ready<()>;
-+                                     scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
-+                                     }
-+                                     scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
-+                                     }
-+                                 }
-+                                 scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) {
-+                                 }
-+                             }
-+                             scope 19 (inlined Option::<()>::take) {
-+                                 let mut _48: std::option::Option<()>;
-+                                 scope 20 (inlined std::mem::replace::>) {
-+                                     scope 21 {
++                                 let mut _39: *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>>;
++                                 let mut _40: *mut std::pin::Pin<&mut std::future::Ready<()>>;
++                                 scope 15 (inlined > as pin::helper::PinDerefMutHelper>::deref_mut) {
++                                     let mut _41: &mut &mut std::future::Ready<()>;
++                                     scope 16 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
 +                                     }
 +                                 }
 +                             }
-+                             scope 22 (inlined #[track_caller] Option::<()>::expect) {
-+                                 let mut _49: isize;
-+                                 let mut _50: !;
-+                                 scope 23 {
++                             scope 17 (inlined Option::<()>::take) {
++                                 let mut _42: std::option::Option<()>;
++                                 scope 18 (inlined std::mem::replace::>) {
++                                     scope 19 {
++                                     }
++                                 }
++                             }
++                             scope 20 (inlined #[track_caller] Option::<()>::expect) {
++                                 let mut _43: isize;
++                                 let mut _44: !;
++                                 scope 21 {
 +                                 }
 +                             }
 +                         }
 +                     }
 +                     scope 10 (inlined ready::<()>) {
-+                         let mut _41: std::option::Option<()>;
++                         let mut _33: std::option::Option<()>;
 +                     }
 +                     scope 11 (inlined  as IntoFuture>::into_future) {
 +                     }
@@ -127,28 +117,17 @@
           StorageLive(_8);
           _8 = move _4;
           StorageLive(_9);
-          _10 = deref_copy (_1.1: &mut std::task::Context<'_>);
+          _10 = copy (_1.1: &mut std::task::Context<'_>);
           _9 = &mut (*_10);
 -         _7 = <{async fn body of ActionPermit<'_, T>::perform()} as Future>::poll(move _8, move _9) -> [return: bb3, unwind unreachable];
-+         StorageLive(_11);
-+         StorageLive(_15);
 +         StorageLive(_16);
 +         StorageLive(_25);
-+         StorageLive(_27);
 +         StorageLive(_30);
 +         StorageLive(_31);
 +         StorageLive(_32);
-+         StorageLive(_33);
-+         StorageLive(_34);
-+         StorageLive(_35);
-+         StorageLive(_36);
-+         StorageLive(_37);
-+         StorageLive(_38);
-+         StorageLive(_39);
-+         StorageLive(_40);
-+         _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _32 = discriminant((*_33));
-+         switchInt(move _32) -> [0: bb3, 1: bb10, 3: bb9, otherwise: bb5];
++         _32 = copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
++         _31 = discriminant((*_32));
++         switchInt(move _31) -> [0: bb3, 1: bb10, 3: bb9, otherwise: bb5];
       }
   
 -     bb3: {
@@ -158,22 +137,11 @@
 +     }
 + 
 +     bb2: {
-+         StorageDead(_40);
-+         StorageDead(_39);
-+         StorageDead(_38);
-+         StorageDead(_37);
-+         StorageDead(_36);
-+         StorageDead(_35);
-+         StorageDead(_34);
-+         StorageDead(_33);
 +         StorageDead(_32);
 +         StorageDead(_31);
 +         StorageDead(_30);
-+         StorageDead(_27);
 +         StorageDead(_25);
 +         StorageDead(_16);
-+         StorageDead(_15);
-+         StorageDead(_11);
           StorageDead(_9);
           StorageDead(_8);
           StorageDead(_7);
@@ -186,23 +154,19 @@
       }
   
 +     bb3: {
-+         _31 = move _9;
-+         _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>);
++         (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>);
 +         StorageLive(_12);
 +         StorageLive(_13);
 +         StorageLive(_14);
 +         _14 = ();
-+         StorageLive(_41);
-+         _41 = Option::<()>::Some(copy _14);
-+         _13 = std::future::Ready::<()>(move _41);
-+         StorageDead(_41);
++         StorageLive(_33);
++         _33 = Option::<()>::Some(copy _14);
++         _13 = std::future::Ready::<()>(move _33);
++         StorageDead(_33);
 +         StorageDead(_14);
 +         _12 = move _13;
 +         StorageDead(_13);
-+         _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         (((*_36) as variant#3).1: std::future::Ready<()>) = move _12;
++         (((*_32) as variant#3).1: std::future::Ready<()>) = move _12;
 +         goto -> bb4;
 +     }
 + 
@@ -214,39 +178,35 @@
 +         StorageLive(_19);
 +         StorageLive(_20);
 +         StorageLive(_21);
-+         _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>);
++         _21 = &mut (((*_32) as variant#3).1: std::future::Ready<()>);
 +         _20 = &mut (*_21);
 +         _19 = Pin::<&mut std::future::Ready<()>> { pointer: copy _20 };
 +         StorageDead(_20);
 +         StorageLive(_22);
 +         StorageLive(_23);
 +         StorageLive(_24);
-+         _24 = copy _31;
++         _24 = copy _9;
 +         _23 = move _24;
 +         _22 = &mut (*_23);
 +         StorageDead(_24);
-+         StorageLive(_45);
-+         StorageLive(_46);
-+         StorageLive(_50);
-+         StorageLive(_42);
-+         StorageLive(_43);
++         StorageLive(_37);
++         StorageLive(_39);
 +         StorageLive(_44);
-+         _46 = &mut _19;
-+         StorageLive(_47);
-+         _47 = &mut (_19.0: &mut std::future::Ready<()>);
-+         _45 = copy (_19.0: &mut std::future::Ready<()>);
-+         StorageDead(_47);
-+         _44 = &mut ((*_45).0: std::option::Option<()>);
-+         StorageLive(_48);
-+         _48 = Option::<()>::None;
-+         _43 = copy ((*_45).0: std::option::Option<()>);
-+         ((*_45).0: std::option::Option<()>) = copy _48;
-+         StorageDead(_48);
-+         StorageDead(_44);
-+         StorageLive(_49);
-+         _49 = discriminant(_43);
-+         switchInt(move _49) -> [0: bb11, 1: bb12, otherwise: bb5];
++         StorageLive(_34);
++         StorageLive(_35);
++         StorageLive(_40);
++         _40 = &raw mut _19;
++         _39 = copy _40 as *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>> (PtrToPtr);
++         StorageDead(_40);
++         _37 = copy ((*_39).0: &mut std::future::Ready<()>);
++         StorageLive(_42);
++         _42 = Option::<()>::None;
++         _35 = copy ((*_37).0: std::option::Option<()>);
++         ((*_37).0: std::option::Option<()>) = copy _42;
++         StorageDead(_42);
++         StorageLive(_43);
++         _43 = discriminant(_35);
++         switchInt(move _43) -> [0: bb11, 1: bb12, otherwise: bb5];
       }
 + 
 +     bb5: {
@@ -266,8 +226,7 @@
 +         StorageDead(_12);
 +         StorageDead(_28);
 +         StorageDead(_29);
-+         _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         discriminant((*_38)) = 3;
++         discriminant((*_32)) = 3;
 +         goto -> bb2;
 +     }
 + 
@@ -281,14 +240,12 @@
 +         StorageDead(_18);
 +         StorageDead(_17);
 +         StorageDead(_12);
-+         _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb8, unwind unreachable];
++         drop((((*_32) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb8, unwind unreachable];
 +     }
 + 
 +     bb8: {
 +         _7 = Poll::<()>::Ready(move _30);
-+         _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         discriminant((*_40)) = 1;
++         discriminant((*_32)) = 1;
 +         goto -> bb2;
 +     }
 + 
@@ -298,7 +255,7 @@
 +         StorageLive(_29);
 +         _28 = move _9;
 +         StorageDead(_29);
-+         _31 = move _28;
++         _9 = move _28;
 +         StorageDead(_28);
 +         _16 = const ();
 +         goto -> bb4;
@@ -309,18 +266,18 @@
 +     }
 + 
 +     bb11: {
-+         _50 = option::expect_failed(const "`Ready` polled after completion") -> unwind unreachable;
++         _44 = option::expect_failed(const "`Ready` polled after completion") -> unwind unreachable;
 +     }
 + 
 +     bb12: {
-+         _42 = move ((_43 as Some).0: ());
-+         StorageDead(_49);
++         _34 = move ((_35 as Some).0: ());
 +         StorageDead(_43);
-+         _18 = Poll::<()>::Ready(move _42);
-+         StorageDead(_42);
-+         StorageDead(_50);
-+         StorageDead(_46);
-+         StorageDead(_45);
++         StorageDead(_35);
++         _18 = Poll::<()>::Ready(move _34);
++         StorageDead(_34);
++         StorageDead(_44);
++         StorageDead(_39);
++         StorageDead(_37);
 +         StorageDead(_22);
 +         StorageDead(_19);
 +         _25 = discriminant(_18);
diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff
index 8b027e988b8e..59417ce64651 100644
--- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff
@@ -39,18 +39,8 @@
 +                 let mut _28: &mut std::task::Context<'_>;
 +                 let mut _29: ();
 +                 let mut _30: ();
-+                 let mut _31: &mut std::task::Context<'_>;
-+                 let mut _32: u32;
-+                 let mut _33: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _34: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _35: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _36: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _37: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _38: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _39: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _40: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _41: &mut {async fn body of ActionPermit<'_, T>::perform()};
-+                 let mut _42: &mut {async fn body of ActionPermit<'_, T>::perform()};
++                 let mut _31: u32;
++                 let mut _32: &mut {async fn body of ActionPermit<'_, T>::perform()};
 +                 scope 7 {
 +                     let mut _15: std::future::Ready<()>;
 +                     scope 8 {
@@ -60,39 +50,37 @@
 +                         scope 12 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
 +                         }
 +                         scope 13 (inlined  as Future>::poll) {
-+                             let mut _44: ();
-+                             let mut _45: std::option::Option<()>;
-+                             let mut _46: &mut std::option::Option<()>;
-+                             let mut _47: &mut std::future::Ready<()>;
-+                             let mut _48: &mut std::pin::Pin<&mut std::future::Ready<()>>;
++                             let mut _34: ();
++                             let mut _35: std::option::Option<()>;
++                             let mut _36: &mut std::option::Option<()>;
++                             let mut _37: &mut std::future::Ready<()>;
++                             let mut _38: &mut std::pin::Pin<&mut std::future::Ready<()>>;
 +                             scope 14 (inlined > as DerefMut>::deref_mut) {
-+                                 scope 15 (inlined Pin::<&mut std::future::Ready<()>>::as_mut) {
-+                                     let mut _49: &mut &mut std::future::Ready<()>;
-+                                     scope 16 (inlined Pin::<&mut std::future::Ready<()>>::new_unchecked) {
-+                                     }
-+                                     scope 18 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
-+                                     }
-+                                 }
-+                                 scope 17 (inlined Pin::<&mut std::future::Ready<()>>::get_mut) {
-+                                 }
-+                             }
-+                             scope 19 (inlined Option::<()>::take) {
-+                                 let mut _50: std::option::Option<()>;
-+                                 scope 20 (inlined std::mem::replace::>) {
-+                                     scope 21 {
++                                 let mut _39: *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>>;
++                                 let mut _40: *mut std::pin::Pin<&mut std::future::Ready<()>>;
++                                 scope 15 (inlined > as pin::helper::PinDerefMutHelper>::deref_mut) {
++                                     let mut _41: &mut &mut std::future::Ready<()>;
++                                     scope 16 (inlined <&mut std::future::Ready<()> as DerefMut>::deref_mut) {
 +                                     }
 +                                 }
 +                             }
-+                             scope 22 (inlined #[track_caller] Option::<()>::expect) {
-+                                 let mut _51: isize;
-+                                 let mut _52: !;
-+                                 scope 23 {
++                             scope 17 (inlined Option::<()>::take) {
++                                 let mut _42: std::option::Option<()>;
++                                 scope 18 (inlined std::mem::replace::>) {
++                                     scope 19 {
++                                     }
++                                 }
++                             }
++                             scope 20 (inlined #[track_caller] Option::<()>::expect) {
++                                 let mut _43: isize;
++                                 let mut _44: !;
++                                 scope 21 {
 +                                 }
 +                             }
 +                         }
 +                     }
 +                     scope 10 (inlined ready::<()>) {
-+                         let mut _43: std::option::Option<()>;
++                         let mut _33: std::option::Option<()>;
 +                     }
 +                     scope 11 (inlined  as IntoFuture>::into_future) {
 +                     }
@@ -129,30 +117,17 @@
           StorageLive(_8);
           _8 = move _4;
           StorageLive(_9);
-          _10 = deref_copy (_1.1: &mut std::task::Context<'_>);
+          _10 = copy (_1.1: &mut std::task::Context<'_>);
           _9 = &mut (*_10);
 -         _7 = <{async fn body of ActionPermit<'_, T>::perform()} as Future>::poll(move _8, move _9) -> [return: bb3, unwind: bb5];
-+         StorageLive(_11);
-+         StorageLive(_15);
 +         StorageLive(_16);
 +         StorageLive(_25);
-+         StorageLive(_27);
 +         StorageLive(_30);
 +         StorageLive(_31);
 +         StorageLive(_32);
-+         StorageLive(_33);
-+         StorageLive(_34);
-+         StorageLive(_35);
-+         StorageLive(_36);
-+         StorageLive(_37);
-+         StorageLive(_38);
-+         StorageLive(_39);
-+         StorageLive(_40);
-+         StorageLive(_41);
-+         StorageLive(_42);
-+         _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _32 = discriminant((*_33));
-+         switchInt(move _32) -> [0: bb5, 1: bb15, 2: bb14, 3: bb13, otherwise: bb7];
++         _32 = copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
++         _31 = discriminant((*_32));
++         switchInt(move _31) -> [0: bb5, 1: bb15, 2: bb14, 3: bb13, otherwise: bb7];
       }
   
 -     bb3: {
@@ -170,24 +145,11 @@
 +     }
 + 
 +     bb4: {
-+         StorageDead(_42);
-+         StorageDead(_41);
-+         StorageDead(_40);
-+         StorageDead(_39);
-+         StorageDead(_38);
-+         StorageDead(_37);
-+         StorageDead(_36);
-+         StorageDead(_35);
-+         StorageDead(_34);
-+         StorageDead(_33);
 +         StorageDead(_32);
 +         StorageDead(_31);
 +         StorageDead(_30);
-+         StorageDead(_27);
 +         StorageDead(_25);
 +         StorageDead(_16);
-+         StorageDead(_15);
-+         StorageDead(_11);
           StorageDead(_9);
           StorageDead(_8);
           StorageDead(_7);
@@ -203,23 +165,19 @@
 -         StorageDead(_2);
 -         return;
 +     bb5: {
-+         _31 = move _9;
-+         _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>);
++         (((*_32) as variant#3).0: ActionPermit<'_, T>) = move ((*_32).0: ActionPermit<'_, T>);
 +         StorageLive(_12);
 +         StorageLive(_13);
 +         StorageLive(_14);
 +         _14 = ();
-+         StorageLive(_43);
-+         _43 = Option::<()>::Some(copy _14);
-+         _13 = std::future::Ready::<()>(move _43);
-+         StorageDead(_43);
++         StorageLive(_33);
++         _33 = Option::<()>::Some(copy _14);
++         _13 = std::future::Ready::<()>(move _33);
++         StorageDead(_33);
 +         StorageDead(_14);
 +         _12 = move _13;
 +         StorageDead(_13);
-+         _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         (((*_36) as variant#3).1: std::future::Ready<()>) = move _12;
++         (((*_32) as variant#3).1: std::future::Ready<()>) = move _12;
 +         goto -> bb6;
       }
   
@@ -231,39 +189,35 @@
 +         StorageLive(_19);
 +         StorageLive(_20);
 +         StorageLive(_21);
-+         _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>);
++         _21 = &mut (((*_32) as variant#3).1: std::future::Ready<()>);
 +         _20 = &mut (*_21);
 +         _19 = Pin::<&mut std::future::Ready<()>> { pointer: copy _20 };
 +         StorageDead(_20);
 +         StorageLive(_22);
 +         StorageLive(_23);
 +         StorageLive(_24);
-+         _24 = copy _31;
++         _24 = copy _9;
 +         _23 = move _24;
 +         _22 = &mut (*_23);
 +         StorageDead(_24);
-+         StorageLive(_47);
-+         StorageLive(_48);
-+         StorageLive(_52);
++         StorageLive(_37);
++         StorageLive(_39);
 +         StorageLive(_44);
-+         StorageLive(_45);
-+         StorageLive(_46);
-+         _48 = &mut _19;
-+         StorageLive(_49);
-+         _49 = &mut (_19.0: &mut std::future::Ready<()>);
-+         _47 = copy (_19.0: &mut std::future::Ready<()>);
-+         StorageDead(_49);
-+         _46 = &mut ((*_47).0: std::option::Option<()>);
-+         StorageLive(_50);
-+         _50 = Option::<()>::None;
-+         _45 = copy ((*_47).0: std::option::Option<()>);
-+         ((*_47).0: std::option::Option<()>) = copy _50;
-+         StorageDead(_50);
-+         StorageDead(_46);
-+         StorageLive(_51);
-+         _51 = discriminant(_45);
-+         switchInt(move _51) -> [0: bb16, 1: bb17, otherwise: bb7];
++         StorageLive(_34);
++         StorageLive(_35);
++         StorageLive(_40);
++         _40 = &raw mut _19;
++         _39 = copy _40 as *mut std::pin::helper::PinHelper<&mut std::future::Ready<()>> (PtrToPtr);
++         StorageDead(_40);
++         _37 = copy ((*_39).0: &mut std::future::Ready<()>);
++         StorageLive(_42);
++         _42 = Option::<()>::None;
++         _35 = copy ((*_37).0: std::option::Option<()>);
++         ((*_37).0: std::option::Option<()>) = copy _42;
++         StorageDead(_42);
++         StorageLive(_43);
++         _43 = discriminant(_35);
++         switchInt(move _43) -> [0: bb16, 1: bb17, otherwise: bb7];
       }
   
 -     bb6 (cleanup): {
@@ -285,8 +239,7 @@
 +         StorageDead(_12);
 +         StorageDead(_28);
 +         StorageDead(_29);
-+         _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         discriminant((*_38)) = 3;
++         discriminant((*_32)) = 3;
 +         goto -> bb4;
 +     }
 + 
@@ -300,14 +253,12 @@
 +         StorageDead(_18);
 +         StorageDead(_17);
 +         StorageDead(_12);
-+         _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb10, unwind: bb12];
++         drop((((*_32) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb10, unwind: bb12];
 +     }
 + 
 +     bb10: {
 +         _7 = Poll::<()>::Ready(move _30);
-+         _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         discriminant((*_40)) = 1;
++         discriminant((*_32)) = 1;
 +         goto -> bb4;
 +     }
 + 
@@ -319,13 +270,11 @@
 +         StorageDead(_18);
 +         StorageDead(_17);
 +         StorageDead(_12);
-+         _41 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb12, unwind terminate(cleanup)];
++         drop((((*_32) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb12, unwind terminate(cleanup)];
 +     }
 + 
 +     bb12 (cleanup): {
-+         _42 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()});
-+         discriminant((*_42)) = 2;
++         discriminant((*_32)) = 2;
 +         goto -> bb2;
 +     }
 + 
@@ -335,7 +284,7 @@
 +         StorageLive(_29);
 +         _28 = move _9;
 +         StorageDead(_29);
-+         _31 = move _28;
++         _9 = move _28;
 +         StorageDead(_28);
 +         _16 = const ();
 +         goto -> bb6;
@@ -350,18 +299,18 @@
 +     }
 + 
 +     bb16: {
-+         _52 = option::expect_failed(const "`Ready` polled after completion") -> bb11;
++         _44 = option::expect_failed(const "`Ready` polled after completion") -> bb11;
 +     }
 + 
 +     bb17: {
-+         _44 = move ((_45 as Some).0: ());
-+         StorageDead(_51);
-+         StorageDead(_45);
-+         _18 = Poll::<()>::Ready(move _44);
++         _34 = move ((_35 as Some).0: ());
++         StorageDead(_43);
++         StorageDead(_35);
++         _18 = Poll::<()>::Ready(move _34);
++         StorageDead(_34);
 +         StorageDead(_44);
-+         StorageDead(_52);
-+         StorageDead(_48);
-+         StorageDead(_47);
++         StorageDead(_39);
++         StorageDead(_37);
 +         StorageDead(_22);
 +         StorageDead(_19);
 +         _25 = discriminant(_18);
diff --git a/tests/mir-opt/instsimplify/align_of_slice.of_val_slice.InstSimplify-after-simplifycfg.diff b/tests/mir-opt/instsimplify/align_of_slice.of_val_slice.InstSimplify-after-simplifycfg.diff
index 042e19bf2aa1..bbcc9012a25d 100644
--- a/tests/mir-opt/instsimplify/align_of_slice.of_val_slice.InstSimplify-after-simplifycfg.diff
+++ b/tests/mir-opt/instsimplify/align_of_slice.of_val_slice.InstSimplify-after-simplifycfg.diff
@@ -10,7 +10,7 @@
           StorageLive(_2);
           _2 = &raw const (*_1);
 -         _0 = std::intrinsics::align_of_val::<[T]>(move _2) -> [return: bb1, unwind unreachable];
-+         _0 = AlignOf(T);
++         _0 = const ::ALIGN;
 +         goto -> bb1;
       }
   
diff --git a/tests/mir-opt/instsimplify/align_of_slice.rs b/tests/mir-opt/instsimplify/align_of_slice.rs
index 0af05cb6b0b0..99a68e444fe5 100644
--- a/tests/mir-opt/instsimplify/align_of_slice.rs
+++ b/tests/mir-opt/instsimplify/align_of_slice.rs
@@ -7,6 +7,6 @@
 // EMIT_MIR align_of_slice.of_val_slice.InstSimplify-after-simplifycfg.diff
 pub fn of_val_slice(slice: &[T]) -> usize {
     // CHECK-LABEL: fn of_val_slice(_1: &[T])
-    // CHECK: _0 = AlignOf(T);
+    // CHECK: _0 = const ::ALIGN;
     unsafe { core::intrinsics::align_of_val(slice) }
 }
diff --git a/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-abort.diff b/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-abort.diff
new file mode 100644
index 000000000000..feba0fe7c4a2
--- /dev/null
+++ b/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-abort.diff
@@ -0,0 +1,71 @@
+- // MIR for `norm2` before InstSimplify-after-simplifycfg
++ // MIR for `norm2` after InstSimplify-after-simplifycfg
+  
+  fn norm2(_1: [f32; 2]) -> f32 {
+      debug x => _1;
+      let mut _0: f32;
+      let _2: f32;
+      let _3: usize;
+      let mut _4: bool;
+      let _6: usize;
+      let mut _7: bool;
+      let mut _8: f32;
+      let mut _9: f32;
+      let mut _10: f32;
+      let mut _11: f32;
+      let mut _12: f32;
+      let mut _13: f32;
+      scope 1 {
+          debug a => _2;
+          let _5: f32;
+          scope 2 {
+              debug b => _5;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const 0_usize;
+          _4 = Lt(copy _3, const 2_usize);
+          assert(move _4, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _3) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _2 = copy _1[_3];
+          StorageDead(_3);
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = const 1_usize;
+          _7 = Lt(copy _6, const 2_usize);
+          assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _6) -> [success: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          _5 = copy _1[_6];
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _2;
+          StorageLive(_10);
+          _10 = copy _2;
+          _8 = Mul(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = copy _5;
+          StorageLive(_13);
+          _13 = copy _5;
+          _11 = Mul(move _12, move _13);
+          StorageDead(_13);
+          StorageDead(_12);
+          _0 = Add(move _8, move _11);
+          StorageDead(_11);
+          StorageDead(_8);
+          StorageDead(_5);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-unwind.diff b/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-unwind.diff
new file mode 100644
index 000000000000..0ccf6a825c49
--- /dev/null
+++ b/tests/mir-opt/instsimplify/combine_array_len.norm2.InstSimplify-after-simplifycfg.panic-unwind.diff
@@ -0,0 +1,71 @@
+- // MIR for `norm2` before InstSimplify-after-simplifycfg
++ // MIR for `norm2` after InstSimplify-after-simplifycfg
+  
+  fn norm2(_1: [f32; 2]) -> f32 {
+      debug x => _1;
+      let mut _0: f32;
+      let _2: f32;
+      let _3: usize;
+      let mut _4: bool;
+      let _6: usize;
+      let mut _7: bool;
+      let mut _8: f32;
+      let mut _9: f32;
+      let mut _10: f32;
+      let mut _11: f32;
+      let mut _12: f32;
+      let mut _13: f32;
+      scope 1 {
+          debug a => _2;
+          let _5: f32;
+          scope 2 {
+              debug b => _5;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const 0_usize;
+          _4 = Lt(copy _3, const 2_usize);
+          assert(move _4, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _3) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+          _2 = copy _1[_3];
+          StorageDead(_3);
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = const 1_usize;
+          _7 = Lt(copy _6, const 2_usize);
+          assert(move _7, "index out of bounds: the length is {} but the index is {}", const 2_usize, copy _6) -> [success: bb2, unwind continue];
+      }
+  
+      bb2: {
+          _5 = copy _1[_6];
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _2;
+          StorageLive(_10);
+          _10 = copy _2;
+          _8 = Mul(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = copy _5;
+          StorageLive(_13);
+          _13 = copy _5;
+          _11 = Mul(move _12, move _13);
+          StorageDead(_13);
+          StorageDead(_12);
+          _0 = Add(move _8, move _11);
+          StorageDead(_11);
+          StorageDead(_8);
+          StorageDead(_5);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-abort.diff b/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-abort.diff
new file mode 100644
index 000000000000..e9cb2c581772
--- /dev/null
+++ b/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-abort.diff
@@ -0,0 +1,71 @@
+- // MIR for `normN` before InstSimplify-after-simplifycfg
++ // MIR for `normN` after InstSimplify-after-simplifycfg
+  
+  fn normN(_1: [f32; N]) -> f32 {
+      debug x => _1;
+      let mut _0: f32;
+      let _2: f32;
+      let _3: usize;
+      let mut _4: bool;
+      let _6: usize;
+      let mut _7: bool;
+      let mut _8: f32;
+      let mut _9: f32;
+      let mut _10: f32;
+      let mut _11: f32;
+      let mut _12: f32;
+      let mut _13: f32;
+      scope 1 {
+          debug a => _2;
+          let _5: f32;
+          scope 2 {
+              debug b => _5;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const 0_usize;
+          _4 = Lt(copy _3, const N);
+          assert(move _4, "index out of bounds: the length is {} but the index is {}", const N, copy _3) -> [success: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _2 = copy _1[_3];
+          StorageDead(_3);
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = const 1_usize;
+          _7 = Lt(copy _6, const N);
+          assert(move _7, "index out of bounds: the length is {} but the index is {}", const N, copy _6) -> [success: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          _5 = copy _1[_6];
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _2;
+          StorageLive(_10);
+          _10 = copy _2;
+          _8 = Mul(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = copy _5;
+          StorageLive(_13);
+          _13 = copy _5;
+          _11 = Mul(move _12, move _13);
+          StorageDead(_13);
+          StorageDead(_12);
+          _0 = Add(move _8, move _11);
+          StorageDead(_11);
+          StorageDead(_8);
+          StorageDead(_5);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-unwind.diff b/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-unwind.diff
new file mode 100644
index 000000000000..0ddc70f2003e
--- /dev/null
+++ b/tests/mir-opt/instsimplify/combine_array_len.normN.InstSimplify-after-simplifycfg.panic-unwind.diff
@@ -0,0 +1,71 @@
+- // MIR for `normN` before InstSimplify-after-simplifycfg
++ // MIR for `normN` after InstSimplify-after-simplifycfg
+  
+  fn normN(_1: [f32; N]) -> f32 {
+      debug x => _1;
+      let mut _0: f32;
+      let _2: f32;
+      let _3: usize;
+      let mut _4: bool;
+      let _6: usize;
+      let mut _7: bool;
+      let mut _8: f32;
+      let mut _9: f32;
+      let mut _10: f32;
+      let mut _11: f32;
+      let mut _12: f32;
+      let mut _13: f32;
+      scope 1 {
+          debug a => _2;
+          let _5: f32;
+          scope 2 {
+              debug b => _5;
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = const 0_usize;
+          _4 = Lt(copy _3, const N);
+          assert(move _4, "index out of bounds: the length is {} but the index is {}", const N, copy _3) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+          _2 = copy _1[_3];
+          StorageDead(_3);
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = const 1_usize;
+          _7 = Lt(copy _6, const N);
+          assert(move _7, "index out of bounds: the length is {} but the index is {}", const N, copy _6) -> [success: bb2, unwind continue];
+      }
+  
+      bb2: {
+          _5 = copy _1[_6];
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = copy _2;
+          StorageLive(_10);
+          _10 = copy _2;
+          _8 = Mul(move _9, move _10);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = copy _5;
+          StorageLive(_13);
+          _13 = copy _5;
+          _11 = Mul(move _12, move _13);
+          StorageDead(_13);
+          StorageDead(_12);
+          _0 = Add(move _8, move _11);
+          StorageDead(_11);
+          StorageDead(_8);
+          StorageDead(_5);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/instsimplify/combine_array_len.rs b/tests/mir-opt/instsimplify/combine_array_len.rs
new file mode 100644
index 000000000000..1c4d42d3bbe8
--- /dev/null
+++ b/tests/mir-opt/instsimplify/combine_array_len.rs
@@ -0,0 +1,25 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ test-mir-pass: InstSimplify-after-simplifycfg
+
+// EMIT_MIR combine_array_len.norm2.InstSimplify-after-simplifycfg.diff
+fn norm2(x: [f32; 2]) -> f32 {
+    // CHECK-LABEL: fn norm2(
+    // CHECK-NOT: PtrMetadata(
+    let a = x[0];
+    let b = x[1];
+    a * a + b * b
+}
+
+// EMIT_MIR combine_array_len.normN.InstSimplify-after-simplifycfg.diff
+fn normN(x: [f32; N]) -> f32 {
+    // CHECK-LABEL: fn normN(
+    // CHECK-NOT: PtrMetadata(
+    let a = x[0];
+    let b = x[1];
+    a * a + b * b
+}
+
+fn main() {
+    assert_eq!(norm2([3.0, 4.0]), 5.0 * 5.0);
+    assert_eq!(normN([3.0, 4.0]), 5.0 * 5.0);
+}
diff --git a/tests/mir-opt/issue_101973.inner.GVN.panic-abort.diff b/tests/mir-opt/issue_101973.inner.GVN.panic-abort.diff
index ac88fe67bb86..3ea7387a48d3 100644
--- a/tests/mir-opt/issue_101973.inner.GVN.panic-abort.diff
+++ b/tests/mir-opt/issue_101973.inner.GVN.panic-abort.diff
@@ -30,7 +30,6 @@
           StorageLive(_4);
           StorageLive(_5);
           _5 = copy _1;
-          nop;
 -         StorageLive(_14);
 -         _14 = BitAnd(copy _5, const 255_u32);
 -         _4 = BitOr(const 0_u32, move _14);
diff --git a/tests/mir-opt/issue_101973.inner.GVN.panic-unwind.diff b/tests/mir-opt/issue_101973.inner.GVN.panic-unwind.diff
index 96c3cae2d334..832db856b2cf 100644
--- a/tests/mir-opt/issue_101973.inner.GVN.panic-unwind.diff
+++ b/tests/mir-opt/issue_101973.inner.GVN.panic-unwind.diff
@@ -30,7 +30,6 @@
           StorageLive(_4);
           StorageLive(_5);
           _5 = copy _1;
-          nop;
 -         StorageLive(_14);
 -         _14 = BitAnd(copy _5, const 255_u32);
 -         _4 = BitOr(const 0_u32, move _14);
diff --git a/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-abort.mir b/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-abort.mir
index 736a8bbca49c..c46549c5742f 100644
--- a/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-abort.mir
+++ b/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-abort.mir
@@ -3,49 +3,45 @@
 fn test() -> Option> {
     let mut _0: std::option::Option>;
     let mut _1: std::boxed::Box;
-    let mut _2: usize;
-    let mut _3: usize;
-    let mut _4: *mut u8;
-    let mut _5: std::boxed::Box;
-    let mut _6: std::ops::ControlFlow, u32>;
-    let mut _7: std::option::Option;
-    let mut _8: isize;
-    let _9: std::option::Option;
-    let mut _10: !;
-    let mut _11: std::option::Option;
-    let _12: u32;
+    let mut _2: *mut u8;
+    let mut _3: std::boxed::Box;
+    let mut _4: std::ops::ControlFlow, u32>;
+    let mut _5: std::option::Option;
+    let mut _6: isize;
+    let _7: std::option::Option;
+    let mut _8: !;
+    let mut _9: std::option::Option;
+    let _10: u32;
     scope 1 {
-        debug residual => _9;
+        debug residual => _7;
         scope 2 {
         }
     }
     scope 3 {
-        debug val => _12;
+        debug val => _10;
         scope 4 {
         }
     }
 
     bb0: {
         StorageLive(_1);
-        _2 = SizeOf(u32);
-        _3 = AlignOf(u32);
-        _4 = alloc::alloc::exchange_malloc(move _2, move _3) -> [return: bb1, unwind: bb13];
+        _2 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind: bb13];
     }
 
     bb1: {
+        StorageLive(_3);
+        _3 = ShallowInitBox(move _2, u32);
+        StorageLive(_4);
         StorageLive(_5);
-        _5 = ShallowInitBox(move _4, u32);
-        StorageLive(_6);
-        StorageLive(_7);
-        _7 = Option::::None;
-        _6 =  as Try>::branch(move _7) -> [return: bb2, unwind: bb12];
+        _5 = Option::::None;
+        _4 =  as Try>::branch(move _5) -> [return: bb2, unwind: bb12];
     }
 
     bb2: {
-        StorageDead(_7);
-        PlaceMention(_6);
-        _8 = discriminant(_6);
-        switchInt(move _8) -> [0: bb4, 1: bb5, otherwise: bb3];
+        StorageDead(_5);
+        PlaceMention(_4);
+        _6 = discriminant(_4);
+        switchInt(move _6) -> [0: bb4, 1: bb5, otherwise: bb3];
     }
 
     bb3: {
@@ -53,44 +49,44 @@ fn test() -> Option> {
     }
 
     bb4: {
-        StorageLive(_12);
-        _12 = copy ((_6 as Continue).0: u32);
-        (*_5) = copy _12;
-        StorageDead(_12);
-        _1 = move _5;
-        drop(_5) -> [return: bb7, unwind: bb11];
+        StorageLive(_10);
+        _10 = copy ((_4 as Continue).0: u32);
+        (*_3) = copy _10;
+        StorageDead(_10);
+        _1 = move _3;
+        drop(_3) -> [return: bb7, unwind: bb11];
     }
 
     bb5: {
+        StorageLive(_7);
+        _7 = copy ((_4 as Break).0: std::option::Option);
         StorageLive(_9);
-        _9 = copy ((_6 as Break).0: std::option::Option);
-        StorageLive(_11);
-        _11 = copy _9;
-        _0 = > as FromResidual>>::from_residual(move _11) -> [return: bb6, unwind: bb12];
+        _9 = copy _7;
+        _0 = > as FromResidual>>::from_residual(move _9) -> [return: bb6, unwind: bb12];
     }
 
     bb6: {
-        StorageDead(_11);
         StorageDead(_9);
-        drop(_5) -> [return: bb9, unwind: bb13];
+        StorageDead(_7);
+        drop(_3) -> [return: bb9, unwind: bb13];
     }
 
     bb7: {
-        StorageDead(_5);
+        StorageDead(_3);
         _0 = Option::>::Some(move _1);
         drop(_1) -> [return: bb8, unwind: bb13];
     }
 
     bb8: {
         StorageDead(_1);
-        StorageDead(_6);
+        StorageDead(_4);
         goto -> bb10;
     }
 
     bb9: {
-        StorageDead(_5);
+        StorageDead(_3);
         StorageDead(_1);
-        StorageDead(_6);
+        StorageDead(_4);
         goto -> bb10;
     }
 
@@ -103,7 +99,7 @@ fn test() -> Option> {
     }
 
     bb12 (cleanup): {
-        drop(_5) -> [return: bb13, unwind terminate(cleanup)];
+        drop(_3) -> [return: bb13, unwind terminate(cleanup)];
     }
 
     bb13 (cleanup): {
diff --git a/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-unwind.mir b/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-unwind.mir
index 1acb1ae20b69..9f26debdbb6d 100644
--- a/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-unwind.mir
+++ b/tests/mir-opt/issue_62289.test.ElaborateDrops.before.panic-unwind.mir
@@ -3,49 +3,45 @@
 fn test() -> Option> {
     let mut _0: std::option::Option>;
     let mut _1: std::boxed::Box;
-    let mut _2: usize;
-    let mut _3: usize;
-    let mut _4: *mut u8;
-    let mut _5: std::boxed::Box;
-    let mut _6: std::ops::ControlFlow, u32>;
-    let mut _7: std::option::Option;
-    let mut _8: isize;
-    let _9: std::option::Option;
-    let mut _10: !;
-    let mut _11: std::option::Option;
-    let _12: u32;
+    let mut _2: *mut u8;
+    let mut _3: std::boxed::Box;
+    let mut _4: std::ops::ControlFlow, u32>;
+    let mut _5: std::option::Option;
+    let mut _6: isize;
+    let _7: std::option::Option;
+    let mut _8: !;
+    let mut _9: std::option::Option;
+    let _10: u32;
     scope 1 {
-        debug residual => _9;
+        debug residual => _7;
         scope 2 {
         }
     }
     scope 3 {
-        debug val => _12;
+        debug val => _10;
         scope 4 {
         }
     }
 
     bb0: {
         StorageLive(_1);
-        _2 = SizeOf(u32);
-        _3 = AlignOf(u32);
-        _4 = alloc::alloc::exchange_malloc(move _2, move _3) -> [return: bb1, unwind continue];
+        _2 = alloc::alloc::exchange_malloc(const ::SIZE, const ::ALIGN) -> [return: bb1, unwind continue];
     }
 
     bb1: {
+        StorageLive(_3);
+        _3 = ShallowInitBox(move _2, u32);
+        StorageLive(_4);
         StorageLive(_5);
-        _5 = ShallowInitBox(move _4, u32);
-        StorageLive(_6);
-        StorageLive(_7);
-        _7 = Option::::None;
-        _6 =  as Try>::branch(move _7) -> [return: bb2, unwind: bb12];
+        _5 = Option::::None;
+        _4 =  as Try>::branch(move _5) -> [return: bb2, unwind: bb12];
     }
 
     bb2: {
-        StorageDead(_7);
-        PlaceMention(_6);
-        _8 = discriminant(_6);
-        switchInt(move _8) -> [0: bb4, 1: bb5, otherwise: bb3];
+        StorageDead(_5);
+        PlaceMention(_4);
+        _6 = discriminant(_4);
+        switchInt(move _6) -> [0: bb4, 1: bb5, otherwise: bb3];
     }
 
     bb3: {
@@ -53,44 +49,44 @@ fn test() -> Option> {
     }
 
     bb4: {
-        StorageLive(_12);
-        _12 = copy ((_6 as Continue).0: u32);
-        (*_5) = copy _12;
-        StorageDead(_12);
-        _1 = move _5;
-        drop(_5) -> [return: bb7, unwind: bb11];
+        StorageLive(_10);
+        _10 = copy ((_4 as Continue).0: u32);
+        (*_3) = copy _10;
+        StorageDead(_10);
+        _1 = move _3;
+        drop(_3) -> [return: bb7, unwind: bb11];
     }
 
     bb5: {
+        StorageLive(_7);
+        _7 = copy ((_4 as Break).0: std::option::Option);
         StorageLive(_9);
-        _9 = copy ((_6 as Break).0: std::option::Option);
-        StorageLive(_11);
-        _11 = copy _9;
-        _0 = > as FromResidual>>::from_residual(move _11) -> [return: bb6, unwind: bb12];
+        _9 = copy _7;
+        _0 = > as FromResidual>>::from_residual(move _9) -> [return: bb6, unwind: bb12];
     }
 
     bb6: {
-        StorageDead(_11);
         StorageDead(_9);
-        drop(_5) -> [return: bb9, unwind continue];
+        StorageDead(_7);
+        drop(_3) -> [return: bb9, unwind continue];
     }
 
     bb7: {
-        StorageDead(_5);
+        StorageDead(_3);
         _0 = Option::>::Some(move _1);
         drop(_1) -> [return: bb8, unwind continue];
     }
 
     bb8: {
         StorageDead(_1);
-        StorageDead(_6);
+        StorageDead(_4);
         goto -> bb10;
     }
 
     bb9: {
-        StorageDead(_5);
+        StorageDead(_3);
         StorageDead(_1);
-        StorageDead(_6);
+        StorageDead(_4);
         goto -> bb10;
     }
 
@@ -103,7 +99,7 @@ fn test() -> Option> {
     }
 
     bb12 (cleanup): {
-        drop(_5) -> [return: bb13, unwind terminate(cleanup)];
+        drop(_3) -> [return: bb13, unwind terminate(cleanup)];
     }
 
     bb13 (cleanup): {
diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
index 08dee3697e07..614d9ad440d2 100644
--- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
+++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
@@ -7,18 +7,16 @@
       let _2: &[T];
       let _3: &[T; 3];
       let _4: [T; 3];
-      let mut _5: usize;
-      let mut _6: bool;
-      let mut _10: !;
+      let mut _8: !;
       scope 1 {
           debug v => _2;
+          let _5: &T;
+          let _6: &T;
           let _7: &T;
-          let _8: &T;
-          let _9: &T;
           scope 2 {
-              debug v1 => _7;
-              debug v2 => _8;
-              debug v3 => _9;
+              debug v1 => _5;
+              debug v2 => _6;
+              debug v3 => _7;
           }
       }
   
@@ -27,25 +25,17 @@
           _4 = [copy _1, copy _1, copy _1];
           _3 = &_4;
           _2 = copy _3 as &[T] (PointerCoercion(Unsize, Implicit));
-          nop;
-          nop;
           goto -> bb2;
       }
   
       bb1: {
-          _10 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind unreachable;
+          _8 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind unreachable;
       }
   
       bb2: {
-          StorageLive(_7);
-          _7 = &(*_2)[0 of 3];
-          StorageLive(_8);
-          _8 = &(*_2)[1 of 3];
-          StorageLive(_9);
-          _9 = &(*_2)[2 of 3];
-          StorageDead(_9);
-          StorageDead(_8);
-          StorageDead(_7);
+          // DBG: _5 = &(*_2)[0 of 3];
+          // DBG: _6 = &(*_2)[1 of 3];
+          // DBG: _7 = &(*_2)[2 of 3];
           StorageDead(_4);
           return;
       }
diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
index aa44a2ad532b..57a88cf89841 100644
--- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
+++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
@@ -7,18 +7,16 @@
       let _2: &[T];
       let _3: &[T; 3];
       let _4: [T; 3];
-      let mut _5: usize;
-      let mut _6: bool;
-      let mut _10: !;
+      let mut _8: !;
       scope 1 {
           debug v => _2;
+          let _5: &T;
+          let _6: &T;
           let _7: &T;
-          let _8: &T;
-          let _9: &T;
           scope 2 {
-              debug v1 => _7;
-              debug v2 => _8;
-              debug v3 => _9;
+              debug v1 => _5;
+              debug v2 => _6;
+              debug v3 => _7;
           }
       }
   
@@ -27,25 +25,17 @@
           _4 = [copy _1, copy _1, copy _1];
           _3 = &_4;
           _2 = copy _3 as &[T] (PointerCoercion(Unsize, Implicit));
-          nop;
-          nop;
           goto -> bb2;
       }
   
       bb1: {
-          _10 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind continue;
+          _8 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind continue;
       }
   
       bb2: {
-          StorageLive(_7);
-          _7 = &(*_2)[0 of 3];
-          StorageLive(_8);
-          _8 = &(*_2)[1 of 3];
-          StorageLive(_9);
-          _9 = &(*_2)[2 of 3];
-          StorageDead(_9);
-          StorageDead(_8);
-          StorageDead(_7);
+          // DBG: _5 = &(*_2)[0 of 3];
+          // DBG: _6 = &(*_2)[1 of 3];
+          // DBG: _7 = &(*_2)[2 of 3];
           StorageDead(_4);
           return;
       }
diff --git a/tests/mir-opt/issue_91633.fun.built.after.mir b/tests/mir-opt/issue_91633.fun.built.after.mir
index d2fc438d3e81..7468e591b37b 100644
--- a/tests/mir-opt/issue_91633.fun.built.after.mir
+++ b/tests/mir-opt/issue_91633.fun.built.after.mir
@@ -23,8 +23,8 @@ fn fun(_1: &[T]) -> &T {
     bb1: {
         _2 = &(*_1)[_3];
         FakeRead(ForLet(None), _2);
-        _0 = &(*_2);
         StorageDead(_3);
+        _0 = &(*_2);
         StorageDead(_2);
         return;
     }
diff --git a/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-abort.diff b/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-abort.diff
index 180a7db02979..714a12804e69 100644
--- a/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-abort.diff
+++ b/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-abort.diff
@@ -18,8 +18,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
           _4 = &_1;
@@ -41,8 +40,7 @@
       bb1: {
           StorageDead(_6);
           StorageDead(_5);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           StorageDead(_7);
           return;
       }
diff --git a/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-unwind.diff b/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-unwind.diff
index 180a7db02979..714a12804e69 100644
--- a/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-unwind.diff
+++ b/tests/mir-opt/lower_array_len.array_len_raw.GVN.panic-unwind.diff
@@ -18,8 +18,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
           _4 = &_1;
@@ -41,8 +40,7 @@
       bb1: {
           StorageDead(_6);
           StorageDead(_5);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           StorageDead(_7);
           return;
       }
diff --git a/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-abort.diff b/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-abort.diff
index 49964f8b49e2..b236b22f067d 100644
--- a/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-abort.diff
+++ b/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-abort.diff
@@ -17,8 +17,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
           _4 = &mut _1;
@@ -38,8 +37,7 @@
       bb1: {
           StorageDead(_6);
           StorageDead(_5);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           return;
       }
   }
diff --git a/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-unwind.diff b/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-unwind.diff
index 49964f8b49e2..b236b22f067d 100644
--- a/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-unwind.diff
+++ b/tests/mir-opt/lower_array_len.array_len_reborrow.GVN.panic-unwind.diff
@@ -17,8 +17,7 @@
       }
   
       bb0: {
--         StorageLive(_2);
-+         nop;
+          StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
           _4 = &mut _1;
@@ -38,8 +37,7 @@
       bb1: {
           StorageDead(_6);
           StorageDead(_5);
--         StorageDead(_2);
-+         nop;
+          StorageDead(_2);
           return;
       }
   }
diff --git a/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-abort.diff
deleted file mode 100644
index e61a3e8d7383..000000000000
--- a/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-abort.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-- // MIR for `align_of` before LowerIntrinsics
-+ // MIR for `align_of` after LowerIntrinsics
-  
-  fn align_of() -> usize {
-      let mut _0: usize;
-  
-      bb0: {
--         _0 = std::intrinsics::align_of::() -> [return: bb1, unwind unreachable];
-+         _0 = AlignOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-unwind.diff
deleted file mode 100644
index e61a3e8d7383..000000000000
--- a/tests/mir-opt/lower_intrinsics.align_of.LowerIntrinsics.panic-unwind.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-- // MIR for `align_of` before LowerIntrinsics
-+ // MIR for `align_of` after LowerIntrinsics
-  
-  fn align_of() -> usize {
-      let mut _0: usize;
-  
-      bb0: {
--         _0 = std::intrinsics::align_of::() -> [return: bb1, unwind unreachable];
-+         _0 = AlignOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-abort.diff
index 5aa986984768..42b8ab3da66c 100644
--- a/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-abort.diff
+++ b/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-abort.diff
@@ -3,26 +3,21 @@
   
   fn non_const() -> usize {
       let mut _0: usize;
-      let _1: fn() -> usize {std::intrinsics::size_of::};
-      let mut _2: fn() -> usize {std::intrinsics::size_of::};
+      let _1: unsafe fn() -> ! {std::intrinsics::unreachable};
+      let mut _2: !;
+      let mut _3: unsafe fn() -> ! {std::intrinsics::unreachable};
       scope 1 {
-          debug size_of_t => _1;
+          debug unreachable => _1;
       }
   
       bb0: {
           StorageLive(_1);
-          _1 = std::intrinsics::size_of::;
+          _1 = std::intrinsics::unreachable;
           StorageLive(_2);
-          _2 = copy _1;
--         _0 = move _2() -> [return: bb1, unwind unreachable];
-+         _0 = SizeOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          StorageDead(_2);
-          StorageDead(_1);
-          return;
+          StorageLive(_3);
+          _3 = copy _1;
+-         _2 = move _3() -> unwind unreachable;
++         unreachable;
       }
   }
   
diff --git a/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-unwind.diff
index 5aa986984768..42b8ab3da66c 100644
--- a/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-unwind.diff
+++ b/tests/mir-opt/lower_intrinsics.non_const.LowerIntrinsics.panic-unwind.diff
@@ -3,26 +3,21 @@
   
   fn non_const() -> usize {
       let mut _0: usize;
-      let _1: fn() -> usize {std::intrinsics::size_of::};
-      let mut _2: fn() -> usize {std::intrinsics::size_of::};
+      let _1: unsafe fn() -> ! {std::intrinsics::unreachable};
+      let mut _2: !;
+      let mut _3: unsafe fn() -> ! {std::intrinsics::unreachable};
       scope 1 {
-          debug size_of_t => _1;
+          debug unreachable => _1;
       }
   
       bb0: {
           StorageLive(_1);
-          _1 = std::intrinsics::size_of::;
+          _1 = std::intrinsics::unreachable;
           StorageLive(_2);
-          _2 = copy _1;
--         _0 = move _2() -> [return: bb1, unwind unreachable];
-+         _0 = SizeOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          StorageDead(_2);
-          StorageDead(_1);
-          return;
+          StorageLive(_3);
+          _3 = copy _1;
+-         _2 = move _3() -> unwind unreachable;
++         unreachable;
       }
   }
   
diff --git a/tests/mir-opt/lower_intrinsics.rs b/tests/mir-opt/lower_intrinsics.rs
index 7729e502ad6b..6b8c2a790202 100644
--- a/tests/mir-opt/lower_intrinsics.rs
+++ b/tests/mir-opt/lower_intrinsics.rs
@@ -40,20 +40,6 @@ pub unsafe fn unchecked(a: i32, b: i32, c: u32) {
     let _l = core::intrinsics::unchecked_shr(a, c);
 }
 
-// EMIT_MIR lower_intrinsics.size_of.LowerIntrinsics.diff
-pub fn size_of() -> usize {
-    // CHECK-LABEL: fn size_of(
-    // CHECK: {{_.*}} = SizeOf(T);
-    core::intrinsics::size_of::()
-}
-
-// EMIT_MIR lower_intrinsics.align_of.LowerIntrinsics.diff
-pub fn align_of() -> usize {
-    // CHECK-LABEL: fn align_of(
-    // CHECK: {{_.*}} = AlignOf(T);
-    core::intrinsics::align_of::()
-}
-
 // EMIT_MIR lower_intrinsics.forget.LowerIntrinsics.diff
 pub fn forget(t: T) {
     // CHECK-LABEL: fn forget(
@@ -73,11 +59,13 @@ pub fn unreachable() -> ! {
 // EMIT_MIR lower_intrinsics.non_const.LowerIntrinsics.diff
 pub fn non_const() -> usize {
     // CHECK-LABEL: fn non_const(
-    // CHECK: SizeOf(T);
+    // CHECK: _1 = std::intrinsics::unreachable;
+    // CHECK: unreachable;
+    // CHECK-NEXT: }
 
     // Check that lowering works with non-const operand as a func.
-    let size_of_t = core::intrinsics::size_of::;
-    size_of_t()
+    let unreachable = core::intrinsics::unreachable;
+    unsafe { unreachable() }
 }
 
 // EMIT_MIR lower_intrinsics.transmute_inhabited.LowerIntrinsics.diff
diff --git a/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-abort.diff
deleted file mode 100644
index a547bdf3737b..000000000000
--- a/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-abort.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-- // MIR for `size_of` before LowerIntrinsics
-+ // MIR for `size_of` after LowerIntrinsics
-  
-  fn size_of() -> usize {
-      let mut _0: usize;
-  
-      bb0: {
--         _0 = std::intrinsics::size_of::() -> [return: bb1, unwind unreachable];
-+         _0 = SizeOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-unwind.diff
deleted file mode 100644
index a547bdf3737b..000000000000
--- a/tests/mir-opt/lower_intrinsics.size_of.LowerIntrinsics.panic-unwind.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-- // MIR for `size_of` before LowerIntrinsics
-+ // MIR for `size_of` after LowerIntrinsics
-  
-  fn size_of() -> usize {
-      let mut _0: usize;
-  
-      bb0: {
--         _0 = std::intrinsics::size_of::() -> [return: bb1, unwind unreachable];
-+         _0 = SizeOf(T);
-+         goto -> bb1;
-      }
-  
-      bb1: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff b/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
index b3eb3e1f8b9d..484bc7dad128 100644
--- a/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
+++ b/tests/mir-opt/match_arm_scopes.complicated_match.panic-abort.SimplifyCfg-initial.after-ElaborateDrops.after.diff
@@ -85,11 +85,11 @@
           _8 = &(_2.2: std::string::String);
 -         _3 = &fake shallow (_2.0: bool);
 -         _4 = &fake shallow (_2.1: bool);
-          StorageLive(_12);
-          StorageLive(_13);
-          _13 = copy _1;
--         switchInt(move _13) -> [0: bb16, otherwise: bb15];
-+         switchInt(move _13) -> [0: bb13, otherwise: bb12];
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = copy _1;
+-         switchInt(move _10) -> [0: bb12, otherwise: bb11];
++         switchInt(move _10) -> [0: bb9, otherwise: bb8];
       }
   
 -     bb9: {
@@ -100,11 +100,11 @@
           _8 = &(_2.2: std::string::String);
 -         _3 = &fake shallow (_2.0: bool);
 -         _4 = &fake shallow (_2.1: bool);
-          StorageLive(_9);
-          StorageLive(_10);
-          _10 = copy _1;
--         switchInt(move _10) -> [0: bb12, otherwise: bb11];
-+         switchInt(move _10) -> [0: bb9, otherwise: bb8];
+          StorageLive(_12);
+          StorageLive(_13);
+          _13 = copy _1;
+-         switchInt(move _13) -> [0: bb16, otherwise: bb15];
++         switchInt(move _13) -> [0: bb13, otherwise: bb12];
       }
   
 -     bb10: {
@@ -139,7 +139,7 @@
 -         FakeRead(ForGuardBinding, _6);
 -         FakeRead(ForGuardBinding, _8);
           StorageLive(_5);
-          _5 = copy (_2.1: bool);
+          _5 = copy (_2.0: bool);
           StorageLive(_7);
           _7 = move (_2.2: std::string::String);
 -         goto -> bb10;
@@ -152,8 +152,8 @@
           StorageDead(_9);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb1, imaginary: bb1];
-+         goto -> bb1;
+-         falseEdge -> [real: bb3, imaginary: bb3];
++         goto -> bb2;
       }
   
 -     bb15: {
@@ -181,7 +181,7 @@
 -         FakeRead(ForGuardBinding, _6);
 -         FakeRead(ForGuardBinding, _8);
           StorageLive(_5);
-          _5 = copy (_2.0: bool);
+          _5 = copy (_2.1: bool);
           StorageLive(_7);
           _7 = move (_2.2: std::string::String);
 -         goto -> bb10;
@@ -194,8 +194,8 @@
           StorageDead(_12);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb3, imaginary: bb3];
-+         goto -> bb2;
+-         falseEdge -> [real: bb1, imaginary: bb1];
++         goto -> bb1;
       }
   
 -     bb19: {
diff --git a/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff b/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
index b3eb3e1f8b9d..484bc7dad128 100644
--- a/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
+++ b/tests/mir-opt/match_arm_scopes.complicated_match.panic-unwind.SimplifyCfg-initial.after-ElaborateDrops.after.diff
@@ -85,11 +85,11 @@
           _8 = &(_2.2: std::string::String);
 -         _3 = &fake shallow (_2.0: bool);
 -         _4 = &fake shallow (_2.1: bool);
-          StorageLive(_12);
-          StorageLive(_13);
-          _13 = copy _1;
--         switchInt(move _13) -> [0: bb16, otherwise: bb15];
-+         switchInt(move _13) -> [0: bb13, otherwise: bb12];
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = copy _1;
+-         switchInt(move _10) -> [0: bb12, otherwise: bb11];
++         switchInt(move _10) -> [0: bb9, otherwise: bb8];
       }
   
 -     bb9: {
@@ -100,11 +100,11 @@
           _8 = &(_2.2: std::string::String);
 -         _3 = &fake shallow (_2.0: bool);
 -         _4 = &fake shallow (_2.1: bool);
-          StorageLive(_9);
-          StorageLive(_10);
-          _10 = copy _1;
--         switchInt(move _10) -> [0: bb12, otherwise: bb11];
-+         switchInt(move _10) -> [0: bb9, otherwise: bb8];
+          StorageLive(_12);
+          StorageLive(_13);
+          _13 = copy _1;
+-         switchInt(move _13) -> [0: bb16, otherwise: bb15];
++         switchInt(move _13) -> [0: bb13, otherwise: bb12];
       }
   
 -     bb10: {
@@ -139,7 +139,7 @@
 -         FakeRead(ForGuardBinding, _6);
 -         FakeRead(ForGuardBinding, _8);
           StorageLive(_5);
-          _5 = copy (_2.1: bool);
+          _5 = copy (_2.0: bool);
           StorageLive(_7);
           _7 = move (_2.2: std::string::String);
 -         goto -> bb10;
@@ -152,8 +152,8 @@
           StorageDead(_9);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb1, imaginary: bb1];
-+         goto -> bb1;
+-         falseEdge -> [real: bb3, imaginary: bb3];
++         goto -> bb2;
       }
   
 -     bb15: {
@@ -181,7 +181,7 @@
 -         FakeRead(ForGuardBinding, _6);
 -         FakeRead(ForGuardBinding, _8);
           StorageLive(_5);
-          _5 = copy (_2.0: bool);
+          _5 = copy (_2.1: bool);
           StorageLive(_7);
           _7 = move (_2.2: std::string::String);
 -         goto -> bb10;
@@ -194,8 +194,8 @@
           StorageDead(_12);
           StorageDead(_8);
           StorageDead(_6);
--         falseEdge -> [real: bb3, imaginary: bb3];
-+         goto -> bb2;
+-         falseEdge -> [real: bb1, imaginary: bb1];
++         goto -> bb1;
       }
   
 -     bb19: {
diff --git a/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.32bit.mir b/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.32bit.mir
index 42b388033360..c069ce3fdf3c 100644
--- a/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.32bit.mir
+++ b/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.32bit.mir
@@ -5,20 +5,20 @@
 | '?1 | Local | ['?1]
 |
 | Inferred Region Values
-| '?0 | U0 | {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0], '?0, '?1}
-| '?1 | U0 | {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0], '?1}
-| '?2 | U0 | {bb1[0..=7], bb2[0..=2]}
-| '?3 | U0 | {bb1[1..=7], bb2[0..=2]}
-| '?4 | U0 | {bb1[4..=7], bb2[0..=2]}
+| '?0 | U0 | {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0], '?0, '?1}
+| '?1 | U0 | {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0], '?1}
+| '?2 | U0 | {bb1[0..=8], bb2[0..=2]}
+| '?3 | U0 | {bb1[1..=8], bb2[0..=2]}
+| '?4 | U0 | {bb1[5..=8], bb2[0..=2]}
 |
 | Inference Constraints
-| '?0 live at {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0]}
-| '?1 live at {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0]}
+| '?0 live at {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0]}
+| '?1 live at {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0]}
 | '?2 live at {bb1[0]}
-| '?3 live at {bb1[1..=3]}
-| '?4 live at {bb1[4..=7], bb2[0..=2]}
+| '?3 live at {bb1[1..=4]}
+| '?4 live at {bb1[5..=8], bb2[0..=2]}
 | '?2: '?3 due to Assignment at Single(bb1[0]) ($DIR/region_subtyping_basic.rs:19:13: 19:18 (#0)
-| '?3: '?4 due to Assignment at Single(bb1[3]) ($DIR/region_subtyping_basic.rs:20:13: 20:14 (#0)
+| '?3: '?4 due to Assignment at Single(bb1[4]) ($DIR/region_subtyping_basic.rs:20:13: 20:14 (#0)
 |
 | Borrows
 | bw0: issued at bb1[0] in '?2
@@ -59,6 +59,7 @@ fn main() -> () {
     bb1: {
         _2 = &'?2 _1[_3];
         FakeRead(ForLet(None), _2);
+        StorageDead(_3);
         StorageLive(_5);
         _5 = copy _2;
         FakeRead(ForLet(None), _5);
@@ -95,7 +96,6 @@ fn main() -> () {
     bb6: {
         StorageDead(_6);
         StorageDead(_5);
-        StorageDead(_3);
         StorageDead(_2);
         StorageDead(_1);
         return;
diff --git a/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.64bit.mir b/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.64bit.mir
index 15395fd470e9..a866521e7c97 100644
--- a/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.64bit.mir
+++ b/tests/mir-opt/nll/region_subtyping_basic.main.nll.0.64bit.mir
@@ -5,20 +5,20 @@
 | '?1 | Local | ['?1]
 |
 | Inferred Region Values
-| '?0 | U0 | {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0], '?0, '?1}
-| '?1 | U0 | {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0], '?1}
-| '?2 | U0 | {bb1[0..=7], bb2[0..=2]}
-| '?3 | U0 | {bb1[1..=7], bb2[0..=2]}
-| '?4 | U0 | {bb1[4..=7], bb2[0..=2]}
+| '?0 | U0 | {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0], '?0, '?1}
+| '?1 | U0 | {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0], '?1}
+| '?2 | U0 | {bb1[0..=8], bb2[0..=2]}
+| '?3 | U0 | {bb1[1..=8], bb2[0..=2]}
+| '?4 | U0 | {bb1[5..=8], bb2[0..=2]}
 |
 | Inference Constraints
-| '?0 live at {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0]}
-| '?1 live at {bb0[0..=8], bb1[0..=7], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=5], bb7[0]}
+| '?0 live at {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0]}
+| '?1 live at {bb0[0..=8], bb1[0..=8], bb2[0..=3], bb3[0..=3], bb4[0..=1], bb5[0..=2], bb6[0..=4], bb7[0]}
 | '?2 live at {bb1[0]}
-| '?3 live at {bb1[1..=3]}
-| '?4 live at {bb1[4..=7], bb2[0..=2]}
+| '?3 live at {bb1[1..=4]}
+| '?4 live at {bb1[5..=8], bb2[0..=2]}
 | '?2: '?3 due to Assignment at Single(bb1[0]) ($DIR/region_subtyping_basic.rs:19:13: 19:18 (#0)
-| '?3: '?4 due to Assignment at Single(bb1[3]) ($DIR/region_subtyping_basic.rs:20:13: 20:14 (#0)
+| '?3: '?4 due to Assignment at Single(bb1[4]) ($DIR/region_subtyping_basic.rs:20:13: 20:14 (#0)
 |
 | Borrows
 | bw0: issued at bb1[0] in '?2
@@ -59,6 +59,7 @@ fn main() -> () {
     bb1: {
         _2 = &'?2 _1[_3];
         FakeRead(ForLet(None), _2);
+        StorageDead(_3);
         StorageLive(_5);
         _5 = copy _2;
         FakeRead(ForLet(None), _5);
@@ -95,7 +96,6 @@ fn main() -> () {
     bb6: {
         StorageDead(_6);
         StorageDead(_5);
-        StorageDead(_3);
         StorageDead(_2);
         StorageDead(_1);
         return;
diff --git a/tests/mir-opt/nrvo_miscompile_111005.rs b/tests/mir-opt/nrvo_miscompile_111005.rs
deleted file mode 100644
index 03008fa81915..000000000000
--- a/tests/mir-opt/nrvo_miscompile_111005.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// skip-filecheck
-// This is a miscompilation, #111005 to track
-
-//@ test-mir-pass: RenameReturnPlace
-
-#![feature(custom_mir, core_intrinsics)]
-extern crate core;
-use core::intrinsics::mir::*;
-
-// EMIT_MIR nrvo_miscompile_111005.wrong.RenameReturnPlace.diff
-#[custom_mir(dialect = "runtime", phase = "initial")]
-pub fn wrong(arg: char) -> char {
-    mir! {
-        {
-            let temp = arg;
-            RET = temp;
-            temp = 'b';
-            Return()
-        }
-    }
-}
-
-fn main() {
-    assert_eq!(wrong('a'), 'a');
-}
diff --git a/tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff b/tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff
deleted file mode 100644
index d248c76f261b..000000000000
--- a/tests/mir-opt/nrvo_miscompile_111005.wrong.RenameReturnPlace.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-- // MIR for `wrong` before RenameReturnPlace
-+ // MIR for `wrong` after RenameReturnPlace
-  
-  fn wrong(_1: char) -> char {
-      let mut _0: char;
-      let mut _2: char;
-  
-      bb0: {
--         _2 = copy _1;
--         _0 = copy _2;
--         _2 = const 'b';
-+         _0 = copy _1;
-+         _0 = const 'b';
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff b/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff
deleted file mode 100644
index 6dce3ec5303d..000000000000
--- a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-abort.diff
+++ /dev/null
@@ -1,42 +0,0 @@
-- // MIR for `nrvo` before RenameReturnPlace
-+ // MIR for `nrvo` after RenameReturnPlace
-  
-  fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] {
-      debug init => _1;
-      let mut _0: [u8; 1024];
-      let mut _2: [u8; 1024];
-      let _3: ();
-      let mut _4: for<'a> fn(&'a mut [u8; 1024]);
-      let mut _5: &mut [u8; 1024];
-      let mut _6: &mut [u8; 1024];
-      scope 1 {
--         debug buf => _2;
-+         debug buf => _0;
-      }
-  
-      bb0: {
--         StorageLive(_2);
--         _2 = [const 0_u8; 1024];
-+         _0 = [const 0_u8; 1024];
-          StorageLive(_3);
-          StorageLive(_4);
-          _4 = copy _1;
-          StorageLive(_5);
-          StorageLive(_6);
--         _6 = &mut _2;
-+         _6 = &mut _0;
-          _5 = &mut (*_6);
-          _3 = move _4(move _5) -> [return: bb1, unwind unreachable];
-      }
-  
-      bb1: {
-          StorageDead(_5);
-          StorageDead(_4);
-          StorageDead(_6);
-          StorageDead(_3);
--         _0 = copy _2;
--         StorageDead(_2);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff b/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff
deleted file mode 100644
index 54cbe2871f15..000000000000
--- a/tests/mir-opt/nrvo_simple.nrvo.RenameReturnPlace.panic-unwind.diff
+++ /dev/null
@@ -1,42 +0,0 @@
-- // MIR for `nrvo` before RenameReturnPlace
-+ // MIR for `nrvo` after RenameReturnPlace
-  
-  fn nrvo(_1: for<'a> fn(&'a mut [u8; 1024])) -> [u8; 1024] {
-      debug init => _1;
-      let mut _0: [u8; 1024];
-      let mut _2: [u8; 1024];
-      let _3: ();
-      let mut _4: for<'a> fn(&'a mut [u8; 1024]);
-      let mut _5: &mut [u8; 1024];
-      let mut _6: &mut [u8; 1024];
-      scope 1 {
--         debug buf => _2;
-+         debug buf => _0;
-      }
-  
-      bb0: {
--         StorageLive(_2);
--         _2 = [const 0_u8; 1024];
-+         _0 = [const 0_u8; 1024];
-          StorageLive(_3);
-          StorageLive(_4);
-          _4 = copy _1;
-          StorageLive(_5);
-          StorageLive(_6);
--         _6 = &mut _2;
-+         _6 = &mut _0;
-          _5 = &mut (*_6);
-          _3 = move _4(move _5) -> [return: bb1, unwind continue];
-      }
-  
-      bb1: {
-          StorageDead(_5);
-          StorageDead(_4);
-          StorageDead(_6);
-          StorageDead(_3);
--         _0 = copy _2;
--         StorageDead(_2);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/nrvo_simple.rs b/tests/mir-opt/nrvo_simple.rs
deleted file mode 100644
index df540472e1cc..000000000000
--- a/tests/mir-opt/nrvo_simple.rs
+++ /dev/null
@@ -1,16 +0,0 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
-//@ test-mir-pass: RenameReturnPlace
-
-// EMIT_MIR nrvo_simple.nrvo.RenameReturnPlace.diff
-fn nrvo(init: fn(&mut [u8; 1024])) -> [u8; 1024] {
-    let mut buf = [0; 1024];
-    init(&mut buf);
-    buf
-}
-
-fn main() {
-    let _ = nrvo(|buf| {
-        buf[4] = 4;
-    });
-}
diff --git a/tests/mir-opt/pre-codegen/checked_ops.rs b/tests/mir-opt/pre-codegen/checked_ops.rs
index 8fd340503f54..dfbc0f4a1108 100644
--- a/tests/mir-opt/pre-codegen/checked_ops.rs
+++ b/tests/mir-opt/pre-codegen/checked_ops.rs
@@ -44,8 +44,7 @@ pub fn saturating_sub_at_home(lhs: u32, rhs: u32) -> u32 {
     // CHECK-LABEL: fn saturating_sub_at_home
     // CHECK: [[DELTA:_[0-9]+]] = SubUnchecked(copy _1, copy _2)
     // CHECK: [[TEMP1:_.+]] = Option::::Some({{move|copy}} [[DELTA]]);
-    // CHECK: [[TEMP2:_.+]] = {{move|copy}} (([[TEMP1]] as Some).0: u32);
-    // CHECK: _0 = {{move|copy}} [[TEMP2]];
+    // CHECK: _0 = {{move|copy}} (([[TEMP1]] as Some).0: u32);
     u32::checked_sub(lhs, rhs).unwrap_or(0)
 }
 
diff --git a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir
index 5b4fdeda8573..dc7567b57e70 100644
--- a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-abort.mir
@@ -10,7 +10,6 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 {
         let mut _4: u32;
     }
     scope 2 (inlined Option::::unwrap_or) {
-        let _6: u32;
         scope 3 {
         }
     }
@@ -28,10 +27,7 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 {
         _5 = Option::::Some(move _4);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_6);
-        _6 = move ((_5 as Some).0: u32);
-        _0 = move _6;
-        StorageDead(_6);
+        _0 = move ((_5 as Some).0: u32);
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir
index 5b4fdeda8573..dc7567b57e70 100644
--- a/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/checked_ops.saturating_sub_at_home.PreCodegen.after.panic-unwind.mir
@@ -10,7 +10,6 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 {
         let mut _4: u32;
     }
     scope 2 (inlined Option::::unwrap_or) {
-        let _6: u32;
         scope 3 {
         }
     }
@@ -28,10 +27,7 @@ fn saturating_sub_at_home(_1: u32, _2: u32) -> u32 {
         _5 = Option::::Some(move _4);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_6);
-        _6 = move ((_5 as Some).0: u32);
-        _0 = move _6;
-        StorageDead(_6);
+        _0 = move ((_5 as Some).0: u32);
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/pre-codegen/clone_as_copy.clone_as_copy.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/clone_as_copy.clone_as_copy.PreCodegen.after.mir
index 34747e5a9285..1e6e2ee1b8b7 100644
--- a/tests/mir-opt/pre-codegen/clone_as_copy.clone_as_copy.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/clone_as_copy.clone_as_copy.PreCodegen.after.mir
@@ -12,10 +12,8 @@ fn clone_as_copy(_1: &NestCopy) -> NestCopy {
     }
 
     bb0: {
-        StorageLive(_2);
-        _2 = &((*_1).1: AllCopy);
+        // DBG: _2 = &((*_1).1: AllCopy);
         _0 = copy (*_1);
-        StorageDead(_2);
         return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir
index 9f88e1961ec8..76bb49bc9c1b 100644
--- a/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir
@@ -5,58 +5,28 @@ fn enum_clone_as_copy(_1: &Enum1) -> Enum1 {
     let mut _0: Enum1;
     scope 1 (inlined ::clone) {
         debug self => _1;
-        let mut _2: isize;
-        let mut _3: &AllCopy;
-        let mut _4: &NestCopy;
+        let _2: &AllCopy;
+        let _3: &NestCopy;
         scope 2 {
-            debug __self_0 => _3;
+            debug __self_0 => _2;
             scope 6 (inlined ::clone) {
-                debug self => _3;
+                debug self => _2;
             }
         }
         scope 3 {
-            debug __self_0 => _4;
+            debug __self_0 => _3;
             scope 4 (inlined ::clone) {
-                debug self => _4;
-                let _5: &AllCopy;
+                debug self => _3;
+                let _4: &AllCopy;
                 scope 5 (inlined ::clone) {
-                    debug self => _5;
+                    debug self => _4;
                 }
             }
         }
     }
 
     bb0: {
-        StorageLive(_2);
-        StorageLive(_3);
-        StorageLive(_4);
-        _2 = discriminant((*_1));
-        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
-    }
-
-    bb1: {
-        _3 = &(((*_1) as A).0: AllCopy);
         _0 = copy (*_1);
-        goto -> bb3;
-    }
-
-    bb2: {
-        _4 = &(((*_1) as B).0: NestCopy);
-        StorageLive(_5);
-        _5 = &((((*_1) as B).0: NestCopy).1: AllCopy);
-        StorageDead(_5);
-        _0 = copy (*_1);
-        goto -> bb3;
-    }
-
-    bb3: {
-        StorageDead(_4);
-        StorageDead(_3);
-        StorageDead(_2);
         return;
     }
-
-    bb4: {
-        unreachable;
-    }
 }
diff --git a/tests/mir-opt/pre-codegen/clone_as_copy.rs b/tests/mir-opt/pre-codegen/clone_as_copy.rs
index f5ff1854d387..00f24754d591 100644
--- a/tests/mir-opt/pre-codegen/clone_as_copy.rs
+++ b/tests/mir-opt/pre-codegen/clone_as_copy.rs
@@ -25,19 +25,19 @@ enum Enum1 {
 // EMIT_MIR clone_as_copy.clone_as_copy.PreCodegen.after.mir
 fn clone_as_copy(v: &NestCopy) -> NestCopy {
     // CHECK-LABEL: fn clone_as_copy(
-    // CHECK-NOT: = AllCopy { {{.*}} };
-    // CHECK-NOT: = NestCopy { {{.*}} };
-    // CHECK: _0 = copy (*_1);
-    // CHECK: return;
+    // CHECK: let [[DEAD_VAR:_.*]]: &AllCopy;
+    // CHECK: bb0: {
+    // CHECK-NEXT: DBG: [[DEAD_VAR]] = &((*_1).1: AllCopy)
+    // CHECK-NEXT: _0 = copy (*_1);
+    // CHECK-NEXT: return;
     v.clone()
 }
 
-// FIXME: We can merge into exactly one assignment statement.
 // EMIT_MIR clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir
 fn enum_clone_as_copy(v: &Enum1) -> Enum1 {
     // CHECK-LABEL: fn enum_clone_as_copy(
-    // CHECK-NOT: = Enum1::
-    // CHECK: _0 = copy (*_1);
-    // CHECK: _0 = copy (*_1);
+    // CHECK: bb0: {
+    // CHECK-NEXT: _0 = copy (*_1);
+    // CHECK-NEXT: return;
     v.clone()
 }
diff --git a/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.direct.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.direct.PreCodegen.after.mir
new file mode 100644
index 000000000000..66b51ad48e7a
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.direct.PreCodegen.after.mir
@@ -0,0 +1,46 @@
+// MIR for `direct` after PreCodegen
+
+fn direct(_1: Option) -> bool {
+    debug e => _1;
+    let mut _0: bool;
+    scope 1 (inlined  as PartialEq>::eq) {
+        let mut _2: isize;
+        scope 2 {
+            scope 3 (inlined ::eq) {
+                let _3: i8;
+                scope 4 {
+                    scope 5 {
+                    }
+                }
+            }
+        }
+    }
+
+    bb0: {
+        StorageLive(_2);
+        _2 = discriminant(_1);
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
+    }
+
+    bb1: {
+        _0 = const false;
+        goto -> bb3;
+    }
+
+    bb2: {
+        StorageLive(_3);
+        _3 = discriminant(((_1 as Some).0: std::cmp::Ordering));
+        _0 = Eq(copy _3, const 0_i8);
+        StorageDead(_3);
+        goto -> bb3;
+    }
+
+    bb3: {
+        StorageDead(_2);
+        return;
+    }
+
+    bb4: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.rs b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.rs
new file mode 100644
index 000000000000..3c1e96fa4748
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.rs
@@ -0,0 +1,30 @@
+//@ compile-flags: -O -Zmir-opt-level=2 -Cdebuginfo=0
+
+// Check that comparing `Option` to a constant inlined `Some(...)`
+// does not produce unnecessarily complex MIR compared to using a local binding.
+//
+// Regression test for .
+// Originally, inlined constants like `Some(Ordering::Equal)` would get promoted,
+// leading to more MIR (and extra LLVM IR checks) than necessary.
+// Both cases should now generate identical MIR.
+
+use std::cmp::Ordering;
+
+// EMIT_MIR const_promotion_option_ordering_eq.direct.PreCodegen.after.mir
+pub fn direct(e: Option) -> bool {
+    // CHECK-LABEL: fn direct(
+    // CHECK-NOT: promoted[
+    // CHECK: switchInt(
+    // CHECK: return
+    e == Some(Ordering::Equal)
+}
+
+// EMIT_MIR const_promotion_option_ordering_eq.with_let.PreCodegen.after.mir
+pub fn with_let(e: Option) -> bool {
+    // CHECK-LABEL: fn with_let(
+    // CHECK-NOT: promoted[
+    // CHECK: switchInt(
+    // CHECK: return
+    let eq = Ordering::Equal;
+    e == Some(eq)
+}
diff --git a/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.with_let.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.with_let.PreCodegen.after.mir
new file mode 100644
index 000000000000..f089e9ad960b
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/const_promotion_option_ordering_eq.with_let.PreCodegen.after.mir
@@ -0,0 +1,49 @@
+// MIR for `with_let` after PreCodegen
+
+fn with_let(_1: Option) -> bool {
+    debug e => _1;
+    let mut _0: bool;
+    scope 1 {
+        debug eq => const Equal;
+        scope 2 (inlined  as PartialEq>::eq) {
+            let mut _2: isize;
+            scope 3 {
+                scope 4 (inlined ::eq) {
+                    let _3: i8;
+                    scope 5 {
+                        scope 6 {
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    bb0: {
+        StorageLive(_2);
+        _2 = discriminant(_1);
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
+    }
+
+    bb1: {
+        _0 = const false;
+        goto -> bb3;
+    }
+
+    bb2: {
+        StorageLive(_3);
+        _3 = discriminant(((_1 as Some).0: std::cmp::Ordering));
+        _0 = Eq(copy _3, const 0_i8);
+        StorageDead(_3);
+        goto -> bb3;
+    }
+
+    bb3: {
+        StorageDead(_2);
+        return;
+    }
+
+    bb4: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/dead_on_invalid_place.invalid_place.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/dead_on_invalid_place.invalid_place.PreCodegen.after.mir
new file mode 100644
index 000000000000..4a2127178fb0
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/dead_on_invalid_place.invalid_place.PreCodegen.after.mir
@@ -0,0 +1,13 @@
+// MIR for `invalid_place` after PreCodegen
+
+fn invalid_place(_1: bool) -> bool {
+    debug c1_ref => _2;
+    let mut _0: bool;
+    let mut _2: &bool;
+
+    bb0: {
+        // DBG: _2 = &?;
+        _0 = copy _1;
+        return;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/dead_on_invalid_place.rs b/tests/mir-opt/pre-codegen/dead_on_invalid_place.rs
new file mode 100644
index 000000000000..5abe9fa43a5d
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/dead_on_invalid_place.rs
@@ -0,0 +1,27 @@
+#![feature(core_intrinsics, custom_mir)]
+#![crate_type = "lib"]
+
+use std::intrinsics::mir::*;
+
+// EMIT_MIR dead_on_invalid_place.invalid_place.PreCodegen.after.mir
+#[custom_mir(dialect = "runtime")]
+pub fn invalid_place(c: bool) -> bool {
+    // CHECK-LABEL: fn invalid_place
+    // CHECK: debug c1_ref => [[c1_ref:_[0-9]+]];
+    // CHECK: bb0: {
+    // We cannot read the reference, since `c1` is dead.
+    // CHECK-NEXT: DBG: [[c1_ref]] = &?
+    // CHECK-NEXT: _0 = copy _1;
+    // CHECK-NEXT: return;
+    mir! {
+        let _c1_ref: &bool;
+        let c1: bool;
+        debug c1_ref => _c1_ref;
+        {
+            c1 = c;
+            _c1_ref = &c1;
+            RET = c;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-abort.diff b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-abort.diff
index 993857f225a8..269af438e37e 100644
--- a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-abort.diff
+++ b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-abort.diff
@@ -16,17 +16,15 @@
   
       bb0: {
 -         StorageLive(_2);
--         _6 = deref_copy (*_1);
 +         nop;
-+         _6 = copy (*_1);
+          _6 = copy (*_1);
           _2 = copy (*_6);
           _3 = unknown() -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
           StorageLive(_4);
--         _7 = deref_copy (*_1);
-+         _7 = copy (*_1);
+          _7 = copy (*_1);
           _4 = copy (*_7);
           StorageLive(_5);
           _5 = copy _2;
diff --git a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff
index d81bfa9310bc..9ce17342a445 100644
--- a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff
+++ b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff
@@ -16,17 +16,15 @@
   
       bb0: {
 -         StorageLive(_2);
--         _6 = deref_copy (*_1);
 +         nop;
-+         _6 = copy (*_1);
+          _6 = copy (*_1);
           _2 = copy (*_6);
           _3 = unknown() -> [return: bb1, unwind continue];
       }
   
       bb1: {
           StorageLive(_4);
--         _7 = deref_copy (*_1);
-+         _7 = copy (*_1);
+          _7 = copy (*_1);
           _4 = copy (*_7);
           StorageLive(_5);
           _5 = copy _2;
diff --git a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir
index 8746cb089916..e235fa35c023 100644
--- a/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/derived_ord.demo_le.PreCodegen.after.mir
@@ -5,8 +5,9 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
     debug b => _2;
     let mut _0: bool;
     scope 1 (inlined ::le) {
-        let mut _11: std::option::Option;
+        let mut _6: std::option::Option;
         scope 2 (inlined Option::::is_some_and:: bool {std::cmp::Ordering::is_le}>) {
+            let mut _11: isize;
             let _12: std::cmp::Ordering;
             scope 3 {
                 scope 4 (inlined  bool {std::cmp::Ordering::is_le} as FnOnce<(std::cmp::Ordering,)>>::call_once - shim(fn(std::cmp::Ordering) -> bool {std::cmp::Ordering::is_le})) {
@@ -19,7 +20,6 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
             }
         }
         scope 7 (inlined ::partial_cmp) {
-            let mut _6: std::option::Option;
             let mut _7: i8;
             scope 8 {
             }
@@ -38,9 +38,8 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
 
     bb0: {
         StorageLive(_12);
-        StorageLive(_11);
-        StorageLive(_5);
         StorageLive(_6);
+        StorageLive(_5);
         StorageLive(_7);
         StorageLive(_3);
         _3 = copy ((*_1).0: char);
@@ -63,30 +62,44 @@ fn demo_le(_1: &MultiField, _2: &MultiField) -> bool {
         _10 = Cmp(move _8, move _9);
         StorageDead(_9);
         StorageDead(_8);
-        _11 = Option::::Some(move _10);
+        _6 = Option::::Some(move _10);
         StorageDead(_10);
         StorageDead(_7);
-        StorageDead(_6);
         StorageDead(_5);
-        goto -> bb3;
+        StorageLive(_11);
+        goto -> bb4;
     }
 
     bb2: {
-        _11 = copy _6;
         StorageDead(_7);
-        StorageDead(_6);
         StorageDead(_5);
-        goto -> bb3;
+        StorageLive(_11);
+        _11 = discriminant(_6);
+        switchInt(move _11) -> [0: bb3, 1: bb4, otherwise: bb6];
     }
 
     bb3: {
-        _12 = move ((_11 as Some).0: std::cmp::Ordering);
+        _0 = const false;
+        goto -> bb5;
+    }
+
+    bb4: {
+        _12 = move ((_6 as Some).0: std::cmp::Ordering);
         StorageLive(_13);
         _13 = discriminant(_12);
         _0 = Le(move _13, const 0_i8);
         StorageDead(_13);
+        goto -> bb5;
+    }
+
+    bb5: {
         StorageDead(_11);
+        StorageDead(_6);
         StorageDead(_12);
         return;
     }
+
+    bb6: {
+        unreachable;
+    }
 }
diff --git a/tests/mir-opt/pre-codegen/derived_ord.rs b/tests/mir-opt/pre-codegen/derived_ord.rs
index 73ae923a6cb9..823e0f6d09c2 100644
--- a/tests/mir-opt/pre-codegen/derived_ord.rs
+++ b/tests/mir-opt/pre-codegen/derived_ord.rs
@@ -25,7 +25,10 @@ pub fn demo_le(a: &MultiField, b: &MultiField) -> bool {
     // CHECK: Cmp(move [[A1]], move [[B1]]);
 
     // CHECK: [[D1:_[0-9]+]] = discriminant({{.+}});
-    // CHECK: _0 = Le(move [[D1]], const 0_i8);
+    // CHECK: switchInt(move [[D1]]) -> [0: bb{{[0-9]+}}, 1: bb{{[0-9]+}}, otherwise: bb{{[0-9]+}}];
+
+    // CHECK: [[D2:_[0-9]+]] = discriminant({{.+}});
+    // CHECK: _0 = Le(move [[D2]], const 0_i8);
     *a <= *b
 }
 
diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
index de25eebee77f..5993bd79d274 100644
--- a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
@@ -4,10 +4,9 @@ fn ::partial_cmp(_1: &MultiField, _2: &M
     debug self => _1;
     debug other => _2;
     let mut _0: std::option::Option;
-    let mut _6: std::option::Option;
-    let mut _7: i8;
+    let mut _6: i8;
     scope 1 {
-        debug cmp => _6;
+        debug cmp => _0;
     }
     scope 2 (inlined std::cmp::impls::::partial_cmp) {
         let mut _3: char;
@@ -15,9 +14,9 @@ fn ::partial_cmp(_1: &MultiField, _2: &M
         let mut _5: std::cmp::Ordering;
     }
     scope 3 (inlined std::cmp::impls::::partial_cmp) {
+        let mut _7: i16;
         let mut _8: i16;
-        let mut _9: i16;
-        let mut _10: std::cmp::Ordering;
+        let mut _9: std::cmp::Ordering;
     }
 
     bb0: {
@@ -28,27 +27,26 @@ fn ::partial_cmp(_1: &MultiField, _2: &M
         _5 = Cmp(move _3, move _4);
         StorageDead(_4);
         StorageDead(_3);
-        _6 = Option::::Some(copy _5);
-        _7 = discriminant(_5);
-        switchInt(move _7) -> [0: bb1, otherwise: bb2];
+        _0 = Option::::Some(copy _5);
+        _6 = discriminant(_5);
+        switchInt(move _6) -> [0: bb1, otherwise: bb2];
     }
 
     bb1: {
-        StorageLive(_10);
-        StorageLive(_8);
-        _8 = copy ((*_1).1: i16);
         StorageLive(_9);
-        _9 = copy ((*_2).1: i16);
-        _10 = Cmp(move _8, move _9);
-        StorageDead(_9);
+        StorageLive(_7);
+        _7 = copy ((*_1).1: i16);
+        StorageLive(_8);
+        _8 = copy ((*_2).1: i16);
+        _9 = Cmp(move _7, move _8);
         StorageDead(_8);
-        _0 = Option::::Some(move _10);
-        StorageDead(_10);
+        StorageDead(_7);
+        _0 = Option::::Some(move _9);
+        StorageDead(_9);
         goto -> bb3;
     }
 
     bb2: {
-        _0 = copy _6;
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/pre-codegen/derived_ord_debug.rs b/tests/mir-opt/pre-codegen/derived_ord_debug.rs
new file mode 100644
index 000000000000..1d6a884cee44
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord_debug.rs
@@ -0,0 +1,18 @@
+//@ compile-flags: -Copt-level=0 -Zmir-opt-level=1 -Cdebuginfo=limited
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
+#![crate_type = "lib"]
+
+#[derive(PartialOrd, Ord, PartialEq, Eq)]
+pub struct MultiField(char, i16);
+
+// EMIT_MIR derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.mir
+// EMIT_MIR derived_ord_debug.{impl#1}-cmp.PreCodegen.after.mir
+
+// CHECK-LABEL: partial_cmp(_1: &MultiField, _2: &MultiField) -> Option
+// CHECK: = ::partial_cmp(
+// CHECK: = ::partial_cmp(
+
+// CHECK-LABEL: cmp(_1: &MultiField, _2: &MultiField) -> std::cmp::Ordering
+// CHECK: = ::cmp(
+// CHECK: = ::cmp(
diff --git a/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-abort.mir
new file mode 100644
index 000000000000..9fc8da3a1120
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,52 @@
+// MIR for `::partial_cmp` after PreCodegen
+
+fn ::partial_cmp(_1: &MultiField, _2: &MultiField) -> Option {
+    debug self => _1;
+    debug other => _2;
+    let mut _0: std::option::Option;
+    let _3: &char;
+    let _4: &char;
+    let mut _5: std::option::Option;
+    let mut _6: isize;
+    let mut _7: i8;
+    let _8: &i16;
+    let _9: &i16;
+    scope 1 {
+        debug cmp => _5;
+    }
+
+    bb0: {
+        _3 = &((*_1).0: char);
+        _4 = &((*_2).0: char);
+        _5 = ::partial_cmp(copy _3, copy _4) -> [return: bb1, unwind unreachable];
+    }
+
+    bb1: {
+        _6 = discriminant(_5);
+        switchInt(move _6) -> [1: bb2, 0: bb4, otherwise: bb6];
+    }
+
+    bb2: {
+        _7 = discriminant(((_5 as Some).0: std::cmp::Ordering));
+        switchInt(move _7) -> [0: bb3, otherwise: bb4];
+    }
+
+    bb3: {
+        _8 = &((*_1).1: i16);
+        _9 = &((*_2).1: i16);
+        _0 = ::partial_cmp(copy _8, copy _9) -> [return: bb5, unwind unreachable];
+    }
+
+    bb4: {
+        _0 = copy _5;
+        goto -> bb5;
+    }
+
+    bb5: {
+        return;
+    }
+
+    bb6: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-unwind.mir
new file mode 100644
index 000000000000..29cc54150764
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#0}-partial_cmp.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,52 @@
+// MIR for `::partial_cmp` after PreCodegen
+
+fn ::partial_cmp(_1: &MultiField, _2: &MultiField) -> Option {
+    debug self => _1;
+    debug other => _2;
+    let mut _0: std::option::Option;
+    let _3: &char;
+    let _4: &char;
+    let mut _5: std::option::Option;
+    let mut _6: isize;
+    let mut _7: i8;
+    let _8: &i16;
+    let _9: &i16;
+    scope 1 {
+        debug cmp => _5;
+    }
+
+    bb0: {
+        _3 = &((*_1).0: char);
+        _4 = &((*_2).0: char);
+        _5 = ::partial_cmp(copy _3, copy _4) -> [return: bb1, unwind continue];
+    }
+
+    bb1: {
+        _6 = discriminant(_5);
+        switchInt(move _6) -> [1: bb2, 0: bb4, otherwise: bb6];
+    }
+
+    bb2: {
+        _7 = discriminant(((_5 as Some).0: std::cmp::Ordering));
+        switchInt(move _7) -> [0: bb3, otherwise: bb4];
+    }
+
+    bb3: {
+        _8 = &((*_1).1: i16);
+        _9 = &((*_2).1: i16);
+        _0 = ::partial_cmp(copy _8, copy _9) -> [return: bb5, unwind continue];
+    }
+
+    bb4: {
+        _0 = copy _5;
+        goto -> bb5;
+    }
+
+    bb5: {
+        return;
+    }
+
+    bb6: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-abort.mir
new file mode 100644
index 000000000000..66d96021aa9b
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,42 @@
+// MIR for `::cmp` after PreCodegen
+
+fn ::cmp(_1: &MultiField, _2: &MultiField) -> std::cmp::Ordering {
+    debug self => _1;
+    debug other => _2;
+    let mut _0: std::cmp::Ordering;
+    let _3: &char;
+    let _4: &char;
+    let mut _5: std::cmp::Ordering;
+    let mut _6: i8;
+    let _7: &i16;
+    let _8: &i16;
+    scope 1 {
+        debug cmp => _5;
+    }
+
+    bb0: {
+        _3 = &((*_1).0: char);
+        _4 = &((*_2).0: char);
+        _5 = ::cmp(copy _3, copy _4) -> [return: bb1, unwind unreachable];
+    }
+
+    bb1: {
+        _6 = discriminant(_5);
+        switchInt(move _6) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        _7 = &((*_1).1: i16);
+        _8 = &((*_2).1: i16);
+        _0 = ::cmp(copy _7, copy _8) -> [return: bb4, unwind unreachable];
+    }
+
+    bb3: {
+        _0 = copy _5;
+        goto -> bb4;
+    }
+
+    bb4: {
+        return;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-unwind.mir
new file mode 100644
index 000000000000..2dcba6195f71
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord_debug.{impl#1}-cmp.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,42 @@
+// MIR for `::cmp` after PreCodegen
+
+fn ::cmp(_1: &MultiField, _2: &MultiField) -> std::cmp::Ordering {
+    debug self => _1;
+    debug other => _2;
+    let mut _0: std::cmp::Ordering;
+    let _3: &char;
+    let _4: &char;
+    let mut _5: std::cmp::Ordering;
+    let mut _6: i8;
+    let _7: &i16;
+    let _8: &i16;
+    scope 1 {
+        debug cmp => _5;
+    }
+
+    bb0: {
+        _3 = &((*_1).0: char);
+        _4 = &((*_2).0: char);
+        _5 = ::cmp(copy _3, copy _4) -> [return: bb1, unwind continue];
+    }
+
+    bb1: {
+        _6 = discriminant(_5);
+        switchInt(move _6) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        _7 = &((*_1).1: i16);
+        _8 = &((*_2).1: i16);
+        _0 = ::cmp(copy _7, copy _8) -> [return: bb4, unwind continue];
+    }
+
+    bb3: {
+        _0 = copy _5;
+        goto -> bb4;
+    }
+
+    bb4: {
+        return;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
index 2777bba893bb..791d6b71a6f7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-abort.mir
@@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
             let _2: std::ptr::NonNull<[T]>;
             let mut _3: *mut [T];
             let mut _4: *const [T];
-            let _12: ();
+            let _11: ();
             scope 3 {
                 let _8: std::ptr::alignment::AlignmentEnum;
                 scope 4 {
@@ -31,12 +31,11 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         scope 20 (inlined NonNull::::as_ptr) {
                         }
                         scope 21 (inlined std::alloc::dealloc) {
-                            let mut _11: usize;
+                            let mut _10: usize;
                             scope 22 (inlined Layout::size) {
                             }
                             scope 23 (inlined Layout::align) {
                                 scope 24 (inlined std::ptr::Alignment::as_usize) {
-                                    let mut _10: u32;
                                 }
                             }
                         }
@@ -74,7 +73,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     }
 
     bb1: {
-        _6 = AlignOf(T);
+        _6 = const ::ALIGN;
         StorageLive(_7);
         _7 = copy _6 as std::ptr::Alignment (Transmute);
         _8 = move (_7.0: std::ptr::alignment::AlignmentEnum);
@@ -87,16 +86,13 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     bb2: {
         StorageLive(_9);
         _9 = copy _3 as *mut u8 (PtrToPtr);
-        StorageLive(_11);
         StorageLive(_10);
         _10 = discriminant(_8);
-        _11 = move _10 as usize (IntToInt);
-        StorageDead(_10);
-        _12 = alloc::alloc::__rust_dealloc(move _9, move _5, move _11) -> [return: bb3, unwind unreachable];
+        _11 = alloc::alloc::__rust_dealloc(move _9, move _5, move _10) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
         goto -> bb4;
     }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
index 2777bba893bb..791d6b71a6f7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.32bit.panic-unwind.mir
@@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
             let _2: std::ptr::NonNull<[T]>;
             let mut _3: *mut [T];
             let mut _4: *const [T];
-            let _12: ();
+            let _11: ();
             scope 3 {
                 let _8: std::ptr::alignment::AlignmentEnum;
                 scope 4 {
@@ -31,12 +31,11 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         scope 20 (inlined NonNull::::as_ptr) {
                         }
                         scope 21 (inlined std::alloc::dealloc) {
-                            let mut _11: usize;
+                            let mut _10: usize;
                             scope 22 (inlined Layout::size) {
                             }
                             scope 23 (inlined Layout::align) {
                                 scope 24 (inlined std::ptr::Alignment::as_usize) {
-                                    let mut _10: u32;
                                 }
                             }
                         }
@@ -74,7 +73,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     }
 
     bb1: {
-        _6 = AlignOf(T);
+        _6 = const ::ALIGN;
         StorageLive(_7);
         _7 = copy _6 as std::ptr::Alignment (Transmute);
         _8 = move (_7.0: std::ptr::alignment::AlignmentEnum);
@@ -87,16 +86,13 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     bb2: {
         StorageLive(_9);
         _9 = copy _3 as *mut u8 (PtrToPtr);
-        StorageLive(_11);
         StorageLive(_10);
         _10 = discriminant(_8);
-        _11 = move _10 as usize (IntToInt);
-        StorageDead(_10);
-        _12 = alloc::alloc::__rust_dealloc(move _9, move _5, move _11) -> [return: bb3, unwind unreachable];
+        _11 = alloc::alloc::__rust_dealloc(move _9, move _5, move _10) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
         goto -> bb4;
     }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
index 2be0a478c852..791d6b71a6f7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-abort.mir
@@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
             let _2: std::ptr::NonNull<[T]>;
             let mut _3: *mut [T];
             let mut _4: *const [T];
-            let _12: ();
+            let _11: ();
             scope 3 {
                 let _8: std::ptr::alignment::AlignmentEnum;
                 scope 4 {
@@ -31,12 +31,11 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         scope 20 (inlined NonNull::::as_ptr) {
                         }
                         scope 21 (inlined std::alloc::dealloc) {
-                            let mut _11: usize;
+                            let mut _10: usize;
                             scope 22 (inlined Layout::size) {
                             }
                             scope 23 (inlined Layout::align) {
                                 scope 24 (inlined std::ptr::Alignment::as_usize) {
-                                    let mut _10: u64;
                                 }
                             }
                         }
@@ -74,7 +73,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     }
 
     bb1: {
-        _6 = AlignOf(T);
+        _6 = const ::ALIGN;
         StorageLive(_7);
         _7 = copy _6 as std::ptr::Alignment (Transmute);
         _8 = move (_7.0: std::ptr::alignment::AlignmentEnum);
@@ -87,16 +86,13 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     bb2: {
         StorageLive(_9);
         _9 = copy _3 as *mut u8 (PtrToPtr);
-        StorageLive(_11);
         StorageLive(_10);
         _10 = discriminant(_8);
-        _11 = move _10 as usize (IntToInt);
-        StorageDead(_10);
-        _12 = alloc::alloc::__rust_dealloc(move _9, move _5, move _11) -> [return: bb3, unwind unreachable];
+        _11 = alloc::alloc::__rust_dealloc(move _9, move _5, move _10) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
         goto -> bb4;
     }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
index 2be0a478c852..791d6b71a6f7 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.generic_in_place.PreCodegen.after.64bit.panic-unwind.mir
@@ -8,7 +8,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
             let _2: std::ptr::NonNull<[T]>;
             let mut _3: *mut [T];
             let mut _4: *const [T];
-            let _12: ();
+            let _11: ();
             scope 3 {
                 let _8: std::ptr::alignment::AlignmentEnum;
                 scope 4 {
@@ -31,12 +31,11 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
                         scope 20 (inlined NonNull::::as_ptr) {
                         }
                         scope 21 (inlined std::alloc::dealloc) {
-                            let mut _11: usize;
+                            let mut _10: usize;
                             scope 22 (inlined Layout::size) {
                             }
                             scope 23 (inlined Layout::align) {
                                 scope 24 (inlined std::ptr::Alignment::as_usize) {
-                                    let mut _10: u64;
                                 }
                             }
                         }
@@ -74,7 +73,7 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     }
 
     bb1: {
-        _6 = AlignOf(T);
+        _6 = const ::ALIGN;
         StorageLive(_7);
         _7 = copy _6 as std::ptr::Alignment (Transmute);
         _8 = move (_7.0: std::ptr::alignment::AlignmentEnum);
@@ -87,16 +86,13 @@ fn generic_in_place(_1: *mut Box<[T]>) -> () {
     bb2: {
         StorageLive(_9);
         _9 = copy _3 as *mut u8 (PtrToPtr);
-        StorageLive(_11);
         StorageLive(_10);
         _10 = discriminant(_8);
-        _11 = move _10 as usize (IntToInt);
-        StorageDead(_10);
-        _12 = alloc::alloc::__rust_dealloc(move _9, move _5, move _11) -> [return: bb3, unwind unreachable];
+        _11 = alloc::alloc::__rust_dealloc(move _9, move _5, move _10) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
         goto -> bb4;
     }
diff --git a/tests/mir-opt/pre-codegen/drop_boxed_slice.rs b/tests/mir-opt/pre-codegen/drop_boxed_slice.rs
index 11fb7afef0fb..83b10f8bd688 100644
--- a/tests/mir-opt/pre-codegen/drop_boxed_slice.rs
+++ b/tests/mir-opt/pre-codegen/drop_boxed_slice.rs
@@ -9,11 +9,10 @@ pub unsafe fn generic_in_place(ptr: *mut Box<[T]>) {
     // CHECK-LABEL: fn generic_in_place(_1: *mut Box<[T]>)
     // CHECK: (inlined  as Drop>::drop)
     // CHECK: [[SIZE:_.+]] = std::intrinsics::size_of_val::<[T]>
-    // CHECK: [[ALIGN:_.+]] = AlignOf(T);
+    // CHECK: [[ALIGN:_.+]] = const ::ALIGN;
     // CHECK: [[B:_.+]] = copy [[ALIGN]] as std::ptr::Alignment (Transmute);
     // CHECK: [[C:_.+]] = move ([[B]].0: std::ptr::alignment::AlignmentEnum);
     // CHECK: [[D:_.+]] = discriminant([[C]]);
-    // CHECK: [[E:_.+]] = move [[D]] as usize (IntToInt);
-    // CHECK: = alloc::alloc::__rust_dealloc({{.+}}, move [[SIZE]], move [[E]]) ->
+    // CHECK: = alloc::alloc::__rust_dealloc({{.+}}, move [[SIZE]], move [[D]]) ->
     std::ptr::drop_in_place(ptr)
 }
diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff
index c2d144c98c3a..3f854b6cbcfb 100644
--- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff
+++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-abort.diff
@@ -103,7 +103,7 @@
           StorageDead(_6);
           _4 = copy _5 as *mut [u8] (Transmute);
           StorageDead(_5);
-          _3 = move _4 as *mut u8 (PtrToPtr);
+          _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);
           StorageDead(_3);
 -         StorageDead(_1);
diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff
index 88bd4628c297..15a9d9e39c49 100644
--- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff
+++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.32bit.panic-unwind.diff
@@ -46,7 +46,7 @@
           StorageDead(_6);
           _4 = copy _5 as *mut [u8] (Transmute);
           StorageDead(_5);
-          _3 = move _4 as *mut u8 (PtrToPtr);
+          _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);
           StorageDead(_3);
 -         StorageDead(_1);
diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff
index 8641d2d6fa85..1dbe9394e709 100644
--- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff
+++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-abort.diff
@@ -103,7 +103,7 @@
           StorageDead(_6);
           _4 = copy _5 as *mut [u8] (Transmute);
           StorageDead(_5);
-          _3 = move _4 as *mut u8 (PtrToPtr);
+          _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);
           StorageDead(_3);
 -         StorageDead(_1);
diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff
index 0c52f1e05836..df008ececae3 100644
--- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff
+++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff
@@ -46,7 +46,7 @@
           StorageDead(_6);
           _4 = copy _5 as *mut [u8] (Transmute);
           StorageDead(_5);
-          _3 = move _4 as *mut u8 (PtrToPtr);
+          _3 = copy _4 as *mut u8 (PtrToPtr);
           StorageDead(_4);
           StorageDead(_3);
 -         StorageDead(_1);
diff --git a/tests/mir-opt/pre-codegen/loops.filter_mapped.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.filter_mapped.PreCodegen.after.mir
index 75e8cb1d8618..8f30ad30fccd 100644
--- a/tests/mir-opt/pre-codegen/loops.filter_mapped.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/loops.filter_mapped.PreCodegen.after.mir
@@ -6,20 +6,20 @@ fn filter_mapped(_1: impl Iterator, _2: impl Fn(T) -> Option) -> ()
     let mut _0: ();
     let mut _3: std::iter::FilterMap, impl Fn(T) -> Option>;
     let mut _4: std::iter::FilterMap, impl Fn(T) -> Option>;
-    let mut _5: &mut std::iter::FilterMap, impl Fn(T) -> Option>;
-    let mut _8: std::option::Option;
-    let mut _9: isize;
-    let _11: ();
+    let mut _7: std::option::Option;
+    let mut _8: isize;
+    let _10: ();
+    let mut _11: &mut std::iter::FilterMap, impl Fn(T) -> Option>;
     scope 1 {
         debug iter => _4;
-        let _10: U;
+        let _9: U;
         scope 2 {
-            debug x => _10;
+            debug x => _9;
         }
         scope 4 (inlined , impl Fn(T) -> Option> as Iterator>::next) {
-            debug self => _5;
-            let mut _6: &mut impl Iterator;
-            let mut _7: &mut impl Fn(T) -> Option;
+            debug self => _11;
+            let mut _5: &mut impl Iterator;
+            let mut _6: &mut impl Fn(T) -> Option;
         }
     }
     scope 3 (inlined , impl Fn(T) -> Option> as IntoIterator>::into_iter) {
@@ -37,24 +37,24 @@ fn filter_mapped(_1: impl Iterator, _2: impl Fn(T) -> Option) -> ()
     }
 
     bb2: {
-        StorageLive(_8);
-        _5 = &mut _4;
-        StorageLive(_6);
-        _6 = &mut (_4.0: impl Iterator);
         StorageLive(_7);
-        _7 = &mut (_4.1: impl Fn(T) -> Option);
-        _8 =  as Iterator>::find_map:: Option>(move _6, move _7) -> [return: bb3, unwind: bb9];
+        // DBG: _11 = &_4;
+        StorageLive(_5);
+        _5 = &mut (_4.0: impl Iterator);
+        StorageLive(_6);
+        _6 = &mut (_4.1: impl Fn(T) -> Option);
+        _7 =  as Iterator>::find_map:: Option>(move _5, move _6) -> [return: bb3, unwind: bb9];
     }
 
     bb3: {
-        StorageDead(_7);
         StorageDead(_6);
-        _9 = discriminant(_8);
-        switchInt(move _9) -> [0: bb4, 1: bb6, otherwise: bb8];
+        StorageDead(_5);
+        _8 = discriminant(_7);
+        switchInt(move _8) -> [0: bb4, 1: bb6, otherwise: bb8];
     }
 
     bb4: {
-        StorageDead(_8);
+        StorageDead(_7);
         drop(_4) -> [return: bb5, unwind continue];
     }
 
@@ -64,12 +64,12 @@ fn filter_mapped(_1: impl Iterator, _2: impl Fn(T) -> Option) -> ()
     }
 
     bb6: {
-        _10 = move ((_8 as Some).0: U);
-        _11 = opaque::(move _10) -> [return: bb7, unwind: bb9];
+        _9 = move ((_7 as Some).0: U);
+        _10 = opaque::(move _9) -> [return: bb7, unwind: bb9];
     }
 
     bb7: {
-        StorageDead(_8);
+        StorageDead(_7);
         goto -> bb2;
     }
 
diff --git a/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir
index 154cbd3791cb..beb7b936ccf7 100644
--- a/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir
@@ -5,32 +5,31 @@ fn int_range(_1: usize, _2: usize) -> () {
     debug end => _2;
     let mut _0: ();
     let mut _3: std::ops::Range;
-    let mut _4: std::ops::Range;
-    let mut _5: &mut std::ops::Range;
-    let mut _13: std::option::Option;
-    let _15: ();
+    let mut _9: std::option::Option;
+    let _11: ();
+    let mut _12: &mut std::ops::Range;
     scope 1 {
-        debug iter => _4;
-        let _14: usize;
+        debug iter => _3;
+        let _10: usize;
         scope 2 {
-            debug i => _14;
+            debug i => _10;
         }
         scope 4 (inlined iter::range::>::next) {
-            debug self => _5;
+            debug self => _12;
             scope 5 (inlined  as iter::range::RangeIteratorImpl>::spec_next) {
-                debug self => _5;
-                let mut _6: &usize;
-                let mut _7: &usize;
-                let mut _10: bool;
-                let _11: usize;
-                let mut _12: usize;
+                debug self => _12;
+                let mut _6: bool;
+                let _7: usize;
+                let mut _8: usize;
+                let mut _13: &usize;
+                let mut _14: &usize;
                 scope 6 {
-                    debug old => _11;
+                    debug old => _7;
                     scope 8 (inlined ::forward_unchecked) {
-                        debug start => _11;
+                        debug start => _7;
                         debug n => const 1_usize;
                         scope 9 (inlined #[track_caller] core::num::::unchecked_add) {
-                            debug self => _11;
+                            debug self => _7;
                             debug rhs => const 1_usize;
                             scope 10 (inlined core::ub_checks::check_language_ub) {
                                 scope 11 (inlined core::ub_checks::check_language_ub::runtime) {
@@ -40,10 +39,10 @@ fn int_range(_1: usize, _2: usize) -> () {
                     }
                 }
                 scope 7 (inlined std::cmp::impls::::lt) {
-                    debug self => _6;
-                    debug other => _7;
-                    let mut _8: usize;
-                    let mut _9: usize;
+                    debug self => _13;
+                    debug other => _14;
+                    let mut _4: usize;
+                    let mut _5: usize;
                 }
             }
         }
@@ -54,54 +53,45 @@ fn int_range(_1: usize, _2: usize) -> () {
 
     bb0: {
         _3 = std::ops::Range:: { start: copy _1, end: copy _2 };
-        StorageLive(_4);
-        _4 = copy _3;
         goto -> bb1;
     }
 
     bb1: {
-        StorageLive(_13);
-        _5 = &mut _4;
-        StorageLive(_10);
-        StorageLive(_6);
-        _6 = &(_4.0: usize);
-        StorageLive(_7);
-        _7 = &(_4.1: usize);
-        StorageLive(_8);
-        _8 = copy (_4.0: usize);
         StorageLive(_9);
-        _9 = copy (_4.1: usize);
-        _10 = Lt(move _8, move _9);
-        StorageDead(_9);
-        StorageDead(_8);
-        switchInt(move _10) -> [0: bb2, otherwise: bb3];
+        // DBG: _12 = &_3;
+        StorageLive(_6);
+        // DBG: _13 = &(_3.0: usize);
+        // DBG: _14 = &(_3.1: usize);
+        StorageLive(_4);
+        _4 = copy (_3.0: usize);
+        StorageLive(_5);
+        _5 = copy (_3.1: usize);
+        _6 = Lt(move _4, move _5);
+        StorageDead(_5);
+        StorageDead(_4);
+        switchInt(move _6) -> [0: bb2, otherwise: bb3];
     }
 
     bb2: {
-        StorageDead(_7);
         StorageDead(_6);
-        StorageDead(_10);
-        StorageDead(_13);
-        StorageDead(_4);
+        StorageDead(_9);
         return;
     }
 
     bb3: {
-        StorageDead(_7);
+        _7 = copy (_3.0: usize);
+        StorageLive(_8);
+        _8 = AddUnchecked(copy _7, const 1_usize);
+        (_3.0: usize) = move _8;
+        StorageDead(_8);
+        _9 = Option::::Some(copy _7);
         StorageDead(_6);
-        _11 = copy (_4.0: usize);
-        StorageLive(_12);
-        _12 = AddUnchecked(copy _11, const 1_usize);
-        (_4.0: usize) = move _12;
-        StorageDead(_12);
-        _13 = Option::::Some(copy _11);
-        StorageDead(_10);
-        _14 = copy ((_13 as Some).0: usize);
-        _15 = opaque::(move _14) -> [return: bb4, unwind continue];
+        _10 = copy ((_9 as Some).0: usize);
+        _11 = opaque::(move _10) -> [return: bb4, unwind continue];
     }
 
     bb4: {
-        StorageDead(_13);
+        StorageDead(_9);
         goto -> bb1;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/loops.mapped.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.mapped.PreCodegen.after.mir
index d22ea54004c9..406c96fc32f4 100644
--- a/tests/mir-opt/pre-codegen/loops.mapped.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/loops.mapped.PreCodegen.after.mir
@@ -6,32 +6,32 @@ fn mapped(_1: impl Iterator, _2: impl Fn(T) -> U) -> () {
     let mut _0: ();
     let mut _3: std::iter::Map, impl Fn(T) -> U>;
     let mut _4: std::iter::Map, impl Fn(T) -> U>;
-    let mut _5: &mut std::iter::Map, impl Fn(T) -> U>;
-    let mut _13: std::option::Option;
-    let _15: ();
+    let mut _12: std::option::Option;
+    let _14: ();
+    let mut _15: &mut std::iter::Map, impl Fn(T) -> U>;
     scope 1 {
         debug iter => _4;
-        let _14: U;
+        let _13: U;
         scope 2 {
-            debug x => _14;
+            debug x => _13;
         }
         scope 4 (inlined , impl Fn(T) -> U> as Iterator>::next) {
-            debug self => _5;
-            let mut _6: &mut impl Iterator;
-            let mut _7: std::option::Option;
-            let mut _8: &mut impl Fn(T) -> U;
+            debug self => _15;
+            let mut _5: &mut impl Iterator;
+            let mut _6: std::option::Option;
+            let mut _7: &mut impl Fn(T) -> U;
             scope 5 (inlined Option::::map:: U>) {
-                debug self => _7;
-                debug f => _8;
-                let mut _9: isize;
-                let _10: T;
-                let mut _11: (T,);
-                let mut _12: U;
+                debug self => _6;
+                debug f => _7;
+                let mut _8: isize;
+                let _9: T;
+                let mut _10: (T,);
+                let mut _11: U;
                 scope 6 {
-                    debug x => _10;
+                    debug x => _9;
                     scope 7 (inlined ops::function::impls:: for &mut impl Fn(T) -> U>::call_once) {
-                        debug self => _8;
-                        debug args => _11;
+                        debug self => _7;
+                        debug args => _10;
                     }
                 }
             }
@@ -52,30 +52,30 @@ fn mapped(_1: impl Iterator, _2: impl Fn(T) -> U) -> () {
     }
 
     bb2: {
-        StorageLive(_13);
-        _5 = &mut _4;
-        StorageLive(_8);
+        StorageLive(_12);
+        // DBG: _15 = &_4;
         StorageLive(_7);
         StorageLive(_6);
-        _6 = &mut (_4.0: impl Iterator);
-        _7 =  as Iterator>::next(move _6) -> [return: bb3, unwind: bb10];
+        StorageLive(_5);
+        _5 = &mut (_4.0: impl Iterator);
+        _6 =  as Iterator>::next(move _5) -> [return: bb3, unwind: bb10];
     }
 
     bb3: {
-        StorageDead(_6);
-        _8 = &mut (_4.1: impl Fn(T) -> U);
+        StorageDead(_5);
+        _7 = &mut (_4.1: impl Fn(T) -> U);
+        StorageLive(_8);
         StorageLive(_9);
-        StorageLive(_10);
-        _9 = discriminant(_7);
-        switchInt(move _9) -> [0: bb4, 1: bb6, otherwise: bb9];
+        _8 = discriminant(_6);
+        switchInt(move _8) -> [0: bb4, 1: bb6, otherwise: bb9];
     }
 
     bb4: {
-        StorageDead(_10);
         StorageDead(_9);
-        StorageDead(_7);
         StorageDead(_8);
-        StorageDead(_13);
+        StorageDead(_6);
+        StorageDead(_7);
+        StorageDead(_12);
         drop(_4) -> [return: bb5, unwind continue];
     }
 
@@ -85,27 +85,27 @@ fn mapped(_1: impl Iterator, _2: impl Fn(T) -> U) -> () {
     }
 
     bb6: {
-        _10 = move ((_7 as Some).0: T);
-        StorageLive(_12);
+        _9 = move ((_6 as Some).0: T);
         StorageLive(_11);
-        _11 = (copy _10,);
-        _12 =  U as FnMut<(T,)>>::call_mut(move _8, move _11) -> [return: bb7, unwind: bb10];
+        StorageLive(_10);
+        _10 = (copy _9,);
+        _11 =  U as FnMut<(T,)>>::call_mut(move _7, move _10) -> [return: bb7, unwind: bb10];
     }
 
     bb7: {
-        StorageDead(_11);
-        _13 = Option::::Some(move _12);
-        StorageDead(_12);
         StorageDead(_10);
+        _12 = Option::::Some(move _11);
+        StorageDead(_11);
         StorageDead(_9);
-        StorageDead(_7);
         StorageDead(_8);
-        _14 = move ((_13 as Some).0: U);
-        _15 = opaque::(move _14) -> [return: bb8, unwind: bb10];
+        StorageDead(_6);
+        StorageDead(_7);
+        _13 = move ((_12 as Some).0: U);
+        _14 = opaque::(move _13) -> [return: bb8, unwind: bb10];
     }
 
     bb8: {
-        StorageDead(_13);
+        StorageDead(_12);
         goto -> bb2;
     }
 
diff --git a/tests/mir-opt/pre-codegen/loops.rs b/tests/mir-opt/pre-codegen/loops.rs
index d0b8cc8db7a9..952dd8cac60f 100644
--- a/tests/mir-opt/pre-codegen/loops.rs
+++ b/tests/mir-opt/pre-codegen/loops.rs
@@ -1,5 +1,6 @@
 // skip-filecheck
 //@ compile-flags: -O -Zmir-opt-level=2 -g
+//@ ignore-std-debug-assertions (debug assertions result in different inlines)
 //@ needs-unwind
 
 #![crate_type = "lib"]
diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
index e537dd6a28ef..4260ec3eaedf 100644
--- a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir
@@ -3,72 +3,316 @@
 fn vec_move(_1: Vec) -> () {
     debug v => _1;
     let mut _0: ();
-    let mut _2: std::vec::IntoIter;
-    let mut _3: std::vec::IntoIter;
-    let mut _4: &mut std::vec::IntoIter;
-    let mut _5: std::option::Option;
-    let mut _6: isize;
-    let _8: ();
+    let mut _21: std::vec::IntoIter;
+    let mut _22: std::vec::IntoIter;
+    let mut _23: &mut std::vec::IntoIter;
+    let mut _24: std::option::Option;
+    let mut _25: isize;
+    let _27: ();
     scope 1 {
-        debug iter => _3;
-        let _7: impl Sized;
+        debug iter => _22;
+        let _26: impl Sized;
         scope 2 {
-            debug x => _7;
+            debug x => _26;
+        }
+    }
+    scope 3 (inlined  as IntoIterator>::into_iter) {
+        debug self => _1;
+        let _2: std::mem::ManuallyDrop>;
+        let mut _3: *const std::alloc::Global;
+        let mut _8: usize;
+        let mut _10: *mut impl Sized;
+        let mut _11: *const impl Sized;
+        let mut _12: usize;
+        let _28: &std::vec::Vec;
+        let mut _29: &std::mem::ManuallyDrop>;
+        let mut _30: &alloc::raw_vec::RawVec;
+        let mut _31: &std::mem::ManuallyDrop>;
+        let _32: &std::vec::Vec;
+        let mut _33: &std::mem::ManuallyDrop>;
+        let _34: &std::vec::Vec;
+        let mut _35: &std::mem::ManuallyDrop>;
+        let mut _36: &alloc::raw_vec::RawVec;
+        let mut _37: &std::mem::ManuallyDrop>;
+        scope 4 {
+            debug me => _2;
+            scope 5 {
+                debug alloc => const ManuallyDrop:: {{ value: std::alloc::Global }};
+                let _6: std::ptr::NonNull;
+                scope 6 {
+                    debug buf => _6;
+                    let _7: *mut impl Sized;
+                    scope 7 {
+                        debug begin => _7;
+                        scope 8 {
+                            debug end => _11;
+                            let _19: usize;
+                            scope 9 {
+                                debug cap => _19;
+                            }
+                            scope 39 (inlined > as Deref>::deref) {
+                                debug self => _37;
+                            }
+                            scope 40 (inlined alloc::raw_vec::RawVec::::capacity) {
+                                debug self => _36;
+                                let mut _38: &alloc::raw_vec::RawVecInner;
+                                scope 41 (inlined std::mem::size_of::) {
+                                }
+                                scope 42 (inlined alloc::raw_vec::RawVecInner::capacity) {
+                                    debug self => _38;
+                                    debug elem_size => const ::SIZE;
+                                    let mut _20: core::num::niche_types::UsizeNoHighBit;
+                                    scope 43 (inlined core::num::niche_types::UsizeNoHighBit::as_inner) {
+                                        debug self => _20;
+                                    }
+                                }
+                            }
+                        }
+                        scope 25 (inlined > as Deref>::deref) {
+                            debug self => _33;
+                        }
+                        scope 26 (inlined Vec::::len) {
+                            debug self => _32;
+                            let mut _13: bool;
+                            scope 27 {
+                            }
+                        }
+                        scope 28 (inlined std::ptr::mut_ptr::::wrapping_byte_add) {
+                            debug self => _7;
+                            debug count => _12;
+                            let mut _14: *mut u8;
+                            let mut _18: *mut u8;
+                            scope 29 (inlined std::ptr::mut_ptr::::cast::) {
+                                debug self => _7;
+                            }
+                            scope 30 (inlined std::ptr::mut_ptr::::wrapping_add) {
+                                debug self => _14;
+                                debug count => _12;
+                                let mut _15: isize;
+                                scope 31 (inlined std::ptr::mut_ptr::::wrapping_offset) {
+                                    debug self => _14;
+                                    debug count => _15;
+                                    let mut _16: *const u8;
+                                    let mut _17: *const u8;
+                                }
+                            }
+                            scope 32 (inlined std::ptr::mut_ptr::::with_metadata_of::) {
+                                debug self => _18;
+                                debug meta => _5;
+                                scope 33 (inlined std::ptr::metadata::) {
+                                    debug ptr => _5;
+                                }
+                                scope 34 (inlined std::ptr::from_raw_parts_mut::) {
+                                }
+                            }
+                        }
+                        scope 35 (inlined > as Deref>::deref) {
+                            debug self => _35;
+                        }
+                        scope 36 (inlined Vec::::len) {
+                            debug self => _34;
+                            let mut _9: bool;
+                            scope 37 {
+                            }
+                        }
+                        scope 38 (inlined #[track_caller] std::ptr::mut_ptr::::add) {
+                            debug self => _7;
+                            debug count => _8;
+                        }
+                    }
+                    scope 24 (inlined NonNull::::as_ptr) {
+                        debug self => _6;
+                    }
+                }
+                scope 17 (inlined > as Deref>::deref) {
+                    debug self => _31;
+                }
+                scope 18 (inlined alloc::raw_vec::RawVec::::non_null) {
+                    debug self => _30;
+                    scope 19 (inlined alloc::raw_vec::RawVecInner::non_null::) {
+                        let mut _4: std::ptr::NonNull;
+                        scope 20 (inlined Unique::::cast::) {
+                            scope 21 (inlined NonNull::::cast::) {
+                                let mut _5: *const impl Sized;
+                                scope 22 (inlined NonNull::::as_ptr) {
+                                }
+                            }
+                        }
+                        scope 23 (inlined Unique::::as_non_null_ptr) {
+                        }
+                    }
+                }
+            }
+            scope 11 (inlined > as Deref>::deref) {
+                debug self => _29;
+            }
+            scope 12 (inlined Vec::::allocator) {
+                debug self => _28;
+                scope 13 (inlined alloc::raw_vec::RawVec::::allocator) {
+                    scope 14 (inlined alloc::raw_vec::RawVecInner::allocator) {
+                    }
+                }
+            }
+            scope 15 (inlined #[track_caller] std::ptr::read::) {
+                debug src => _3;
+            }
+            scope 16 (inlined ManuallyDrop::::new) {
+                debug value => const std::alloc::Global;
+            }
+        }
+        scope 10 (inlined ManuallyDrop::>::new) {
+            debug value => _1;
         }
     }
 
     bb0: {
+        StorageLive(_21);
+        StorageLive(_6);
+        StorageLive(_7);
+        StorageLive(_11);
+        StorageLive(_19);
+        StorageLive(_5);
+        StorageLive(_4);
+        StorageLive(_17);
         StorageLive(_2);
-        _2 =  as IntoIterator>::into_iter(move _1) -> [return: bb1, unwind continue];
+        _2 = ManuallyDrop::> { value: copy _1 };
+        StorageLive(_3);
+        // DBG: _29 = &_2;
+        // DBG: _28 = &(_2.0: std::vec::Vec);
+        _3 = &raw const ((((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).2: std::alloc::Global);
+        StorageDead(_3);
+        // DBG: _31 = &_2;
+        // DBG: _30 = &((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec);
+        _4 = copy (((((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).0: std::ptr::Unique).0: std::ptr::NonNull);
+        _5 = copy _4 as *const impl Sized (Transmute);
+        _6 = NonNull:: { pointer: copy _5 };
+        _7 = copy _4 as *mut impl Sized (Transmute);
+        switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
     bb1: {
-        StorageLive(_3);
-        _3 = move _2;
-        goto -> bb2;
+        StorageLive(_10);
+        StorageLive(_8);
+        // DBG: _35 = &_2;
+        // DBG: _34 = &(_2.0: std::vec::Vec);
+        _8 = copy ((_2.0: std::vec::Vec).1: usize);
+        StorageLive(_9);
+        _9 = Le(copy _8, const ::MAX_SLICE_LEN);
+        assume(move _9);
+        StorageDead(_9);
+        _10 = Offset(copy _7, copy _8);
+        _11 = copy _10 as *const impl Sized (PtrToPtr);
+        StorageDead(_8);
+        StorageDead(_10);
+        goto -> bb4;
     }
 
     bb2: {
-        StorageLive(_5);
-        _4 = &mut _3;
-        _5 =  as Iterator>::next(move _4) -> [return: bb3, unwind: bb9];
+        StorageLive(_12);
+        // DBG: _33 = &_2;
+        // DBG: _32 = &(_2.0: std::vec::Vec);
+        _12 = copy ((_2.0: std::vec::Vec).1: usize);
+        StorageLive(_13);
+        _13 = Le(copy _12, const ::MAX_SLICE_LEN);
+        assume(move _13);
+        StorageDead(_13);
+        StorageLive(_18);
+        StorageLive(_14);
+        _14 = copy _4 as *mut u8 (Transmute);
+        StorageLive(_15);
+        _15 = copy _12 as isize (IntToInt);
+        StorageLive(_16);
+        _16 = copy _4 as *const u8 (Transmute);
+        _17 = arith_offset::(move _16, move _15) -> [return: bb3, unwind unreachable];
     }
 
     bb3: {
-        _6 = discriminant(_5);
-        switchInt(move _6) -> [0: bb4, 1: bb6, otherwise: bb8];
+        StorageDead(_16);
+        _18 = copy _17 as *mut u8 (PtrToPtr);
+        StorageDead(_15);
+        StorageDead(_14);
+        StorageDead(_18);
+        StorageDead(_12);
+        _11 = copy _17 as *const impl Sized (PtrToPtr);
+        goto -> bb4;
     }
 
     bb4: {
-        StorageDead(_5);
-        drop(_3) -> [return: bb5, unwind continue];
+        // DBG: _37 = &_2;
+        // DBG: _36 = &((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec);
+        // DBG: _38 = &(((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner);
+        switchInt(const ::SIZE) -> [0: bb5, otherwise: bb6];
     }
 
     bb5: {
-        StorageDead(_3);
-        StorageDead(_2);
-        return;
+        _19 = const usize::MAX;
+        goto -> bb7;
     }
 
     bb6: {
-        _7 = move ((_5 as Some).0: impl Sized);
-        _8 = opaque::(move _7) -> [return: bb7, unwind: bb9];
+        StorageLive(_20);
+        _20 = copy ((((_2.0: std::vec::Vec).0: alloc::raw_vec::RawVec).0: alloc::raw_vec::RawVecInner).1: core::num::niche_types::UsizeNoHighBit);
+        _19 = copy _20 as usize (Transmute);
+        StorageDead(_20);
+        goto -> bb7;
     }
 
     bb7: {
+        _21 = std::vec::IntoIter:: { buf: copy _6, phantom: const ZeroSized: PhantomData, cap: move _19, alloc: const ManuallyDrop:: {{ value: std::alloc::Global }}, ptr: copy _6, end: copy _11 };
+        StorageDead(_2);
+        StorageDead(_17);
+        StorageDead(_4);
         StorageDead(_5);
-        goto -> bb2;
+        StorageDead(_19);
+        StorageDead(_11);
+        StorageDead(_7);
+        StorageDead(_6);
+        StorageLive(_22);
+        _22 = move _21;
+        goto -> bb8;
     }
 
     bb8: {
+        StorageLive(_24);
+        _23 = &mut _22;
+        _24 =  as Iterator>::next(move _23) -> [return: bb9, unwind: bb15];
+    }
+
+    bb9: {
+        _25 = discriminant(_24);
+        switchInt(move _25) -> [0: bb10, 1: bb12, otherwise: bb14];
+    }
+
+    bb10: {
+        StorageDead(_24);
+        drop(_22) -> [return: bb11, unwind continue];
+    }
+
+    bb11: {
+        StorageDead(_22);
+        StorageDead(_21);
+        return;
+    }
+
+    bb12: {
+        _26 = move ((_24 as Some).0: impl Sized);
+        _27 = opaque::(move _26) -> [return: bb13, unwind: bb15];
+    }
+
+    bb13: {
+        StorageDead(_24);
+        goto -> bb8;
+    }
+
+    bb14: {
         unreachable;
     }
 
-    bb9 (cleanup): {
-        drop(_3) -> [return: bb10, unwind terminate(cleanup)];
+    bb15 (cleanup): {
+        drop(_22) -> [return: bb16, unwind terminate(cleanup)];
     }
 
-    bb10 (cleanup): {
+    bb16 (cleanup): {
         resume;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-abort.mir
new file mode 100644
index 000000000000..b29662dfa613
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,37 @@
+// MIR for `option_direct` after PreCodegen
+
+fn option_direct(_1: Option) -> Option {
+    debug x => _1;
+    let mut _0: std::option::Option;
+    let mut _2: isize;
+    let _3: u32;
+    let mut _4: u32;
+    scope 1 {
+        debug x => _3;
+    }
+
+    bb0: {
+        _2 = discriminant(_1);
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
+    }
+
+    bb1: {
+        _0 = Option::::None;
+        goto -> bb3;
+    }
+
+    bb2: {
+        _3 = copy ((_1 as Some).0: u32);
+        _4 = Not(copy _3);
+        _0 = Option::::Some(move _4);
+        goto -> bb3;
+    }
+
+    bb3: {
+        return;
+    }
+
+    bb4: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-unwind.mir
new file mode 100644
index 000000000000..b29662dfa613
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/option_bubble_debug.option_direct.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,37 @@
+// MIR for `option_direct` after PreCodegen
+
+fn option_direct(_1: Option) -> Option {
+    debug x => _1;
+    let mut _0: std::option::Option;
+    let mut _2: isize;
+    let _3: u32;
+    let mut _4: u32;
+    scope 1 {
+        debug x => _3;
+    }
+
+    bb0: {
+        _2 = discriminant(_1);
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
+    }
+
+    bb1: {
+        _0 = Option::::None;
+        goto -> bb3;
+    }
+
+    bb2: {
+        _3 = copy ((_1 as Some).0: u32);
+        _4 = Not(copy _3);
+        _0 = Option::::Some(move _4);
+        goto -> bb3;
+    }
+
+    bb3: {
+        return;
+    }
+
+    bb4: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-abort.mir
new file mode 100644
index 000000000000..5b401064dd0e
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,47 @@
+// MIR for `option_traits` after PreCodegen
+
+fn option_traits(_1: Option) -> Option {
+    debug x => _1;
+    let mut _0: std::option::Option;
+    let mut _2: std::ops::ControlFlow, u32>;
+    let mut _3: isize;
+    let _4: u32;
+    let mut _5: u32;
+    scope 1 {
+        debug residual => const Option::::None;
+        scope 2 {
+        }
+    }
+    scope 3 {
+        debug val => _4;
+        scope 4 {
+        }
+    }
+
+    bb0: {
+        _2 =  as Try>::branch(copy _1) -> [return: bb1, unwind unreachable];
+    }
+
+    bb1: {
+        _3 = discriminant(_2);
+        switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb5];
+    }
+
+    bb2: {
+        _4 = copy ((_2 as Continue).0: u32);
+        _5 = Not(copy _4);
+        _0 =  as Try>::from_output(move _5) -> [return: bb4, unwind unreachable];
+    }
+
+    bb3: {
+        _0 =  as FromResidual>>::from_residual(const Option::::None) -> [return: bb4, unwind unreachable];
+    }
+
+    bb4: {
+        return;
+    }
+
+    bb5: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-unwind.mir
new file mode 100644
index 000000000000..bda9e9d8e604
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/option_bubble_debug.option_traits.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,47 @@
+// MIR for `option_traits` after PreCodegen
+
+fn option_traits(_1: Option) -> Option {
+    debug x => _1;
+    let mut _0: std::option::Option;
+    let mut _2: std::ops::ControlFlow, u32>;
+    let mut _3: isize;
+    let _4: u32;
+    let mut _5: u32;
+    scope 1 {
+        debug residual => const Option::::None;
+        scope 2 {
+        }
+    }
+    scope 3 {
+        debug val => _4;
+        scope 4 {
+        }
+    }
+
+    bb0: {
+        _2 =  as Try>::branch(copy _1) -> [return: bb1, unwind continue];
+    }
+
+    bb1: {
+        _3 = discriminant(_2);
+        switchInt(move _3) -> [0: bb2, 1: bb3, otherwise: bb5];
+    }
+
+    bb2: {
+        _4 = copy ((_2 as Continue).0: u32);
+        _5 = Not(copy _4);
+        _0 =  as Try>::from_output(move _5) -> [return: bb4, unwind continue];
+    }
+
+    bb3: {
+        _0 =  as FromResidual>>::from_residual(const Option::::None) -> [return: bb4, unwind continue];
+    }
+
+    bb4: {
+        return;
+    }
+
+    bb5: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/pre-codegen/option_bubble_debug.rs b/tests/mir-opt/pre-codegen/option_bubble_debug.rs
new file mode 100644
index 000000000000..b9bf78a1d6ec
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/option_bubble_debug.rs
@@ -0,0 +1,29 @@
+//@ compile-flags: -Copt-level=0 -Zmir-opt-level=1 -Cdebuginfo=limited
+//@ edition: 2024
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
+#![crate_type = "lib"]
+#![feature(try_blocks)]
+
+// EMIT_MIR option_bubble_debug.option_direct.PreCodegen.after.mir
+pub fn option_direct(x: Option) -> Option {
+    // CHECK-LABEL: fn option_direct(_1: Option) -> Option
+    // CHECK: = discriminant(_1);
+    // CHECK: [[TEMP:_.+]] = Not({{.+}});
+    // CHECK: _0 = Option::::Some(move [[TEMP]]);
+
+    match x {
+        Some(x) => Some(!x),
+        None => None,
+    }
+}
+
+// EMIT_MIR option_bubble_debug.option_traits.PreCodegen.after.mir
+pub fn option_traits(x: Option) -> Option {
+    // CHECK-LABEL: fn option_traits(_1: Option) -> Option
+    // CHECK: =  as Try>::branch(copy _1)
+    // CHECK: [[TEMP:_.+]] = Not({{.+}});
+    // CHECK: _0 =  as Try>::from_output(move [[TEMP]])
+
+    try { !(x?) }
+}
diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir
index dfe618612ab9..03a52b82b49b 100644
--- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-abort.mir
@@ -5,23 +5,21 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     debug end => _2;
     debug f => _3;
     let mut _0: ();
-    let mut _4: u32;
-    let mut _9: std::option::Option;
-    let mut _11: &impl Fn(u32);
-    let mut _12: (u32,);
-    let _13: ();
+    let mut _7: std::option::Option;
+    let mut _9: &impl Fn(u32);
+    let mut _10: (u32,);
+    let _11: ();
     scope 1 {
-        debug ((iter: std::ops::Range).0: u32) => _4;
+        debug ((iter: std::ops::Range).0: u32) => _1;
         debug ((iter: std::ops::Range).1: u32) => _2;
-        let _10: u32;
+        let _8: u32;
         scope 2 {
-            debug x => _10;
+            debug x => _8;
         }
         scope 4 (inlined iter::range::>::next) {
             scope 5 (inlined  as iter::range::RangeIteratorImpl>::spec_next) {
-                let mut _6: bool;
-                let _7: u32;
-                let mut _8: u32;
+                let mut _5: bool;
+                let _6: u32;
                 scope 6 {
                     scope 8 (inlined ::forward_unchecked) {
                         scope 9 (inlined #[track_caller] core::num::::unchecked_add) {
@@ -33,7 +31,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
                     }
                 }
                 scope 7 (inlined std::cmp::impls::::lt) {
-                    let mut _5: u32;
+                    let mut _4: u32;
                 }
             }
         }
@@ -42,25 +40,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     }
 
     bb0: {
-        StorageLive(_4);
-        _4 = copy _1;
         goto -> bb1;
     }
 
     bb1: {
-        StorageLive(_9);
-        StorageLive(_6);
+        StorageLive(_7);
         StorageLive(_5);
-        _5 = copy _4;
-        _6 = Lt(move _5, copy _2);
-        StorageDead(_5);
-        switchInt(move _6) -> [0: bb2, otherwise: bb4];
+        StorageLive(_4);
+        _4 = copy _1;
+        _5 = Lt(move _4, copy _2);
+        StorageDead(_4);
+        switchInt(move _5) -> [0: bb2, otherwise: bb4];
     }
 
     bb2: {
-        StorageDead(_6);
-        StorageDead(_9);
-        StorageDead(_4);
+        StorageDead(_5);
+        StorageDead(_7);
         drop(_3) -> [return: bb3, unwind unreachable];
     }
 
@@ -69,25 +64,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     }
 
     bb4: {
-        _7 = copy _4;
-        StorageLive(_8);
-        _8 = AddUnchecked(copy _7, const 1_u32);
-        _4 = move _8;
-        StorageDead(_8);
-        _9 = Option::::Some(copy _7);
-        StorageDead(_6);
-        _10 = copy ((_9 as Some).0: u32);
-        StorageLive(_11);
-        _11 = &_3;
-        StorageLive(_12);
-        _12 = (copy _10,);
-        _13 = >::call(move _11, move _12) -> [return: bb5, unwind unreachable];
+        _6 = copy _1;
+        _1 = AddUnchecked(copy _6, const 1_u32);
+        _7 = Option::::Some(copy _6);
+        StorageDead(_5);
+        _8 = copy ((_7 as Some).0: u32);
+        StorageLive(_9);
+        _9 = &_3;
+        StorageLive(_10);
+        _10 = (copy _8,);
+        _11 = >::call(move _9, move _10) -> [return: bb5, unwind unreachable];
     }
 
     bb5: {
-        StorageDead(_12);
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
+        StorageDead(_7);
         goto -> bb1;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir
index e0fcfcaffc59..3b09f33e7333 100644
--- a/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/range_iter.forward_loop.PreCodegen.after.panic-unwind.mir
@@ -5,23 +5,21 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     debug end => _2;
     debug f => _3;
     let mut _0: ();
-    let mut _4: u32;
-    let mut _9: std::option::Option;
-    let mut _11: &impl Fn(u32);
-    let mut _12: (u32,);
-    let _13: ();
+    let mut _7: std::option::Option;
+    let mut _9: &impl Fn(u32);
+    let mut _10: (u32,);
+    let _11: ();
     scope 1 {
-        debug ((iter: std::ops::Range).0: u32) => _4;
+        debug ((iter: std::ops::Range).0: u32) => _1;
         debug ((iter: std::ops::Range).1: u32) => _2;
-        let _10: u32;
+        let _8: u32;
         scope 2 {
-            debug x => _10;
+            debug x => _8;
         }
         scope 4 (inlined iter::range::>::next) {
             scope 5 (inlined  as iter::range::RangeIteratorImpl>::spec_next) {
-                let mut _6: bool;
-                let _7: u32;
-                let mut _8: u32;
+                let mut _5: bool;
+                let _6: u32;
                 scope 6 {
                     scope 8 (inlined ::forward_unchecked) {
                         scope 9 (inlined #[track_caller] core::num::::unchecked_add) {
@@ -33,7 +31,7 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
                     }
                 }
                 scope 7 (inlined std::cmp::impls::::lt) {
-                    let mut _5: u32;
+                    let mut _4: u32;
                 }
             }
         }
@@ -42,25 +40,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     }
 
     bb0: {
-        StorageLive(_4);
-        _4 = copy _1;
         goto -> bb1;
     }
 
     bb1: {
-        StorageLive(_9);
-        StorageLive(_6);
+        StorageLive(_7);
         StorageLive(_5);
-        _5 = copy _4;
-        _6 = Lt(move _5, copy _2);
-        StorageDead(_5);
-        switchInt(move _6) -> [0: bb2, otherwise: bb4];
+        StorageLive(_4);
+        _4 = copy _1;
+        _5 = Lt(move _4, copy _2);
+        StorageDead(_4);
+        switchInt(move _5) -> [0: bb2, otherwise: bb4];
     }
 
     bb2: {
-        StorageDead(_6);
-        StorageDead(_9);
-        StorageDead(_4);
+        StorageDead(_5);
+        StorageDead(_7);
         drop(_3) -> [return: bb3, unwind continue];
     }
 
@@ -69,25 +64,22 @@ fn forward_loop(_1: u32, _2: u32, _3: impl Fn(u32)) -> () {
     }
 
     bb4: {
-        _7 = copy _4;
-        StorageLive(_8);
-        _8 = AddUnchecked(copy _7, const 1_u32);
-        _4 = move _8;
-        StorageDead(_8);
-        _9 = Option::::Some(copy _7);
-        StorageDead(_6);
-        _10 = copy ((_9 as Some).0: u32);
-        StorageLive(_11);
-        _11 = &_3;
-        StorageLive(_12);
-        _12 = (copy _10,);
-        _13 = >::call(move _11, move _12) -> [return: bb5, unwind: bb6];
+        _6 = copy _1;
+        _1 = AddUnchecked(copy _6, const 1_u32);
+        _7 = Option::::Some(copy _6);
+        StorageDead(_5);
+        _8 = copy ((_7 as Some).0: u32);
+        StorageLive(_9);
+        _9 = &_3;
+        StorageLive(_10);
+        _10 = (copy _8,);
+        _11 = >::call(move _9, move _10) -> [return: bb5, unwind: bb6];
     }
 
     bb5: {
-        StorageDead(_12);
-        StorageDead(_11);
+        StorageDead(_10);
         StorageDead(_9);
+        StorageDead(_7);
         goto -> bb1;
     }
 
diff --git a/tests/mir-opt/pre-codegen/slice_filter.variant_a-{closure#0}.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/slice_filter.variant_a-{closure#0}.PreCodegen.after.mir
index cbdd194afd3a..2cab88182962 100644
--- a/tests/mir-opt/pre-codegen/slice_filter.variant_a-{closure#0}.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/slice_filter.variant_a-{closure#0}.PreCodegen.after.mir
@@ -3,183 +3,134 @@
 fn variant_a::{closure#0}(_1: &mut {closure@$DIR/slice_filter.rs:7:25: 7:39}, _2: &&(usize, usize, usize, usize)) -> bool {
     let mut _0: bool;
     let mut _3: &(usize, usize, usize, usize);
-    let _4: &usize;
-    let _5: &usize;
-    let _6: &usize;
-    let _7: &usize;
-    let mut _8: &&usize;
-    let _9: &usize;
-    let mut _10: &&usize;
-    let mut _13: bool;
-    let mut _14: &&usize;
+    let mut _6: bool;
+    let mut _9: bool;
+    let mut _10: bool;
+    let _13: &usize;
+    let _14: &usize;
     let _15: &usize;
-    let mut _16: &&usize;
-    let mut _19: bool;
+    let _16: &usize;
+    let mut _17: &&usize;
+    let mut _18: &&usize;
+    let mut _19: &&usize;
     let mut _20: &&usize;
-    let _21: &usize;
+    let mut _21: &&usize;
     let mut _22: &&usize;
-    let mut _23: bool;
+    let mut _23: &&usize;
     let mut _24: &&usize;
-    let _25: &usize;
-    let mut _26: &&usize;
     scope 1 {
-        debug a => _4;
-        debug b => _5;
-        debug c => _6;
-        debug d => _7;
+        debug a => _13;
+        debug b => _14;
+        debug c => _15;
+        debug d => _16;
         scope 2 (inlined std::cmp::impls::::le) {
-            debug self => _8;
-            debug other => _10;
+            debug self => _17;
+            debug other => _18;
             scope 3 (inlined std::cmp::impls::::le) {
-                debug self => _4;
-                debug other => _6;
-                let mut _11: usize;
-                let mut _12: usize;
+                debug self => _13;
+                debug other => _15;
+                let mut _4: usize;
+                let mut _5: usize;
             }
         }
         scope 4 (inlined std::cmp::impls::::le) {
-            debug self => _14;
-            debug other => _16;
+            debug self => _19;
+            debug other => _20;
             scope 5 (inlined std::cmp::impls::::le) {
-                debug self => _7;
-                debug other => _5;
-                let mut _17: usize;
-                let mut _18: usize;
+                debug self => _16;
+                debug other => _14;
+                let mut _7: usize;
+                let mut _8: usize;
             }
         }
         scope 6 (inlined std::cmp::impls::::le) {
-            debug self => _20;
+            debug self => _21;
             debug other => _22;
             scope 7 (inlined std::cmp::impls::::le) {
-                debug self => _6;
-                debug other => _4;
+                debug self => _15;
+                debug other => _13;
             }
         }
         scope 8 (inlined std::cmp::impls::::le) {
-            debug self => _24;
-            debug other => _26;
+            debug self => _23;
+            debug other => _24;
             scope 9 (inlined std::cmp::impls::::le) {
-                debug self => _5;
-                debug other => _7;
-                let mut _27: usize;
-                let mut _28: usize;
+                debug self => _14;
+                debug other => _16;
+                let mut _11: usize;
+                let mut _12: usize;
             }
         }
     }
 
     bb0: {
         _3 = copy (*_2);
-        _4 = &((*_3).0: usize);
-        _5 = &((*_3).1: usize);
-        _6 = &((*_3).2: usize);
-        _7 = &((*_3).3: usize);
-        StorageLive(_13);
-        StorageLive(_8);
-        _8 = &_4;
-        StorageLive(_10);
-        StorageLive(_9);
-        _9 = copy _6;
-        _10 = &_9;
-        _11 = copy ((*_3).0: usize);
-        _12 = copy ((*_3).2: usize);
-        _13 = Le(copy _11, copy _12);
-        switchInt(move _13) -> [0: bb1, otherwise: bb2];
+        // DBG: _13 = &((*_3).0: usize);
+        // DBG: _14 = &((*_3).1: usize);
+        // DBG: _15 = &((*_3).2: usize);
+        // DBG: _16 = &((*_3).3: usize);
+        StorageLive(_6);
+        // DBG: _17 = &_13;
+        // DBG: _18 = &?;
+        _4 = copy ((*_3).0: usize);
+        _5 = copy ((*_3).2: usize);
+        _6 = Le(copy _4, copy _5);
+        switchInt(move _6) -> [0: bb2, otherwise: bb1];
     }
 
     bb1: {
-        StorageDead(_9);
-        StorageDead(_10);
+        StorageLive(_9);
+        // DBG: _19 = &_16;
+        // DBG: _20 = &?;
+        StorageLive(_7);
+        _7 = copy ((*_3).3: usize);
+        StorageLive(_8);
+        _8 = copy ((*_3).1: usize);
+        _9 = Le(move _7, move _8);
         StorageDead(_8);
-        goto -> bb4;
+        StorageDead(_7);
+        switchInt(move _9) -> [0: bb2, otherwise: bb6];
     }
 
     bb2: {
-        StorageDead(_9);
-        StorageDead(_10);
-        StorageDead(_8);
-        StorageLive(_19);
-        StorageLive(_14);
-        _14 = &_7;
-        StorageLive(_16);
-        StorageLive(_15);
-        _15 = copy _5;
-        _16 = &_15;
-        StorageLive(_17);
-        _17 = copy ((*_3).3: usize);
-        StorageLive(_18);
-        _18 = copy ((*_3).1: usize);
-        _19 = Le(move _17, move _18);
-        StorageDead(_18);
-        StorageDead(_17);
-        switchInt(move _19) -> [0: bb3, otherwise: bb8];
+        StorageLive(_10);
+        // DBG: _21 = &_15;
+        // DBG: _22 = &?;
+        _10 = Le(copy _5, copy _4);
+        switchInt(move _10) -> [0: bb3, otherwise: bb4];
     }
 
     bb3: {
-        StorageDead(_15);
-        StorageDead(_16);
-        StorageDead(_14);
-        goto -> bb4;
+        _0 = const false;
+        goto -> bb5;
     }
 
     bb4: {
-        StorageLive(_23);
-        StorageLive(_20);
-        _20 = &_6;
-        StorageLive(_22);
-        StorageLive(_21);
-        _21 = copy _4;
-        _22 = &_21;
-        _23 = Le(copy _12, copy _11);
-        switchInt(move _23) -> [0: bb5, otherwise: bb6];
+        // DBG: _23 = &_14;
+        // DBG: _24 = &?;
+        StorageLive(_11);
+        _11 = copy ((*_3).1: usize);
+        StorageLive(_12);
+        _12 = copy ((*_3).3: usize);
+        _0 = Le(move _11, move _12);
+        StorageDead(_12);
+        StorageDead(_11);
+        goto -> bb5;
     }
 
     bb5: {
-        StorageDead(_21);
-        StorageDead(_22);
-        StorageDead(_20);
-        _0 = const false;
+        StorageDead(_10);
         goto -> bb7;
     }
 
     bb6: {
-        StorageDead(_21);
-        StorageDead(_22);
-        StorageDead(_20);
-        StorageLive(_24);
-        _24 = &_5;
-        StorageLive(_26);
-        StorageLive(_25);
-        _25 = copy _7;
-        _26 = &_25;
-        StorageLive(_27);
-        _27 = copy ((*_3).1: usize);
-        StorageLive(_28);
-        _28 = copy ((*_3).3: usize);
-        _0 = Le(move _27, move _28);
-        StorageDead(_28);
-        StorageDead(_27);
-        StorageDead(_25);
-        StorageDead(_26);
-        StorageDead(_24);
+        _0 = const true;
         goto -> bb7;
     }
 
     bb7: {
-        StorageDead(_23);
-        goto -> bb9;
-    }
-
-    bb8: {
-        StorageDead(_15);
-        StorageDead(_16);
-        StorageDead(_14);
-        _0 = const true;
-        goto -> bb9;
-    }
-
-    bb9: {
-        StorageDead(_19);
-        StorageDead(_13);
+        StorageDead(_9);
+        StorageDead(_6);
         return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-abort.mir
index d1b1e3d7dd73..ebe8b23354be 100644
--- a/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-abort.mir
@@ -30,7 +30,7 @@ fn slice_get_mut_usize(_1: &mut [u32], _2: usize) -> Option<&mut u32> {
         StorageDead(_3);
         StorageLive(_5);
         _5 = &mut (*_1)[_2];
-        _0 = Option::<&mut u32>::Some(move _5);
+        _0 = Option::<&mut u32>::Some(copy _5);
         StorageDead(_5);
         goto -> bb3;
     }
diff --git a/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-unwind.mir
index d1b1e3d7dd73..ebe8b23354be 100644
--- a/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.panic-unwind.mir
@@ -30,7 +30,7 @@ fn slice_get_mut_usize(_1: &mut [u32], _2: usize) -> Option<&mut u32> {
         StorageDead(_3);
         StorageLive(_5);
         _5 = &mut (*_1)[_2];
-        _0 = Option::<&mut u32>::Some(move _5);
+        _0 = Option::<&mut u32>::Some(copy _5);
         StorageDead(_5);
         goto -> bb3;
     }
diff --git a/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-abort.mir
index 731f6438a6ec..2df2c4b85b8f 100644
--- a/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-abort.mir
@@ -4,14 +4,81 @@ fn slice_index_range(_1: &[u32], _2: std::ops::Range) -> &[u32] {
     debug slice => _1;
     debug index => _2;
     let mut _0: &[u32];
+    let mut _3: usize;
+    let mut _4: usize;
     scope 1 (inlined #[track_caller] core::slice::index::> for [u32]>::index) {
+        scope 2 (inlined #[track_caller]  as SliceIndex<[u32]>>::index) {
+            let mut _7: usize;
+            let mut _8: bool;
+            let mut _9: *const [u32];
+            let _12: *const [u32];
+            let mut _13: usize;
+            let mut _14: !;
+            scope 3 (inlined core::num::::checked_sub) {
+                let mut _5: bool;
+                let mut _6: usize;
+            }
+            scope 4 (inlined core::slice::index::get_offset_len_noubcheck::) {
+                let _10: *const u32;
+                scope 5 {
+                    let _11: *const u32;
+                    scope 6 {
+                    }
+                }
+            }
+        }
     }
 
     bb0: {
-        _0 =  as SliceIndex<[u32]>>::index(move _2, move _1) -> [return: bb1, unwind unreachable];
+        _3 = move (_2.0: usize);
+        _4 = move (_2.1: usize);
+        StorageLive(_5);
+        _5 = Lt(copy _4, copy _3);
+        switchInt(move _5) -> [0: bb1, otherwise: bb4];
     }
 
     bb1: {
+        _6 = SubUnchecked(copy _4, copy _3);
+        StorageDead(_5);
+        StorageLive(_8);
+        StorageLive(_7);
+        _7 = PtrMetadata(copy _1);
+        _8 = Le(copy _4, move _7);
+        switchInt(move _8) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        StorageDead(_7);
+        goto -> bb5;
+    }
+
+    bb3: {
+        StorageDead(_7);
+        StorageLive(_12);
+        StorageLive(_9);
+        _9 = &raw const (*_1);
+        StorageLive(_10);
+        StorageLive(_11);
+        _10 = copy _9 as *const u32 (PtrToPtr);
+        _11 = Offset(copy _10, copy _3);
+        _12 = *const [u32] from (copy _11, copy _6);
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageDead(_9);
+        _0 = &(*_12);
+        StorageDead(_12);
+        StorageDead(_8);
         return;
     }
+
+    bb4: {
+        StorageDead(_5);
+        goto -> bb5;
+    }
+
+    bb5: {
+        StorageLive(_13);
+        _13 = PtrMetadata(copy _1);
+        _14 = core::slice::index::slice_index_fail(move _3, move _4, move _13) -> unwind unreachable;
+    }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-unwind.mir
index d879d06bb4e4..d4b86b9633ac 100644
--- a/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_index.slice_index_range.PreCodegen.after.panic-unwind.mir
@@ -4,14 +4,81 @@ fn slice_index_range(_1: &[u32], _2: std::ops::Range) -> &[u32] {
     debug slice => _1;
     debug index => _2;
     let mut _0: &[u32];
+    let mut _3: usize;
+    let mut _4: usize;
     scope 1 (inlined #[track_caller] core::slice::index::> for [u32]>::index) {
+        scope 2 (inlined #[track_caller]  as SliceIndex<[u32]>>::index) {
+            let mut _7: usize;
+            let mut _8: bool;
+            let mut _9: *const [u32];
+            let _12: *const [u32];
+            let mut _13: usize;
+            let mut _14: !;
+            scope 3 (inlined core::num::::checked_sub) {
+                let mut _5: bool;
+                let mut _6: usize;
+            }
+            scope 4 (inlined core::slice::index::get_offset_len_noubcheck::) {
+                let _10: *const u32;
+                scope 5 {
+                    let _11: *const u32;
+                    scope 6 {
+                    }
+                }
+            }
+        }
     }
 
     bb0: {
-        _0 =  as SliceIndex<[u32]>>::index(move _2, move _1) -> [return: bb1, unwind continue];
+        _3 = move (_2.0: usize);
+        _4 = move (_2.1: usize);
+        StorageLive(_5);
+        _5 = Lt(copy _4, copy _3);
+        switchInt(move _5) -> [0: bb1, otherwise: bb4];
     }
 
     bb1: {
+        _6 = SubUnchecked(copy _4, copy _3);
+        StorageDead(_5);
+        StorageLive(_8);
+        StorageLive(_7);
+        _7 = PtrMetadata(copy _1);
+        _8 = Le(copy _4, move _7);
+        switchInt(move _8) -> [0: bb2, otherwise: bb3];
+    }
+
+    bb2: {
+        StorageDead(_7);
+        goto -> bb5;
+    }
+
+    bb3: {
+        StorageDead(_7);
+        StorageLive(_12);
+        StorageLive(_9);
+        _9 = &raw const (*_1);
+        StorageLive(_10);
+        StorageLive(_11);
+        _10 = copy _9 as *const u32 (PtrToPtr);
+        _11 = Offset(copy _10, copy _3);
+        _12 = *const [u32] from (copy _11, copy _6);
+        StorageDead(_11);
+        StorageDead(_10);
+        StorageDead(_9);
+        _0 = &(*_12);
+        StorageDead(_12);
+        StorageDead(_8);
         return;
     }
+
+    bb4: {
+        StorageDead(_5);
+        goto -> bb5;
+    }
+
+    bb5: {
+        StorageLive(_13);
+        _13 = PtrMetadata(copy _1);
+        _14 = core::slice::index::slice_index_fail(move _3, move _4, move _13) -> unwind continue;
+    }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir
index d389e4069d05..104987b0fdda 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-abort.mir
@@ -4,30 +4,28 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::ptr::NonNull;
-    let mut _12: *const T;
-    let mut _13: usize;
-    let mut _32: std::option::Option<(usize, &T)>;
-    let mut _35: &impl Fn(usize, &T);
-    let mut _36: (usize, &T);
-    let _37: ();
+    let mut _10: usize;
+    let mut _28: std::option::Option<(usize, &T)>;
+    let mut _31: &impl Fn(usize, &T);
+    let mut _32: (usize, &T);
+    let _33: ();
     scope 1 {
-        debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11;
-        debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6;
+        debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).1: *const T) => _9;
         debug (((iter: Enumerate>).0: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
-        debug ((iter: Enumerate>).1: usize) => _13;
-        let _33: usize;
-        let _34: &T;
+        debug ((iter: Enumerate>).1: usize) => _10;
+        let _29: usize;
+        let _30: &T;
         scope 2 {
-            debug i => _33;
-            debug x => _34;
+            debug i => _29;
+            debug x => _30;
         }
         scope 18 (inlined > as Iterator>::next) {
-            let mut _27: std::option::Option<&T>;
-            let mut _30: (usize, bool);
-            let mut _31: (usize, &T);
+            let mut _23: std::option::Option<&T>;
+            let mut _26: (usize, bool);
+            let mut _27: (usize, &T);
             scope 19 {
-                let _29: usize;
+                let _25: usize;
                 scope 24 {
                 }
             }
@@ -42,21 +40,21 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                 }
             }
             scope 25 (inlined  as Try>::branch) {
-                let _28: &T;
+                let _24: &T;
                 scope 26 {
                 }
             }
             scope 28 (inlined  as Iterator>::next) {
-                let _14: std::ptr::NonNull;
-                let _16: std::ptr::NonNull;
-                let mut _19: bool;
-                let mut _22: std::ptr::NonNull;
-                let mut _24: usize;
-                let _26: &T;
+                let mut _6: std::ptr::NonNull;
+                let _11: std::ptr::NonNull;
+                let _13: std::ptr::NonNull;
+                let mut _16: bool;
+                let mut _20: usize;
+                let _22: &T;
                 scope 29 {
-                    let _15: *const T;
+                    let _12: *const T;
                     scope 30 {
-                        let _23: usize;
+                        let _19: usize;
                         scope 31 {
                             scope 34 (inlined #[track_caller] core::num::::unchecked_sub) {
                                 scope 35 (inlined core::ub_checks::check_language_ub) {
@@ -72,21 +70,21 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
                             }
                         }
                         scope 38 (inlined  as PartialEq>::eq) {
-                            let mut _17: *mut T;
-                            let mut _18: *mut T;
+                            let mut _14: *mut T;
+                            let mut _15: *mut T;
                             scope 39 (inlined NonNull::::as_ptr) {
                             }
                             scope 40 (inlined NonNull::::as_ptr) {
                             }
                         }
                         scope 41 (inlined NonNull::::add) {
-                            let mut _20: *const T;
-                            let mut _21: *const T;
+                            let mut _17: *const T;
+                            let mut _18: *const T;
                             scope 42 (inlined NonNull::::as_ptr) {
                             }
                         }
                         scope 43 (inlined NonNull::::as_ref::<'_>) {
-                            let _25: *const T;
+                            let _21: *const T;
                             scope 44 (inlined NonNull::::as_ptr) {
                             }
                             scope 45 (inlined std::ptr::mut_ptr::::cast_const) {
@@ -102,9 +100,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
-                let _6: std::ptr::NonNull;
                 scope 6 {
                     let _9: *const T;
                     scope 7 {
@@ -143,9 +139,8 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -155,7 +150,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -166,97 +161,86 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb3: {
-        _10 = copy _9;
-        StorageDead(_9);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_11);
-        StorageLive(_12);
-        StorageLive(_13);
-        _11 = copy _6;
-        _12 = copy _10;
-        _13 = const 0_usize;
+        StorageLive(_10);
+        _10 = const 0_usize;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_32);
-        StorageLive(_29);
-        StorageLive(_30);
-        StorageLive(_27);
-        StorageLive(_14);
-        StorageLive(_15);
-        StorageLive(_23);
-        StorageLive(_24);
-        StorageLive(_16);
+        StorageLive(_28);
+        StorageLive(_25);
         StorageLive(_26);
-        _14 = copy _11;
-        _15 = copy _12;
+        StorageLive(_23);
+        StorageLive(_11);
+        StorageLive(_12);
+        StorageLive(_19);
+        StorageLive(_20);
+        StorageLive(_13);
+        StorageLive(_22);
+        _11 = copy _6;
+        _12 = copy _9;
         switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        StorageLive(_19);
-        _16 = copy _15 as std::ptr::NonNull (Transmute);
-        StorageLive(_17);
-        _17 = copy _14 as *mut T (Transmute);
-        StorageLive(_18);
-        _18 = copy _16 as *mut T (Transmute);
-        _19 = Eq(move _17, move _18);
-        StorageDead(_18);
-        StorageDead(_17);
-        switchInt(move _19) -> [0: bb6, otherwise: bb7];
+        StorageLive(_16);
+        _13 = copy _12 as std::ptr::NonNull (Transmute);
+        StorageLive(_14);
+        _14 = copy _11 as *mut T (Transmute);
+        StorageLive(_15);
+        _15 = copy _13 as *mut T (Transmute);
+        _16 = Eq(copy _14, copy _15);
+        StorageDead(_15);
+        StorageDead(_14);
+        switchInt(move _16) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_19);
-        StorageLive(_22);
-        StorageLive(_21);
-        StorageLive(_20);
-        _20 = copy _14 as *const T (Transmute);
-        _21 = Offset(move _20, const 1_usize);
-        StorageDead(_20);
-        _22 = NonNull:: { pointer: move _21 };
-        StorageDead(_21);
-        _11 = move _22;
-        StorageDead(_22);
+        StorageDead(_16);
+        StorageLive(_18);
+        StorageLive(_17);
+        _17 = copy _11 as *const T (Transmute);
+        _18 = Offset(copy _17, const 1_usize);
+        StorageDead(_17);
+        _6 = NonNull:: { pointer: copy _18 };
+        StorageDead(_18);
         goto -> bb13;
     }
 
     bb7: {
-        StorageDead(_19);
-        StorageDead(_26);
         StorageDead(_16);
-        StorageDead(_24);
-        StorageDead(_23);
-        StorageDead(_15);
-        StorageDead(_14);
+        StorageDead(_22);
+        StorageDead(_13);
+        StorageDead(_20);
+        StorageDead(_19);
+        StorageDead(_12);
+        StorageDead(_11);
         goto -> bb10;
     }
 
     bb8: {
-        _23 = copy _15 as usize (Transmute);
-        switchInt(copy _23) -> [0: bb9, otherwise: bb12];
+        _19 = copy _12 as usize (Transmute);
+        switchInt(copy _19) -> [0: bb9, otherwise: bb12];
     }
 
     bb9: {
-        StorageDead(_26);
-        StorageDead(_16);
-        StorageDead(_24);
-        StorageDead(_23);
-        StorageDead(_15);
-        StorageDead(_14);
+        StorageDead(_22);
+        StorageDead(_13);
+        StorageDead(_20);
+        StorageDead(_19);
+        StorageDead(_12);
+        StorageDead(_11);
         goto -> bb10;
     }
 
     bb10: {
-        StorageDead(_27);
-        StorageDead(_30);
-        StorageDead(_29);
-        StorageDead(_32);
-        StorageDead(_11);
-        StorageDead(_12);
-        StorageDead(_13);
+        StorageDead(_23);
+        StorageDead(_26);
+        StorageDead(_25);
+        StorageDead(_28);
+        StorageDead(_10);
         drop(_2) -> [return: bb11, unwind unreachable];
     }
 
@@ -265,51 +249,51 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb12: {
-        _24 = SubUnchecked(copy _23, const 1_usize);
-        _12 = copy _24 as *const T (Transmute);
+        _20 = SubUnchecked(copy _19, const 1_usize);
+        _9 = copy _20 as *const T (Transmute);
         goto -> bb13;
     }
 
     bb13: {
-        StorageLive(_25);
-        _25 = copy _14 as *const T (Transmute);
-        _26 = &(*_25);
-        StorageDead(_25);
-        _27 = Option::<&T>::Some(copy _26);
-        StorageDead(_26);
-        StorageDead(_16);
-        StorageDead(_24);
+        StorageLive(_21);
+        _21 = copy _11 as *const T (Transmute);
+        _22 = &(*_21);
+        StorageDead(_21);
+        _23 = Option::<&T>::Some(copy _22);
+        StorageDead(_22);
+        StorageDead(_13);
+        StorageDead(_20);
+        StorageDead(_19);
+        StorageDead(_12);
+        StorageDead(_11);
+        _24 = copy ((_23 as Some).0: &T);
         StorageDead(_23);
-        StorageDead(_15);
-        StorageDead(_14);
-        _28 = move ((_27 as Some).0: &T);
-        StorageDead(_27);
-        _29 = copy _13;
-        _30 = AddWithOverflow(copy _13, const 1_usize);
-        assert(!move (_30.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _13, const 1_usize) -> [success: bb14, unwind unreachable];
+        _25 = copy _10;
+        _26 = AddWithOverflow(copy _10, const 1_usize);
+        assert(!move (_26.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _10, const 1_usize) -> [success: bb14, unwind unreachable];
     }
 
     bb14: {
-        _13 = move (_30.0: usize);
+        _10 = move (_26.0: usize);
+        StorageLive(_27);
+        _27 = (copy _25, copy _24);
+        _28 = Option::<(usize, &T)>::Some(move _27);
+        StorageDead(_27);
+        StorageDead(_26);
+        StorageDead(_25);
+        _29 = copy (((_28 as Some).0: (usize, &T)).0: usize);
+        _30 = copy (((_28 as Some).0: (usize, &T)).1: &T);
         StorageLive(_31);
-        _31 = (copy _29, copy _28);
-        _32 = Option::<(usize, &T)>::Some(move _31);
-        StorageDead(_31);
-        StorageDead(_30);
-        StorageDead(_29);
-        _33 = copy (((_32 as Some).0: (usize, &T)).0: usize);
-        _34 = copy (((_32 as Some).0: (usize, &T)).1: &T);
-        StorageLive(_35);
-        _35 = &_2;
-        StorageLive(_36);
-        _36 = (copy _33, copy _34);
-        _37 = >::call(move _35, move _36) -> [return: bb15, unwind unreachable];
+        _31 = &_2;
+        StorageLive(_32);
+        _32 = (copy _29, copy _30);
+        _33 = >::call(move _31, move _32) -> [return: bb15, unwind unreachable];
     }
 
     bb15: {
-        StorageDead(_36);
-        StorageDead(_35);
         StorageDead(_32);
+        StorageDead(_31);
+        StorageDead(_28);
         goto -> bb4;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir
index 8c5fbda63921..28b12cdf3675 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.enumerated_loop.PreCodegen.after.panic-unwind.mir
@@ -4,22 +4,22 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _10: std::slice::Iter<'_, T>;
+    let mut _11: std::iter::Enumerate>;
     let mut _12: std::iter::Enumerate>;
-    let mut _13: std::iter::Enumerate>;
-    let mut _14: &mut std::iter::Enumerate>;
-    let mut _15: std::option::Option<(usize, &T)>;
-    let mut _16: isize;
-    let mut _19: &impl Fn(usize, &T);
-    let mut _20: (usize, &T);
-    let _21: ();
+    let mut _13: &mut std::iter::Enumerate>;
+    let mut _14: std::option::Option<(usize, &T)>;
+    let mut _15: isize;
+    let mut _18: &impl Fn(usize, &T);
+    let mut _19: (usize, &T);
+    let _20: ();
     scope 1 {
-        debug iter => _13;
-        let _17: usize;
-        let _18: &T;
+        debug iter => _12;
+        let _16: usize;
+        let _17: &T;
         scope 2 {
-            debug i => _17;
-            debug x => _18;
+            debug i => _16;
+            debug x => _17;
         }
     }
     scope 3 (inlined core::slice::::iter) {
@@ -27,7 +27,6 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
                 let _6: std::ptr::NonNull;
                 scope 6 {
@@ -62,17 +61,17 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb0: {
-        StorageLive(_11);
+        StorageLive(_10);
         StorageLive(_3);
         StorageLive(_6);
+        StorageLive(_9);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -82,7 +81,7 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -93,33 +92,30 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
-        _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
-        StorageDead(_9);
+        _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> };
         StorageDead(_4);
+        StorageDead(_9);
         StorageDead(_6);
         StorageDead(_3);
-        _12 = Enumerate::> { iter: copy _11, count: const 0_usize };
-        StorageDead(_11);
-        StorageLive(_13);
-        _13 = copy _12;
+        _11 = Enumerate::> { iter: copy _10, count: const 0_usize };
+        StorageDead(_10);
+        StorageLive(_12);
+        _12 = copy _11;
         goto -> bb4;
     }
 
     bb4: {
-        _14 = &mut _13;
-        _15 = > as Iterator>::next(move _14) -> [return: bb5, unwind: bb11];
+        _13 = &mut _12;
+        _14 = > as Iterator>::next(move _13) -> [return: bb5, unwind: bb11];
     }
 
     bb5: {
-        _16 = discriminant(_15);
-        switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10];
+        _15 = discriminant(_14);
+        switchInt(move _15) -> [0: bb6, 1: bb8, otherwise: bb10];
     }
 
     bb6: {
-        StorageDead(_13);
+        StorageDead(_12);
         drop(_2) -> [return: bb7, unwind continue];
     }
 
@@ -128,18 +124,18 @@ fn enumerated_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     }
 
     bb8: {
-        _17 = copy (((_15 as Some).0: (usize, &T)).0: usize);
-        _18 = copy (((_15 as Some).0: (usize, &T)).1: &T);
+        _16 = copy (((_14 as Some).0: (usize, &T)).0: usize);
+        _17 = copy (((_14 as Some).0: (usize, &T)).1: &T);
+        StorageLive(_18);
+        _18 = &_2;
         StorageLive(_19);
-        _19 = &_2;
-        StorageLive(_20);
-        _20 = copy ((_15 as Some).0: (usize, &T));
-        _21 = >::call(move _19, move _20) -> [return: bb9, unwind: bb11];
+        _19 = copy ((_14 as Some).0: (usize, &T));
+        _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11];
     }
 
     bb9: {
-        StorageDead(_20);
         StorageDead(_19);
+        StorageDead(_18);
         goto -> bb4;
     }
 
diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir
index 216e05ec5b79..4d0e3548e7d6 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-abort.mir
@@ -4,31 +4,29 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::ptr::NonNull;
-    let mut _12: *const T;
-    let mut _26: std::option::Option<&T>;
-    let mut _28: &impl Fn(&T);
-    let mut _29: (&T,);
-    let _30: ();
+    let mut _22: std::option::Option<&T>;
+    let mut _24: &impl Fn(&T);
+    let mut _25: (&T,);
+    let _26: ();
     scope 1 {
-        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11;
-        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6;
+        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9;
         debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
-        let _27: &T;
+        let _23: &T;
         scope 2 {
-            debug x => _27;
+            debug x => _23;
         }
         scope 16 (inlined  as Iterator>::next) {
-            let _13: std::ptr::NonNull;
-            let _15: std::ptr::NonNull;
-            let mut _18: bool;
-            let mut _21: std::ptr::NonNull;
-            let mut _23: usize;
-            let _25: &T;
+            let mut _6: std::ptr::NonNull;
+            let _10: std::ptr::NonNull;
+            let _12: std::ptr::NonNull;
+            let mut _15: bool;
+            let mut _19: usize;
+            let _21: &T;
             scope 17 {
-                let _14: *const T;
+                let _11: *const T;
                 scope 18 {
-                    let _22: usize;
+                    let _18: usize;
                     scope 19 {
                         scope 22 (inlined #[track_caller] core::num::::unchecked_sub) {
                             scope 23 (inlined core::ub_checks::check_language_ub) {
@@ -44,21 +42,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
                         }
                     }
                     scope 26 (inlined  as PartialEq>::eq) {
-                        let mut _16: *mut T;
-                        let mut _17: *mut T;
+                        let mut _13: *mut T;
+                        let mut _14: *mut T;
                         scope 27 (inlined NonNull::::as_ptr) {
                         }
                         scope 28 (inlined NonNull::::as_ptr) {
                         }
                     }
                     scope 29 (inlined NonNull::::add) {
-                        let mut _19: *const T;
-                        let mut _20: *const T;
+                        let mut _16: *const T;
+                        let mut _17: *const T;
                         scope 30 (inlined NonNull::::as_ptr) {
                         }
                     }
                     scope 31 (inlined NonNull::::as_ref::<'_>) {
-                        let _24: *const T;
+                        let _20: *const T;
                         scope 32 (inlined NonNull::::as_ptr) {
                         }
                         scope 33 (inlined std::ptr::mut_ptr::::cast_const) {
@@ -73,9 +71,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
-                let _6: std::ptr::NonNull;
                 scope 6 {
                     let _9: *const T;
                     scope 7 {
@@ -110,9 +106,8 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -122,7 +117,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -133,88 +128,77 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        _10 = copy _9;
-        StorageDead(_9);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_11);
-        StorageLive(_12);
-        _11 = copy _6;
-        _12 = copy _10;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_26);
-        StorageLive(_13);
-        StorageLive(_14);
         StorageLive(_22);
-        StorageLive(_23);
-        StorageLive(_15);
-        StorageLive(_25);
-        _13 = copy _11;
-        _14 = copy _12;
+        StorageLive(_10);
+        StorageLive(_11);
+        StorageLive(_18);
+        StorageLive(_19);
+        StorageLive(_12);
+        StorageLive(_21);
+        _10 = copy _6;
+        _11 = copy _9;
         switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        StorageLive(_18);
-        _15 = copy _14 as std::ptr::NonNull (Transmute);
-        StorageLive(_16);
-        _16 = copy _13 as *mut T (Transmute);
-        StorageLive(_17);
-        _17 = copy _15 as *mut T (Transmute);
-        _18 = Eq(move _16, move _17);
-        StorageDead(_17);
-        StorageDead(_16);
-        switchInt(move _18) -> [0: bb6, otherwise: bb7];
+        StorageLive(_15);
+        _12 = copy _11 as std::ptr::NonNull (Transmute);
+        StorageLive(_13);
+        _13 = copy _10 as *mut T (Transmute);
+        StorageLive(_14);
+        _14 = copy _12 as *mut T (Transmute);
+        _15 = Eq(copy _13, copy _14);
+        StorageDead(_14);
+        StorageDead(_13);
+        switchInt(move _15) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_18);
-        StorageLive(_21);
-        StorageLive(_20);
-        StorageLive(_19);
-        _19 = copy _13 as *const T (Transmute);
-        _20 = Offset(move _19, const 1_usize);
-        StorageDead(_19);
-        _21 = NonNull:: { pointer: move _20 };
-        StorageDead(_20);
-        _11 = move _21;
-        StorageDead(_21);
+        StorageDead(_15);
+        StorageLive(_17);
+        StorageLive(_16);
+        _16 = copy _10 as *const T (Transmute);
+        _17 = Offset(copy _16, const 1_usize);
+        StorageDead(_16);
+        _6 = NonNull:: { pointer: copy _17 };
+        StorageDead(_17);
         goto -> bb13;
     }
 
     bb7: {
-        StorageDead(_18);
-        StorageDead(_25);
         StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
         goto -> bb10;
     }
 
     bb8: {
-        _22 = copy _14 as usize (Transmute);
-        switchInt(copy _22) -> [0: bb9, otherwise: bb12];
+        _18 = copy _11 as usize (Transmute);
+        switchInt(copy _18) -> [0: bb9, otherwise: bb12];
     }
 
     bb9: {
-        StorageDead(_25);
-        StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
         goto -> bb10;
     }
 
     bb10: {
-        StorageDead(_26);
-        StorageDead(_11);
-        StorageDead(_12);
+        StorageDead(_22);
         drop(_2) -> [return: bb11, unwind unreachable];
     }
 
@@ -223,35 +207,35 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb12: {
-        _23 = SubUnchecked(copy _22, const 1_usize);
-        _12 = copy _23 as *const T (Transmute);
+        _19 = SubUnchecked(copy _18, const 1_usize);
+        _9 = copy _19 as *const T (Transmute);
         goto -> bb13;
     }
 
     bb13: {
+        StorageLive(_20);
+        _20 = copy _10 as *const T (Transmute);
+        _21 = &(*_20);
+        StorageDead(_20);
+        _22 = Option::<&T>::Some(copy _21);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
+        _23 = copy ((_22 as Some).0: &T);
         StorageLive(_24);
-        _24 = copy _13 as *const T (Transmute);
-        _25 = &(*_24);
-        StorageDead(_24);
-        _26 = Option::<&T>::Some(copy _25);
-        StorageDead(_25);
-        StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
-        _27 = copy ((_26 as Some).0: &T);
-        StorageLive(_28);
-        _28 = &_2;
-        StorageLive(_29);
-        _29 = (copy _27,);
-        _30 = >::call(move _28, move _29) -> [return: bb14, unwind unreachable];
+        _24 = &_2;
+        StorageLive(_25);
+        _25 = (copy _23,);
+        _26 = >::call(move _24, move _25) -> [return: bb14, unwind unreachable];
     }
 
     bb14: {
-        StorageDead(_29);
-        StorageDead(_28);
-        StorageDead(_26);
+        StorageDead(_25);
+        StorageDead(_24);
+        StorageDead(_22);
         goto -> bb4;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir
index 001023919804..2b5d8c27d710 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.forward_loop.PreCodegen.after.panic-unwind.mir
@@ -4,31 +4,29 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::ptr::NonNull;
-    let mut _12: *const T;
-    let mut _26: std::option::Option<&T>;
-    let mut _28: &impl Fn(&T);
-    let mut _29: (&T,);
-    let _30: ();
+    let mut _22: std::option::Option<&T>;
+    let mut _24: &impl Fn(&T);
+    let mut _25: (&T,);
+    let _26: ();
     scope 1 {
-        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _11;
-        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _12;
+        debug ((iter: std::slice::Iter<'_, T>).0: std::ptr::NonNull) => _6;
+        debug ((iter: std::slice::Iter<'_, T>).1: *const T) => _9;
         debug ((iter: std::slice::Iter<'_, T>).2: std::marker::PhantomData<&T>) => const ZeroSized: PhantomData<&T>;
-        let _27: &T;
+        let _23: &T;
         scope 2 {
-            debug x => _27;
+            debug x => _23;
         }
         scope 16 (inlined  as Iterator>::next) {
-            let _13: std::ptr::NonNull;
-            let _15: std::ptr::NonNull;
-            let mut _18: bool;
-            let mut _21: std::ptr::NonNull;
-            let mut _23: usize;
-            let _25: &T;
+            let mut _6: std::ptr::NonNull;
+            let _10: std::ptr::NonNull;
+            let _12: std::ptr::NonNull;
+            let mut _15: bool;
+            let mut _19: usize;
+            let _21: &T;
             scope 17 {
-                let _14: *const T;
+                let _11: *const T;
                 scope 18 {
-                    let _22: usize;
+                    let _18: usize;
                     scope 19 {
                         scope 22 (inlined #[track_caller] core::num::::unchecked_sub) {
                             scope 23 (inlined core::ub_checks::check_language_ub) {
@@ -44,21 +42,21 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
                         }
                     }
                     scope 26 (inlined  as PartialEq>::eq) {
-                        let mut _16: *mut T;
-                        let mut _17: *mut T;
+                        let mut _13: *mut T;
+                        let mut _14: *mut T;
                         scope 27 (inlined NonNull::::as_ptr) {
                         }
                         scope 28 (inlined NonNull::::as_ptr) {
                         }
                     }
                     scope 29 (inlined NonNull::::add) {
-                        let mut _19: *const T;
-                        let mut _20: *const T;
+                        let mut _16: *const T;
+                        let mut _17: *const T;
                         scope 30 (inlined NonNull::::as_ptr) {
                         }
                     }
                     scope 31 (inlined NonNull::::as_ref::<'_>) {
-                        let _24: *const T;
+                        let _20: *const T;
                         scope 32 (inlined NonNull::::as_ptr) {
                         }
                         scope 33 (inlined std::ptr::mut_ptr::::cast_const) {
@@ -73,9 +71,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
-                let _6: std::ptr::NonNull;
                 scope 6 {
                     let _9: *const T;
                     scope 7 {
@@ -110,9 +106,8 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -122,7 +117,7 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -133,88 +128,77 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        _10 = copy _9;
-        StorageDead(_9);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_11);
-        StorageLive(_12);
-        _11 = copy _6;
-        _12 = copy _10;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_26);
-        StorageLive(_13);
-        StorageLive(_14);
         StorageLive(_22);
-        StorageLive(_23);
-        StorageLive(_15);
-        StorageLive(_25);
-        _13 = copy _11;
-        _14 = copy _12;
+        StorageLive(_10);
+        StorageLive(_11);
+        StorageLive(_18);
+        StorageLive(_19);
+        StorageLive(_12);
+        StorageLive(_21);
+        _10 = copy _6;
+        _11 = copy _9;
         switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8];
     }
 
     bb5: {
-        StorageLive(_18);
-        _15 = copy _14 as std::ptr::NonNull (Transmute);
-        StorageLive(_16);
-        _16 = copy _13 as *mut T (Transmute);
-        StorageLive(_17);
-        _17 = copy _15 as *mut T (Transmute);
-        _18 = Eq(move _16, move _17);
-        StorageDead(_17);
-        StorageDead(_16);
-        switchInt(move _18) -> [0: bb6, otherwise: bb7];
+        StorageLive(_15);
+        _12 = copy _11 as std::ptr::NonNull (Transmute);
+        StorageLive(_13);
+        _13 = copy _10 as *mut T (Transmute);
+        StorageLive(_14);
+        _14 = copy _12 as *mut T (Transmute);
+        _15 = Eq(copy _13, copy _14);
+        StorageDead(_14);
+        StorageDead(_13);
+        switchInt(move _15) -> [0: bb6, otherwise: bb7];
     }
 
     bb6: {
-        StorageDead(_18);
-        StorageLive(_21);
-        StorageLive(_20);
-        StorageLive(_19);
-        _19 = copy _13 as *const T (Transmute);
-        _20 = Offset(move _19, const 1_usize);
-        StorageDead(_19);
-        _21 = NonNull:: { pointer: move _20 };
-        StorageDead(_20);
-        _11 = move _21;
-        StorageDead(_21);
+        StorageDead(_15);
+        StorageLive(_17);
+        StorageLive(_16);
+        _16 = copy _10 as *const T (Transmute);
+        _17 = Offset(copy _16, const 1_usize);
+        StorageDead(_16);
+        _6 = NonNull:: { pointer: copy _17 };
+        StorageDead(_17);
         goto -> bb13;
     }
 
     bb7: {
-        StorageDead(_18);
-        StorageDead(_25);
         StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
         goto -> bb10;
     }
 
     bb8: {
-        _22 = copy _14 as usize (Transmute);
-        switchInt(copy _22) -> [0: bb9, otherwise: bb12];
+        _18 = copy _11 as usize (Transmute);
+        switchInt(copy _18) -> [0: bb9, otherwise: bb12];
     }
 
     bb9: {
-        StorageDead(_25);
-        StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
         goto -> bb10;
     }
 
     bb10: {
-        StorageDead(_26);
-        StorageDead(_11);
-        StorageDead(_12);
+        StorageDead(_22);
         drop(_2) -> [return: bb11, unwind continue];
     }
 
@@ -223,35 +207,35 @@ fn forward_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb12: {
-        _23 = SubUnchecked(copy _22, const 1_usize);
-        _12 = copy _23 as *const T (Transmute);
+        _19 = SubUnchecked(copy _18, const 1_usize);
+        _9 = copy _19 as *const T (Transmute);
         goto -> bb13;
     }
 
     bb13: {
+        StorageLive(_20);
+        _20 = copy _10 as *const T (Transmute);
+        _21 = &(*_20);
+        StorageDead(_20);
+        _22 = Option::<&T>::Some(copy _21);
+        StorageDead(_21);
+        StorageDead(_12);
+        StorageDead(_19);
+        StorageDead(_18);
+        StorageDead(_11);
+        StorageDead(_10);
+        _23 = copy ((_22 as Some).0: &T);
         StorageLive(_24);
-        _24 = copy _13 as *const T (Transmute);
-        _25 = &(*_24);
-        StorageDead(_24);
-        _26 = Option::<&T>::Some(copy _25);
-        StorageDead(_25);
-        StorageDead(_15);
-        StorageDead(_23);
-        StorageDead(_22);
-        StorageDead(_14);
-        StorageDead(_13);
-        _27 = copy ((_26 as Some).0: &T);
-        StorageLive(_28);
-        _28 = &_2;
-        StorageLive(_29);
-        _29 = (copy _27,);
-        _30 = >::call(move _28, move _29) -> [return: bb14, unwind: bb15];
+        _24 = &_2;
+        StorageLive(_25);
+        _25 = (copy _23,);
+        _26 = >::call(move _24, move _25) -> [return: bb14, unwind: bb15];
     }
 
     bb14: {
-        StorageDead(_29);
-        StorageDead(_28);
-        StorageDead(_26);
+        StorageDead(_25);
+        StorageDead(_24);
+        StorageDead(_22);
         goto -> bb4;
     }
 
diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir
index 41e273151eca..145375990710 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-abort.mir
@@ -5,28 +5,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug f => _2;
     let mut _0: ();
     let mut _3: usize;
-    let mut _4: usize;
-    let mut _9: std::option::Option;
-    let mut _11: bool;
-    let mut _13: &impl Fn(usize, &T);
-    let mut _14: (usize, &T);
-    let _15: ();
+    let mut _8: std::option::Option;
+    let mut _10: bool;
+    let mut _12: &impl Fn(usize, &T);
+    let mut _13: (usize, &T);
+    let _14: ();
     scope 1 {
         debug ((iter: std::ops::Range).0: usize) => _4;
         debug ((iter: std::ops::Range).1: usize) => _3;
-        let _10: usize;
+        let _9: usize;
         scope 2 {
-            debug i => _10;
-            let _12: &T;
+            debug i => _9;
+            let _11: &T;
             scope 3 {
-                debug x => _12;
+                debug x => _11;
             }
         }
         scope 5 (inlined iter::range::>::next) {
             scope 6 (inlined  as iter::range::RangeIteratorImpl>::spec_next) {
+                let mut _4: usize;
                 let mut _6: bool;
                 let _7: usize;
-                let mut _8: usize;
                 scope 7 {
                     scope 9 (inlined ::forward_unchecked) {
                         scope 10 (inlined #[track_caller] core::num::::unchecked_add) {
@@ -48,13 +47,12 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb0: {
         _3 = PtrMetadata(copy _1);
-        StorageLive(_4);
         _4 = const 0_usize;
         goto -> bb1;
     }
 
     bb1: {
-        StorageLive(_9);
+        StorageLive(_8);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -65,8 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_9);
-        StorageDead(_4);
+        StorageDead(_8);
         drop(_2) -> [return: bb3, unwind unreachable];
     }
 
@@ -76,30 +73,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb4: {
         _7 = copy _4;
-        StorageLive(_8);
-        _8 = AddUnchecked(copy _7, const 1_usize);
-        _4 = move _8;
-        StorageDead(_8);
-        _9 = Option::::Some(copy _7);
+        _4 = AddUnchecked(copy _7, const 1_usize);
+        _8 = Option::::Some(copy _7);
         StorageDead(_6);
-        _10 = copy ((_9 as Some).0: usize);
-        _11 = Lt(copy _10, copy _3);
-        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind unreachable];
+        _9 = copy ((_8 as Some).0: usize);
+        _10 = Lt(copy _9, copy _3);
+        assert(move _10, "index out of bounds: the length is {} but the index is {}", copy _3, copy _9) -> [success: bb5, unwind unreachable];
     }
 
     bb5: {
-        _12 = &(*_1)[_10];
+        _11 = &(*_1)[_9];
+        StorageLive(_12);
+        _12 = &_2;
         StorageLive(_13);
-        _13 = &_2;
-        StorageLive(_14);
-        _14 = (copy _10, copy _12);
-        _15 = >::call(move _13, move _14) -> [return: bb6, unwind unreachable];
+        _13 = (copy _9, copy _11);
+        _14 = >::call(move _12, move _13) -> [return: bb6, unwind unreachable];
     }
 
     bb6: {
-        StorageDead(_14);
         StorageDead(_13);
-        StorageDead(_9);
+        StorageDead(_12);
+        StorageDead(_8);
         goto -> bb1;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir
index ec781c1480c7..8e573ef488fc 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.range_loop.PreCodegen.after.panic-unwind.mir
@@ -5,28 +5,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
     debug f => _2;
     let mut _0: ();
     let mut _3: usize;
-    let mut _4: usize;
-    let mut _9: std::option::Option;
-    let mut _11: bool;
-    let mut _13: &impl Fn(usize, &T);
-    let mut _14: (usize, &T);
-    let _15: ();
+    let mut _8: std::option::Option;
+    let mut _10: bool;
+    let mut _12: &impl Fn(usize, &T);
+    let mut _13: (usize, &T);
+    let _14: ();
     scope 1 {
         debug ((iter: std::ops::Range).0: usize) => _4;
         debug ((iter: std::ops::Range).1: usize) => _3;
-        let _10: usize;
+        let _9: usize;
         scope 2 {
-            debug i => _10;
-            let _12: &T;
+            debug i => _9;
+            let _11: &T;
             scope 3 {
-                debug x => _12;
+                debug x => _11;
             }
         }
         scope 5 (inlined iter::range::>::next) {
             scope 6 (inlined  as iter::range::RangeIteratorImpl>::spec_next) {
+                let mut _4: usize;
                 let mut _6: bool;
                 let _7: usize;
-                let mut _8: usize;
                 scope 7 {
                     scope 9 (inlined ::forward_unchecked) {
                         scope 10 (inlined #[track_caller] core::num::::unchecked_add) {
@@ -48,13 +47,12 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb0: {
         _3 = PtrMetadata(copy _1);
-        StorageLive(_4);
         _4 = const 0_usize;
         goto -> bb1;
     }
 
     bb1: {
-        StorageLive(_9);
+        StorageLive(_8);
         StorageLive(_6);
         StorageLive(_5);
         _5 = copy _4;
@@ -65,8 +63,7 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb2: {
         StorageDead(_6);
-        StorageDead(_9);
-        StorageDead(_4);
+        StorageDead(_8);
         drop(_2) -> [return: bb3, unwind continue];
     }
 
@@ -76,30 +73,27 @@ fn range_loop(_1: &[T], _2: impl Fn(usize, &T)) -> () {
 
     bb4: {
         _7 = copy _4;
-        StorageLive(_8);
-        _8 = AddUnchecked(copy _7, const 1_usize);
-        _4 = move _8;
-        StorageDead(_8);
-        _9 = Option::::Some(copy _7);
+        _4 = AddUnchecked(copy _7, const 1_usize);
+        _8 = Option::::Some(copy _7);
         StorageDead(_6);
-        _10 = copy ((_9 as Some).0: usize);
-        _11 = Lt(copy _10, copy _3);
-        assert(move _11, "index out of bounds: the length is {} but the index is {}", copy _3, copy _10) -> [success: bb5, unwind: bb7];
+        _9 = copy ((_8 as Some).0: usize);
+        _10 = Lt(copy _9, copy _3);
+        assert(move _10, "index out of bounds: the length is {} but the index is {}", copy _3, copy _9) -> [success: bb5, unwind: bb7];
     }
 
     bb5: {
-        _12 = &(*_1)[_10];
+        _11 = &(*_1)[_9];
+        StorageLive(_12);
+        _12 = &_2;
         StorageLive(_13);
-        _13 = &_2;
-        StorageLive(_14);
-        _14 = (copy _10, copy _12);
-        _15 = >::call(move _13, move _14) -> [return: bb6, unwind: bb7];
+        _13 = (copy _9, copy _11);
+        _14 = >::call(move _12, move _13) -> [return: bb6, unwind: bb7];
     }
 
     bb6: {
-        StorageDead(_14);
         StorageDead(_13);
-        StorageDead(_9);
+        StorageDead(_12);
+        StorageDead(_8);
         goto -> bb1;
     }
 
diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir
index b09e36223441..3009be3f9dc6 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir
@@ -4,22 +4,90 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _10: std::slice::Iter<'_, T>;
+    let mut _11: std::iter::Rev>;
     let mut _12: std::iter::Rev>;
-    let mut _13: std::iter::Rev>;
-    let mut _15: std::option::Option<&T>;
-    let mut _16: isize;
-    let mut _18: &impl Fn(&T);
-    let mut _19: (&T,);
-    let _20: ();
+    let mut _33: std::option::Option<&T>;
+    let mut _35: &impl Fn(&T);
+    let mut _36: (&T,);
+    let _37: ();
     scope 1 {
-        debug iter => _13;
-        let _17: &T;
+        debug iter => _12;
+        let _34: &T;
         scope 2 {
-            debug x => _17;
+            debug x => _34;
         }
         scope 18 (inlined > as Iterator>::next) {
-            let mut _14: &mut std::slice::Iter<'_, T>;
+            scope 19 (inlined  as DoubleEndedIterator>::next_back) {
+                let mut _13: *const T;
+                let mut _18: bool;
+                let mut _19: *const T;
+                let _32: &T;
+                scope 20 {
+                    let _14: std::ptr::NonNull;
+                    let _20: usize;
+                    scope 21 {
+                    }
+                    scope 22 {
+                        scope 25 (inlined  as PartialEq>::eq) {
+                            let mut _15: std::ptr::NonNull;
+                            let mut _16: *mut T;
+                            let mut _17: *mut T;
+                            scope 26 (inlined NonNull::::as_ptr) {
+                            }
+                            scope 27 (inlined NonNull::::as_ptr) {
+                            }
+                        }
+                    }
+                    scope 23 (inlined std::ptr::const_ptr::::addr) {
+                        scope 24 (inlined std::ptr::const_ptr::::cast::<()>) {
+                        }
+                    }
+                }
+                scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) {
+                    let _26: std::ptr::NonNull;
+                    scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) {
+                        let mut _21: *mut *const T;
+                        let mut _22: *mut std::ptr::NonNull;
+                        let mut _23: std::ptr::NonNull;
+                        let mut _27: *mut *const T;
+                        let mut _28: *mut usize;
+                        let mut _29: usize;
+                        let mut _30: usize;
+                        scope 30 {
+                            scope 31 {
+                            }
+                            scope 32 {
+                                scope 35 (inlined NonNull::::sub) {
+                                    scope 36 (inlined #[track_caller] core::num::::unchecked_neg) {
+                                        scope 37 (inlined core::ub_checks::check_language_ub) {
+                                            scope 38 (inlined core::ub_checks::check_language_ub::runtime) {
+                                            }
+                                        }
+                                    }
+                                    scope 39 (inlined NonNull::::offset) {
+                                        let mut _24: *const T;
+                                        let mut _25: *const T;
+                                        scope 40 (inlined NonNull::::as_ptr) {
+                                        }
+                                    }
+                                }
+                            }
+                            scope 33 (inlined std::ptr::mut_ptr::::cast::) {
+                            }
+                            scope 34 (inlined std::ptr::mut_ptr::::cast::>) {
+                            }
+                        }
+                    }
+                    scope 41 (inlined NonNull::::as_ref::<'_>) {
+                        let _31: *const T;
+                        scope 42 (inlined NonNull::::as_ptr) {
+                        }
+                        scope 43 (inlined std::ptr::mut_ptr::::cast_const) {
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::::iter) {
@@ -27,7 +95,6 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
                 let _6: std::ptr::NonNull;
                 scope 6 {
@@ -62,17 +129,17 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb0: {
-        StorageLive(_11);
+        StorageLive(_10);
         StorageLive(_3);
         StorageLive(_6);
+        StorageLive(_9);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -82,7 +149,7 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -93,61 +160,149 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
-        _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
-        StorageDead(_9);
+        _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> };
         StorageDead(_4);
+        StorageDead(_9);
         StorageDead(_6);
         StorageDead(_3);
-        _12 = Rev::> { iter: copy _11 };
-        StorageDead(_11);
-        StorageLive(_13);
-        _13 = copy _12;
+        _11 = Rev::> { iter: copy _10 };
+        StorageDead(_10);
+        StorageLive(_12);
+        _12 = copy _11;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_15);
+        StorageLive(_33);
+        StorageLive(_20);
+        StorageLive(_19);
         StorageLive(_14);
-        _14 = &mut (_13.0: std::slice::Iter<'_, T>);
-        _15 =  as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind unreachable];
+        StorageLive(_32);
+        StorageLive(_18);
+        switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6];
     }
 
     bb5: {
-        StorageDead(_14);
-        _16 = discriminant(_15);
-        switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10];
+        StorageLive(_13);
+        _13 = copy ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _14 = copy _13 as std::ptr::NonNull (Transmute);
+        StorageDead(_13);
+        StorageLive(_16);
+        StorageLive(_15);
+        _15 = copy ((_12.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull);
+        _16 = copy _15 as *mut T (Transmute);
+        StorageDead(_15);
+        StorageLive(_17);
+        _17 = copy _14 as *mut T (Transmute);
+        _18 = Eq(copy _16, copy _17);
+        StorageDead(_17);
+        StorageDead(_16);
+        goto -> bb7;
     }
 
     bb6: {
-        StorageDead(_15);
-        StorageDead(_13);
-        drop(_2) -> [return: bb7, unwind unreachable];
+        _19 = copy ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _20 = copy _19 as usize (Transmute);
+        _18 = Eq(copy _20, const 0_usize);
+        goto -> bb7;
     }
 
     bb7: {
-        return;
+        switchInt(move _18) -> [0: bb8, otherwise: bb15];
     }
 
     bb8: {
-        _17 = copy ((_15 as Some).0: &T);
-        StorageLive(_18);
-        _18 = &_2;
-        StorageLive(_19);
-        _19 = (copy _17,);
-        _20 = >::call(move _18, move _19) -> [return: bb9, unwind unreachable];
+        StorageLive(_26);
+        StorageLive(_28);
+        StorageLive(_22);
+        StorageLive(_23);
+        switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb12];
     }
 
     bb9: {
-        StorageDead(_19);
-        StorageDead(_18);
-        StorageDead(_15);
-        goto -> bb4;
+        StorageLive(_21);
+        _21 = &raw mut ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _22 = copy _21 as *mut std::ptr::NonNull (PtrToPtr);
+        StorageDead(_21);
+        _23 = copy (*_22);
+        switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11];
     }
 
     bb10: {
-        unreachable;
+        StorageLive(_25);
+        StorageLive(_24);
+        _24 = copy _23 as *const T (Transmute);
+        _25 = Offset(copy _24, const -1_isize);
+        StorageDead(_24);
+        _23 = NonNull:: { pointer: copy _25 };
+        StorageDead(_25);
+        goto -> bb11;
+    }
+
+    bb11: {
+        (*_22) = move _23;
+        _26 = copy (*_22);
+        goto -> bb13;
+    }
+
+    bb12: {
+        StorageLive(_27);
+        _27 = &raw mut ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _28 = copy _27 as *mut usize (PtrToPtr);
+        StorageDead(_27);
+        StorageLive(_30);
+        StorageLive(_29);
+        _29 = copy (*_28);
+        _30 = SubUnchecked(move _29, const 1_usize);
+        StorageDead(_29);
+        (*_28) = move _30;
+        StorageDead(_30);
+        _26 = copy ((_12.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull);
+        goto -> bb13;
+    }
+
+    bb13: {
+        StorageDead(_23);
+        StorageDead(_22);
+        StorageDead(_28);
+        StorageLive(_31);
+        _31 = copy _26 as *const T (Transmute);
+        _32 = &(*_31);
+        StorageDead(_31);
+        StorageDead(_26);
+        _33 = Option::<&T>::Some(copy _32);
+        StorageDead(_18);
+        StorageDead(_32);
+        StorageDead(_14);
+        StorageDead(_19);
+        StorageDead(_20);
+        _34 = copy ((_33 as Some).0: &T);
+        StorageLive(_35);
+        _35 = &_2;
+        StorageLive(_36);
+        _36 = (copy _34,);
+        _37 = >::call(move _35, move _36) -> [return: bb14, unwind unreachable];
+    }
+
+    bb14: {
+        StorageDead(_36);
+        StorageDead(_35);
+        StorageDead(_33);
+        goto -> bb4;
+    }
+
+    bb15: {
+        StorageDead(_18);
+        StorageDead(_32);
+        StorageDead(_14);
+        StorageDead(_19);
+        StorageDead(_20);
+        StorageDead(_33);
+        StorageDead(_12);
+        drop(_2) -> [return: bb16, unwind unreachable];
+    }
+
+    bb16: {
+        return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir
index 12b54b57b844..e40bff5ea350 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir
@@ -4,22 +4,90 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     debug slice => _1;
     debug f => _2;
     let mut _0: ();
-    let mut _11: std::slice::Iter<'_, T>;
+    let mut _10: std::slice::Iter<'_, T>;
+    let mut _11: std::iter::Rev>;
     let mut _12: std::iter::Rev>;
-    let mut _13: std::iter::Rev>;
-    let mut _15: std::option::Option<&T>;
-    let mut _16: isize;
-    let mut _18: &impl Fn(&T);
-    let mut _19: (&T,);
-    let _20: ();
+    let mut _33: std::option::Option<&T>;
+    let mut _35: &impl Fn(&T);
+    let mut _36: (&T,);
+    let _37: ();
     scope 1 {
-        debug iter => _13;
-        let _17: &T;
+        debug iter => _12;
+        let _34: &T;
         scope 2 {
-            debug x => _17;
+            debug x => _34;
         }
         scope 18 (inlined > as Iterator>::next) {
-            let mut _14: &mut std::slice::Iter<'_, T>;
+            scope 19 (inlined  as DoubleEndedIterator>::next_back) {
+                let mut _13: *const T;
+                let mut _18: bool;
+                let mut _19: *const T;
+                let _32: &T;
+                scope 20 {
+                    let _14: std::ptr::NonNull;
+                    let _20: usize;
+                    scope 21 {
+                    }
+                    scope 22 {
+                        scope 25 (inlined  as PartialEq>::eq) {
+                            let mut _15: std::ptr::NonNull;
+                            let mut _16: *mut T;
+                            let mut _17: *mut T;
+                            scope 26 (inlined NonNull::::as_ptr) {
+                            }
+                            scope 27 (inlined NonNull::::as_ptr) {
+                            }
+                        }
+                    }
+                    scope 23 (inlined std::ptr::const_ptr::::addr) {
+                        scope 24 (inlined std::ptr::const_ptr::::cast::<()>) {
+                        }
+                    }
+                }
+                scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) {
+                    let _26: std::ptr::NonNull;
+                    scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) {
+                        let mut _21: *mut *const T;
+                        let mut _22: *mut std::ptr::NonNull;
+                        let mut _23: std::ptr::NonNull;
+                        let mut _27: *mut *const T;
+                        let mut _28: *mut usize;
+                        let mut _29: usize;
+                        let mut _30: usize;
+                        scope 30 {
+                            scope 31 {
+                            }
+                            scope 32 {
+                                scope 35 (inlined NonNull::::sub) {
+                                    scope 36 (inlined #[track_caller] core::num::::unchecked_neg) {
+                                        scope 37 (inlined core::ub_checks::check_language_ub) {
+                                            scope 38 (inlined core::ub_checks::check_language_ub::runtime) {
+                                            }
+                                        }
+                                    }
+                                    scope 39 (inlined NonNull::::offset) {
+                                        let mut _24: *const T;
+                                        let mut _25: *const T;
+                                        scope 40 (inlined NonNull::::as_ptr) {
+                                        }
+                                    }
+                                }
+                            }
+                            scope 33 (inlined std::ptr::mut_ptr::::cast::) {
+                            }
+                            scope 34 (inlined std::ptr::mut_ptr::::cast::>) {
+                            }
+                        }
+                    }
+                    scope 41 (inlined NonNull::::as_ref::<'_>) {
+                        let _31: *const T;
+                        scope 42 (inlined NonNull::::as_ptr) {
+                        }
+                        scope 43 (inlined std::ptr::mut_ptr::::cast_const) {
+                        }
+                    }
+                }
+            }
         }
     }
     scope 3 (inlined core::slice::::iter) {
@@ -27,7 +95,6 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
             let _3: usize;
             let mut _7: *mut T;
             let mut _8: *mut T;
-            let mut _10: *const T;
             scope 5 {
                 let _6: std::ptr::NonNull;
                 scope 6 {
@@ -62,17 +129,17 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb0: {
-        StorageLive(_11);
+        StorageLive(_10);
         StorageLive(_3);
         StorageLive(_6);
+        StorageLive(_9);
         StorageLive(_4);
         _3 = PtrMetadata(copy _1);
         _4 = &raw const (*_1);
         StorageLive(_5);
         _5 = copy _4 as *const T (PtrToPtr);
-        _6 = NonNull:: { pointer: move _5 };
+        _6 = NonNull:: { pointer: copy _5 };
         StorageDead(_5);
-        StorageLive(_9);
         switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
@@ -82,7 +149,7 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
         _7 = copy _4 as *mut T (PtrToPtr);
         _8 = Offset(copy _7, copy _3);
         StorageDead(_7);
-        _9 = move _8 as *const T (PtrToPtr);
+        _9 = copy _8 as *const T (PtrToPtr);
         StorageDead(_8);
         goto -> bb3;
     }
@@ -93,69 +160,157 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () {
     }
 
     bb3: {
-        StorageLive(_10);
-        _10 = copy _9;
-        _11 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: move _10, _marker: const ZeroSized: PhantomData<&T> };
-        StorageDead(_10);
-        StorageDead(_9);
+        _10 = std::slice::Iter::<'_, T> { ptr: copy _6, end_or_len: copy _9, _marker: const ZeroSized: PhantomData<&T> };
         StorageDead(_4);
+        StorageDead(_9);
         StorageDead(_6);
         StorageDead(_3);
-        _12 = Rev::> { iter: copy _11 };
-        StorageDead(_11);
-        StorageLive(_13);
-        _13 = copy _12;
+        _11 = Rev::> { iter: copy _10 };
+        StorageDead(_10);
+        StorageLive(_12);
+        _12 = copy _11;
         goto -> bb4;
     }
 
     bb4: {
-        StorageLive(_15);
+        StorageLive(_33);
+        StorageLive(_20);
+        StorageLive(_19);
         StorageLive(_14);
-        _14 = &mut (_13.0: std::slice::Iter<'_, T>);
-        _15 =  as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind: bb11];
+        StorageLive(_32);
+        StorageLive(_18);
+        switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6];
     }
 
     bb5: {
-        StorageDead(_14);
-        _16 = discriminant(_15);
-        switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10];
+        StorageLive(_13);
+        _13 = copy ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _14 = copy _13 as std::ptr::NonNull (Transmute);
+        StorageDead(_13);
+        StorageLive(_16);
+        StorageLive(_15);
+        _15 = copy ((_12.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull);
+        _16 = copy _15 as *mut T (Transmute);
+        StorageDead(_15);
+        StorageLive(_17);
+        _17 = copy _14 as *mut T (Transmute);
+        _18 = Eq(copy _16, copy _17);
+        StorageDead(_17);
+        StorageDead(_16);
+        goto -> bb7;
     }
 
     bb6: {
-        StorageDead(_15);
-        StorageDead(_13);
-        drop(_2) -> [return: bb7, unwind continue];
+        _19 = copy ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _20 = copy _19 as usize (Transmute);
+        _18 = Eq(copy _20, const 0_usize);
+        goto -> bb7;
     }
 
     bb7: {
-        return;
+        switchInt(move _18) -> [0: bb8, otherwise: bb17];
     }
 
     bb8: {
-        _17 = copy ((_15 as Some).0: &T);
-        StorageLive(_18);
-        _18 = &_2;
-        StorageLive(_19);
-        _19 = (copy _17,);
-        _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11];
+        StorageLive(_26);
+        StorageLive(_28);
+        StorageLive(_22);
+        StorageLive(_23);
+        switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb12];
     }
 
     bb9: {
-        StorageDead(_19);
-        StorageDead(_18);
-        StorageDead(_15);
-        goto -> bb4;
+        StorageLive(_21);
+        _21 = &raw mut ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _22 = copy _21 as *mut std::ptr::NonNull (PtrToPtr);
+        StorageDead(_21);
+        _23 = copy (*_22);
+        switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11];
     }
 
     bb10: {
-        unreachable;
+        StorageLive(_25);
+        StorageLive(_24);
+        _24 = copy _23 as *const T (Transmute);
+        _25 = Offset(copy _24, const -1_isize);
+        StorageDead(_24);
+        _23 = NonNull:: { pointer: copy _25 };
+        StorageDead(_25);
+        goto -> bb11;
     }
 
-    bb11 (cleanup): {
-        drop(_2) -> [return: bb12, unwind terminate(cleanup)];
+    bb11: {
+        (*_22) = move _23;
+        _26 = copy (*_22);
+        goto -> bb13;
     }
 
-    bb12 (cleanup): {
+    bb12: {
+        StorageLive(_27);
+        _27 = &raw mut ((_12.0: std::slice::Iter<'_, T>).1: *const T);
+        _28 = copy _27 as *mut usize (PtrToPtr);
+        StorageDead(_27);
+        StorageLive(_30);
+        StorageLive(_29);
+        _29 = copy (*_28);
+        _30 = SubUnchecked(move _29, const 1_usize);
+        StorageDead(_29);
+        (*_28) = move _30;
+        StorageDead(_30);
+        _26 = copy ((_12.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull);
+        goto -> bb13;
+    }
+
+    bb13: {
+        StorageDead(_23);
+        StorageDead(_22);
+        StorageDead(_28);
+        StorageLive(_31);
+        _31 = copy _26 as *const T (Transmute);
+        _32 = &(*_31);
+        StorageDead(_31);
+        StorageDead(_26);
+        _33 = Option::<&T>::Some(copy _32);
+        StorageDead(_18);
+        StorageDead(_32);
+        StorageDead(_14);
+        StorageDead(_19);
+        StorageDead(_20);
+        _34 = copy ((_33 as Some).0: &T);
+        StorageLive(_35);
+        _35 = &_2;
+        StorageLive(_36);
+        _36 = (copy _34,);
+        _37 = >::call(move _35, move _36) -> [return: bb14, unwind: bb15];
+    }
+
+    bb14: {
+        StorageDead(_36);
+        StorageDead(_35);
+        StorageDead(_33);
+        goto -> bb4;
+    }
+
+    bb15 (cleanup): {
+        drop(_2) -> [return: bb16, unwind terminate(cleanup)];
+    }
+
+    bb16 (cleanup): {
         resume;
     }
+
+    bb17: {
+        StorageDead(_18);
+        StorageDead(_32);
+        StorageDead(_14);
+        StorageDead(_19);
+        StorageDead(_20);
+        StorageDead(_33);
+        StorageDead(_12);
+        drop(_2) -> [return: bb18, unwind continue];
+    }
+
+    bb18: {
+        return;
+    }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir
index 38d00cfbabdd..9b510380b10b 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir
@@ -39,7 +39,7 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
     bb1: {
         StorageLive(_2);
         _2 = copy ((*_1).1: *const T);
-        _3 = move _2 as std::ptr::NonNull (Transmute);
+        _3 = copy _2 as std::ptr::NonNull (Transmute);
         StorageDead(_2);
         StorageLive(_5);
         StorageLive(_4);
@@ -48,7 +48,7 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
         StorageDead(_4);
         StorageLive(_6);
         _6 = copy _3 as *mut T (Transmute);
-        _0 = Eq(move _5, move _6);
+        _0 = Eq(copy _5, copy _6);
         StorageDead(_6);
         StorageDead(_5);
         goto -> bb3;
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir
index 38d00cfbabdd..9b510380b10b 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir
@@ -39,7 +39,7 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
     bb1: {
         StorageLive(_2);
         _2 = copy ((*_1).1: *const T);
-        _3 = move _2 as std::ptr::NonNull (Transmute);
+        _3 = copy _2 as std::ptr::NonNull (Transmute);
         StorageDead(_2);
         StorageLive(_5);
         StorageLive(_4);
@@ -48,7 +48,7 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
         StorageDead(_4);
         StorageLive(_6);
         _6 = copy _3 as *mut T (Transmute);
-        _0 = Eq(move _5, move _6);
+        _0 = Eq(copy _5, copy _6);
         StorageDead(_6);
         StorageDead(_5);
         goto -> bb3;
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir
index 78f96bf41955..62b738c36bf4 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir
@@ -3,12 +3,187 @@
 fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> {
     debug it => _1;
     let mut _0: std::option::Option<&mut T>;
+    scope 1 (inlined  as DoubleEndedIterator>::next_back) {
+        let mut _2: *mut T;
+        let mut _7: bool;
+        let mut _8: *mut T;
+        let mut _21: &mut T;
+        scope 2 {
+            let _3: std::ptr::NonNull;
+            let _9: usize;
+            scope 3 {
+            }
+            scope 4 {
+                scope 7 (inlined  as PartialEq>::eq) {
+                    let mut _4: std::ptr::NonNull;
+                    let mut _5: *mut T;
+                    let mut _6: *mut T;
+                    scope 8 (inlined NonNull::::as_ptr) {
+                    }
+                    scope 9 (inlined NonNull::::as_ptr) {
+                    }
+                }
+            }
+            scope 5 (inlined std::ptr::mut_ptr::::addr) {
+                scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) {
+                }
+            }
+        }
+        scope 10 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) {
+            let mut _15: std::ptr::NonNull;
+            scope 11 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) {
+                let mut _10: *mut *mut T;
+                let mut _11: *mut std::ptr::NonNull;
+                let mut _12: std::ptr::NonNull;
+                let mut _16: *mut *mut T;
+                let mut _17: *mut usize;
+                let mut _18: usize;
+                let mut _19: usize;
+                scope 12 {
+                    scope 13 {
+                    }
+                    scope 14 {
+                        scope 17 (inlined NonNull::::sub) {
+                            scope 18 (inlined #[track_caller] core::num::::unchecked_neg) {
+                                scope 19 (inlined core::ub_checks::check_language_ub) {
+                                    scope 20 (inlined core::ub_checks::check_language_ub::runtime) {
+                                    }
+                                }
+                            }
+                            scope 21 (inlined NonNull::::offset) {
+                                let mut _13: *const T;
+                                let mut _14: *const T;
+                                scope 22 (inlined NonNull::::as_ptr) {
+                                }
+                            }
+                        }
+                    }
+                    scope 15 (inlined std::ptr::mut_ptr::::cast::) {
+                    }
+                    scope 16 (inlined std::ptr::mut_ptr::::cast::>) {
+                    }
+                }
+            }
+            scope 23 (inlined NonNull::::as_mut::<'_>) {
+                let mut _20: *mut T;
+                scope 24 (inlined NonNull::::as_ptr) {
+                }
+            }
+        }
+    }
 
     bb0: {
-        _0 =  as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind unreachable];
+        StorageLive(_9);
+        StorageLive(_8);
+        StorageLive(_3);
+        StorageLive(_2);
+        StorageLive(_21);
+        StorageLive(_7);
+        switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
     bb1: {
+        _2 = copy ((*_1).1: *mut T);
+        _3 = copy _2 as std::ptr::NonNull (Transmute);
+        StorageLive(_5);
+        StorageLive(_4);
+        _4 = copy ((*_1).0: std::ptr::NonNull);
+        _5 = copy _4 as *mut T (Transmute);
+        StorageDead(_4);
+        StorageLive(_6);
+        _6 = copy _3 as *mut T (Transmute);
+        _7 = Eq(copy _5, copy _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        goto -> bb3;
+    }
+
+    bb2: {
+        _8 = copy ((*_1).1: *mut T);
+        _9 = copy _8 as usize (Transmute);
+        _7 = Eq(copy _9, const 0_usize);
+        goto -> bb3;
+    }
+
+    bb3: {
+        switchInt(move _7) -> [0: bb4, otherwise: bb10];
+    }
+
+    bb4: {
+        StorageLive(_15);
+        StorageLive(_17);
+        StorageLive(_11);
+        StorageLive(_12);
+        switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8];
+    }
+
+    bb5: {
+        StorageLive(_10);
+        _10 = &raw mut ((*_1).1: *mut T);
+        _11 = copy _10 as *mut std::ptr::NonNull (PtrToPtr);
+        StorageDead(_10);
+        _12 = copy (*_11);
+        switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7];
+    }
+
+    bb6: {
+        StorageLive(_14);
+        StorageLive(_13);
+        _13 = copy _12 as *const T (Transmute);
+        _14 = Offset(copy _13, const -1_isize);
+        StorageDead(_13);
+        _12 = NonNull:: { pointer: copy _14 };
+        StorageDead(_14);
+        goto -> bb7;
+    }
+
+    bb7: {
+        (*_11) = move _12;
+        _15 = copy (*_11);
+        goto -> bb9;
+    }
+
+    bb8: {
+        StorageLive(_16);
+        _16 = &raw mut ((*_1).1: *mut T);
+        _17 = copy _16 as *mut usize (PtrToPtr);
+        StorageDead(_16);
+        StorageLive(_19);
+        StorageLive(_18);
+        _18 = copy (*_17);
+        _19 = SubUnchecked(move _18, const 1_usize);
+        StorageDead(_18);
+        (*_17) = move _19;
+        StorageDead(_19);
+        _15 = copy ((*_1).0: std::ptr::NonNull);
+        goto -> bb9;
+    }
+
+    bb9: {
+        StorageDead(_12);
+        StorageDead(_11);
+        StorageDead(_17);
+        StorageLive(_20);
+        _20 = copy _15 as *mut T (Transmute);
+        _21 = &mut (*_20);
+        StorageDead(_20);
+        StorageDead(_15);
+        _0 = Option::<&mut T>::Some(copy _21);
+        goto -> bb11;
+    }
+
+    bb10: {
+        _0 = const {transmute(0x0000000000000000): Option<&mut T>};
+        goto -> bb11;
+    }
+
+    bb11: {
+        StorageDead(_7);
+        StorageDead(_21);
+        StorageDead(_2);
+        StorageDead(_3);
+        StorageDead(_8);
+        StorageDead(_9);
         return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir
index dfe5e206fada..62b738c36bf4 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir
@@ -3,12 +3,187 @@
 fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> {
     debug it => _1;
     let mut _0: std::option::Option<&mut T>;
+    scope 1 (inlined  as DoubleEndedIterator>::next_back) {
+        let mut _2: *mut T;
+        let mut _7: bool;
+        let mut _8: *mut T;
+        let mut _21: &mut T;
+        scope 2 {
+            let _3: std::ptr::NonNull;
+            let _9: usize;
+            scope 3 {
+            }
+            scope 4 {
+                scope 7 (inlined  as PartialEq>::eq) {
+                    let mut _4: std::ptr::NonNull;
+                    let mut _5: *mut T;
+                    let mut _6: *mut T;
+                    scope 8 (inlined NonNull::::as_ptr) {
+                    }
+                    scope 9 (inlined NonNull::::as_ptr) {
+                    }
+                }
+            }
+            scope 5 (inlined std::ptr::mut_ptr::::addr) {
+                scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) {
+                }
+            }
+        }
+        scope 10 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) {
+            let mut _15: std::ptr::NonNull;
+            scope 11 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) {
+                let mut _10: *mut *mut T;
+                let mut _11: *mut std::ptr::NonNull;
+                let mut _12: std::ptr::NonNull;
+                let mut _16: *mut *mut T;
+                let mut _17: *mut usize;
+                let mut _18: usize;
+                let mut _19: usize;
+                scope 12 {
+                    scope 13 {
+                    }
+                    scope 14 {
+                        scope 17 (inlined NonNull::::sub) {
+                            scope 18 (inlined #[track_caller] core::num::::unchecked_neg) {
+                                scope 19 (inlined core::ub_checks::check_language_ub) {
+                                    scope 20 (inlined core::ub_checks::check_language_ub::runtime) {
+                                    }
+                                }
+                            }
+                            scope 21 (inlined NonNull::::offset) {
+                                let mut _13: *const T;
+                                let mut _14: *const T;
+                                scope 22 (inlined NonNull::::as_ptr) {
+                                }
+                            }
+                        }
+                    }
+                    scope 15 (inlined std::ptr::mut_ptr::::cast::) {
+                    }
+                    scope 16 (inlined std::ptr::mut_ptr::::cast::>) {
+                    }
+                }
+            }
+            scope 23 (inlined NonNull::::as_mut::<'_>) {
+                let mut _20: *mut T;
+                scope 24 (inlined NonNull::::as_ptr) {
+                }
+            }
+        }
+    }
 
     bb0: {
-        _0 =  as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind continue];
+        StorageLive(_9);
+        StorageLive(_8);
+        StorageLive(_3);
+        StorageLive(_2);
+        StorageLive(_21);
+        StorageLive(_7);
+        switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2];
     }
 
     bb1: {
+        _2 = copy ((*_1).1: *mut T);
+        _3 = copy _2 as std::ptr::NonNull (Transmute);
+        StorageLive(_5);
+        StorageLive(_4);
+        _4 = copy ((*_1).0: std::ptr::NonNull);
+        _5 = copy _4 as *mut T (Transmute);
+        StorageDead(_4);
+        StorageLive(_6);
+        _6 = copy _3 as *mut T (Transmute);
+        _7 = Eq(copy _5, copy _6);
+        StorageDead(_6);
+        StorageDead(_5);
+        goto -> bb3;
+    }
+
+    bb2: {
+        _8 = copy ((*_1).1: *mut T);
+        _9 = copy _8 as usize (Transmute);
+        _7 = Eq(copy _9, const 0_usize);
+        goto -> bb3;
+    }
+
+    bb3: {
+        switchInt(move _7) -> [0: bb4, otherwise: bb10];
+    }
+
+    bb4: {
+        StorageLive(_15);
+        StorageLive(_17);
+        StorageLive(_11);
+        StorageLive(_12);
+        switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb8];
+    }
+
+    bb5: {
+        StorageLive(_10);
+        _10 = &raw mut ((*_1).1: *mut T);
+        _11 = copy _10 as *mut std::ptr::NonNull (PtrToPtr);
+        StorageDead(_10);
+        _12 = copy (*_11);
+        switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7];
+    }
+
+    bb6: {
+        StorageLive(_14);
+        StorageLive(_13);
+        _13 = copy _12 as *const T (Transmute);
+        _14 = Offset(copy _13, const -1_isize);
+        StorageDead(_13);
+        _12 = NonNull:: { pointer: copy _14 };
+        StorageDead(_14);
+        goto -> bb7;
+    }
+
+    bb7: {
+        (*_11) = move _12;
+        _15 = copy (*_11);
+        goto -> bb9;
+    }
+
+    bb8: {
+        StorageLive(_16);
+        _16 = &raw mut ((*_1).1: *mut T);
+        _17 = copy _16 as *mut usize (PtrToPtr);
+        StorageDead(_16);
+        StorageLive(_19);
+        StorageLive(_18);
+        _18 = copy (*_17);
+        _19 = SubUnchecked(move _18, const 1_usize);
+        StorageDead(_18);
+        (*_17) = move _19;
+        StorageDead(_19);
+        _15 = copy ((*_1).0: std::ptr::NonNull);
+        goto -> bb9;
+    }
+
+    bb9: {
+        StorageDead(_12);
+        StorageDead(_11);
+        StorageDead(_17);
+        StorageLive(_20);
+        _20 = copy _15 as *mut T (Transmute);
+        _21 = &mut (*_20);
+        StorageDead(_20);
+        StorageDead(_15);
+        _0 = Option::<&mut T>::Some(copy _21);
+        goto -> bb11;
+    }
+
+    bb10: {
+        _0 = const {transmute(0x0000000000000000): Option<&mut T>};
+        goto -> bb11;
+    }
+
+    bb11: {
+        StorageDead(_7);
+        StorageDead(_21);
+        StorageDead(_2);
+        StorageDead(_3);
+        StorageDead(_8);
+        StorageDead(_9);
         return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir
index c0ed0aea1e26..cc0fce26149e 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-abort.mir
@@ -72,7 +72,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
         _5 = copy _2 as *mut T (Transmute);
         StorageLive(_6);
         _6 = copy _4 as *mut T (Transmute);
-        _7 = Eq(move _5, move _6);
+        _7 = Eq(copy _5, copy _6);
         StorageDead(_6);
         StorageDead(_5);
         switchInt(move _7) -> [0: bb2, otherwise: bb3];
@@ -84,9 +84,9 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
         StorageLive(_9);
         StorageLive(_8);
         _8 = copy _2 as *const T (Transmute);
-        _9 = Offset(move _8, const 1_usize);
+        _9 = Offset(copy _8, const 1_usize);
         StorageDead(_8);
-        _10 = NonNull:: { pointer: move _9 };
+        _10 = NonNull:: { pointer: copy _9 };
         StorageDead(_9);
         ((*_1).0: std::ptr::NonNull) = move _10;
         StorageDead(_10);
diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir
index c0ed0aea1e26..cc0fce26149e 100644
--- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_next.PreCodegen.after.panic-unwind.mir
@@ -72,7 +72,7 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
         _5 = copy _2 as *mut T (Transmute);
         StorageLive(_6);
         _6 = copy _4 as *mut T (Transmute);
-        _7 = Eq(move _5, move _6);
+        _7 = Eq(copy _5, copy _6);
         StorageDead(_6);
         StorageDead(_5);
         switchInt(move _7) -> [0: bb2, otherwise: bb3];
@@ -84,9 +84,9 @@ fn slice_iter_next(_1: &mut std::slice::Iter<'_, T>) -> Option<&T> {
         StorageLive(_9);
         StorageLive(_8);
         _8 = copy _2 as *const T (Transmute);
-        _9 = Offset(move _8, const 1_usize);
+        _9 = Offset(copy _8, const 1_usize);
         StorageDead(_8);
-        _10 = NonNull:: { pointer: move _9 };
+        _10 = NonNull:: { pointer: copy _9 };
         StorageDead(_9);
         ((*_1).0: std::ptr::NonNull) = move _10;
         StorageDead(_10);
diff --git a/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-abort.mir
index fe4e2deab870..79aa9c5ae1e3 100644
--- a/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-abort.mir
@@ -9,7 +9,7 @@ fn outer(_1: u8) -> u8 {
     }
 
     bb0: {
-        _2 = &_1;                        // scope 0 at $DIR/spans.rs:11:11: 11:13
+        // DBG: _2 = &_1;
         _0 = copy _1;                    // scope 1 at $DIR/spans.rs:15:5: 15:7
         return;                          // scope 0 at $DIR/spans.rs:12:2: 12:2
     }
diff --git a/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-unwind.mir
index fe4e2deab870..79aa9c5ae1e3 100644
--- a/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/pre-codegen/spans.outer.PreCodegen.after.panic-unwind.mir
@@ -9,7 +9,7 @@ fn outer(_1: u8) -> u8 {
     }
 
     bb0: {
-        _2 = &_1;                        // scope 0 at $DIR/spans.rs:11:11: 11:13
+        // DBG: _2 = &_1;
         _0 = copy _1;                    // scope 1 at $DIR/spans.rs:15:5: 15:7
         return;                          // scope 0 at $DIR/spans.rs:12:2: 12:2
     }
diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir
index c4d0e318b58d..29bfe962974c 100644
--- a/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_ge_partial.PreCodegen.after.mir
@@ -7,7 +7,6 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
     scope 1 (inlined std::cmp::impls::::ge) {
         scope 2 (inlined core::tuple::::ge) {
             let mut _7: std::ops::ControlFlow;
-            let _8: bool;
             scope 3 {
             }
             scope 4 (inlined std::cmp::impls::::__chaining_ge) {
@@ -19,8 +18,8 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
                 }
             }
             scope 6 (inlined std::cmp::impls::::ge) {
+                let mut _8: f32;
                 let mut _9: f32;
-                let mut _10: f32;
             }
         }
     }
@@ -44,10 +43,7 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_8);
-        _8 = copy ((_7 as Break).0: bool);
-        _0 = copy _8;
-        StorageDead(_8);
+        _0 = copy ((_7 as Break).0: bool);
         goto -> bb3;
     }
 
@@ -55,13 +51,13 @@ fn demo_ge_partial(_1: &(f32, f32), _2: &(f32, f32)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
+        StorageLive(_8);
+        _8 = copy ((*_1).1: f32);
         StorageLive(_9);
-        _9 = copy ((*_1).1: f32);
-        StorageLive(_10);
-        _10 = copy ((*_2).1: f32);
-        _0 = Ge(move _9, move _10);
-        StorageDead(_10);
+        _9 = copy ((*_2).1: f32);
+        _0 = Ge(move _8, move _9);
         StorageDead(_9);
+        StorageDead(_8);
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir
index 44df8b279935..7678c92a1f0c 100644
--- a/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/tuple_ord.demo_le_total.PreCodegen.after.mir
@@ -7,7 +7,6 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
     scope 1 (inlined std::cmp::impls::::le) {
         scope 2 (inlined core::tuple::::le) {
             let mut _7: std::ops::ControlFlow;
-            let _8: bool;
             scope 3 {
             }
             scope 4 (inlined std::cmp::impls::::__chaining_le) {
@@ -19,8 +18,8 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
                 }
             }
             scope 6 (inlined std::cmp::impls::::le) {
+                let mut _8: i16;
                 let mut _9: i16;
-                let mut _10: i16;
             }
         }
     }
@@ -44,10 +43,7 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
-        StorageLive(_8);
-        _8 = copy ((_7 as Break).0: bool);
-        _0 = copy _8;
-        StorageDead(_8);
+        _0 = copy ((_7 as Break).0: bool);
         goto -> bb3;
     }
 
@@ -55,13 +51,13 @@ fn demo_le_total(_1: &(u16, i16), _2: &(u16, i16)) -> bool {
         StorageDead(_5);
         StorageDead(_4);
         StorageDead(_3);
+        StorageLive(_8);
+        _8 = copy ((*_1).1: i16);
         StorageLive(_9);
-        _9 = copy ((*_1).1: i16);
-        StorageLive(_10);
-        _10 = copy ((*_2).1: i16);
-        _0 = Le(move _9, move _10);
-        StorageDead(_10);
+        _9 = copy ((*_2).1: i16);
+        _0 = Le(move _8, move _9);
         StorageDead(_9);
+        StorageDead(_8);
         goto -> bb3;
     }
 
diff --git a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.rs b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.rs
new file mode 100644
index 000000000000..7b742b956ae5
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.rs
@@ -0,0 +1,15 @@
+//@ compile-flags: -O
+
+#![crate_type = "lib"]
+
+// EMIT_MIR two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff
+// EMIT_MIR two_unwrap_unchecked.two_unwrap_unchecked.PreCodegen.after.mir
+pub fn two_unwrap_unchecked(v: &Option) -> i32 {
+    // CHECK-LABEL: fn two_unwrap_unchecked(
+    // CHECK: [[DEREF_V:_.*]] = copy (*_1);
+    // CHECK: [[V1V2:_.*]] = copy (([[DEREF_V]] as Some).0: i32);
+    // CHECK: _0 = Add(copy [[V1V2]], copy [[V1V2]]);
+    let v1 = unsafe { v.unwrap_unchecked() };
+    let v2 = unsafe { v.unwrap_unchecked() };
+    v1 + v2
+}
diff --git a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff
new file mode 100644
index 000000000000..5b063e6762e0
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.GVN.diff
@@ -0,0 +1,105 @@
+- // MIR for `two_unwrap_unchecked` before GVN
++ // MIR for `two_unwrap_unchecked` after GVN
+  
+  fn two_unwrap_unchecked(_1: &Option) -> i32 {
+      debug v => _1;
+      let mut _0: i32;
+      let _2: i32;
+      let mut _3: std::option::Option;
+      let mut _5: std::option::Option;
+      let mut _6: i32;
+      let mut _7: i32;
+      scope 1 {
+          debug v1 => _2;
+          let _4: i32;
+          scope 2 {
+              debug v2 => _4;
+          }
+          scope 8 (inlined #[track_caller] Option::::unwrap_unchecked) {
+              let mut _9: isize;
+              scope 9 {
+              }
+              scope 10 (inlined #[track_caller] unreachable_unchecked) {
+                  scope 11 (inlined core::ub_checks::check_language_ub) {
+                      scope 12 (inlined core::ub_checks::check_language_ub::runtime) {
+                      }
+                  }
+              }
+          }
+      }
+      scope 3 (inlined #[track_caller] Option::::unwrap_unchecked) {
+          let mut _8: isize;
+          scope 4 {
+          }
+          scope 5 (inlined #[track_caller] unreachable_unchecked) {
+              scope 6 (inlined core::ub_checks::check_language_ub) {
+                  scope 7 (inlined core::ub_checks::check_language_ub::runtime) {
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
+-         StorageLive(_3);
++         nop;
++         nop;
+          _3 = copy (*_1);
+-         StorageLive(_8);
++         nop;
+          _8 = discriminant(_3);
+-         switchInt(move _8) -> [0: bb2, 1: bb3, otherwise: bb1];
++         switchInt(copy _8) -> [0: bb2, 1: bb3, otherwise: bb1];
+      }
+  
+      bb1: {
+          unreachable;
+      }
+  
+      bb2: {
+          unreachable;
+      }
+  
+      bb3: {
+-         _2 = move ((_3 as Some).0: i32);
+-         StorageDead(_8);
+-         StorageDead(_3);
++         _2 = copy ((_3 as Some).0: i32);
++         nop;
++         nop;
+          StorageLive(_4);
+          StorageLive(_5);
+-         _5 = copy (*_1);
++         _5 = copy _3;
+          StorageLive(_9);
+-         _9 = discriminant(_5);
+-         switchInt(move _9) -> [0: bb4, 1: bb5, otherwise: bb1];
++         _9 = copy _8;
++         switchInt(copy _8) -> [0: bb4, 1: bb5, otherwise: bb1];
+      }
+  
+      bb4: {
+          unreachable;
+      }
+  
+      bb5: {
+-         _4 = move ((_5 as Some).0: i32);
++         _4 = copy _2;
+          StorageDead(_9);
+          StorageDead(_5);
+          StorageLive(_6);
+          _6 = copy _2;
+          StorageLive(_7);
+-         _7 = copy _4;
+-         _0 = Add(move _6, move _7);
++         _7 = copy _2;
++         _0 = Add(copy _2, copy _2);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_4);
+-         StorageDead(_2);
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.PreCodegen.after.mir
new file mode 100644
index 000000000000..b2b7f88d8534
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/two_unwrap_unchecked.two_unwrap_unchecked.PreCodegen.after.mir
@@ -0,0 +1,51 @@
+// MIR for `two_unwrap_unchecked` after PreCodegen
+
+fn two_unwrap_unchecked(_1: &Option) -> i32 {
+    debug v => _1;
+    let mut _0: i32;
+    let mut _2: std::option::Option;
+    let _4: i32;
+    scope 1 {
+        debug v1 => _4;
+        scope 2 {
+            debug v2 => _4;
+        }
+        scope 8 (inlined #[track_caller] Option::::unwrap_unchecked) {
+            scope 9 {
+            }
+            scope 10 (inlined #[track_caller] unreachable_unchecked) {
+                scope 11 (inlined core::ub_checks::check_language_ub) {
+                    scope 12 (inlined core::ub_checks::check_language_ub::runtime) {
+                    }
+                }
+            }
+        }
+    }
+    scope 3 (inlined #[track_caller] Option::::unwrap_unchecked) {
+        let mut _3: isize;
+        scope 4 {
+        }
+        scope 5 (inlined #[track_caller] unreachable_unchecked) {
+            scope 6 (inlined core::ub_checks::check_language_ub) {
+                scope 7 (inlined core::ub_checks::check_language_ub::runtime) {
+                }
+            }
+        }
+    }
+
+    bb0: {
+        _2 = copy (*_1);
+        _3 = discriminant(_2);
+        switchInt(copy _3) -> [0: bb2, 1: bb1, otherwise: bb2];
+    }
+
+    bb1: {
+        _4 = copy ((_2 as Some).0: i32);
+        _0 = Add(copy _4, copy _4);
+        return;
+    }
+
+    bb2: {
+        unreachable;
+    }
+}
diff --git a/tests/mir-opt/reference_prop.debuginfo.ReferencePropagation.diff b/tests/mir-opt/reference_prop.debuginfo.ReferencePropagation.diff
index 05ad9dbf3ccc..375b6096d88d 100644
--- a/tests/mir-opt/reference_prop.debuginfo.ReferencePropagation.diff
+++ b/tests/mir-opt/reference_prop.debuginfo.ReferencePropagation.diff
@@ -17,14 +17,15 @@
       let mut _15: std::ops::RangeFull;
       let mut _16: usize;
       let mut _17: usize;
-      let mut _18: bool;
-      let _23: &&mut u8;
-      let _24: &mut u8;
-      let mut _25: debuginfo::T;
+      let mut _18: usize;
+      let mut _19: bool;
+      let _24: &&mut u8;
+      let _25: &mut u8;
+      let mut _26: debuginfo::T;
       scope 1 {
           debug ref_mut_u8 => _1;
           let _3: &u8;
-          let mut _28: &debuginfo::T;
+          let mut _29: &debuginfo::T;
           scope 2 {
               debug field => _3;
               let _5: &u8;
@@ -32,22 +33,22 @@
 -                 debug reborrow => _5;
 +                 debug reborrow => _1;
                   let _9: &i32;
-                  let _22: &&&mut u8;
-                  let mut _27: &std::option::Option;
+                  let _23: &&&mut u8;
+                  let mut _28: &std::option::Option;
                   scope 4 {
                       debug variant_field => _9;
                   }
                   scope 5 {
-                      debug constant_index => _19;
-                      debug subslice => _20;
-                      debug constant_index_from_end => _21;
-                      let _19: &i32;
-                      let _20: &[i32];
-                      let _21: &i32;
-                      let mut _26: &[i32; 10];
+                      debug constant_index => _20;
+                      debug subslice => _21;
+                      debug constant_index_from_end => _22;
+                      let _20: &i32;
+                      let _21: &[i32];
+                      let _22: &i32;
+                      let mut _27: &[i32; 10];
                   }
                   scope 6 {
-                      debug multiple_borrow => _22;
+                      debug multiple_borrow => _23;
                   }
               }
           }
@@ -59,8 +60,8 @@
           _2 = const 5_u8;
           _1 = &mut _2;
           StorageLive(_3);
-          _28 = const debuginfo::promoted[2];
-          _3 = &((*_28).0: u8);
+          _29 = const debuginfo::promoted[2];
+          _3 = &((*_29).0: u8);
 -         StorageLive(_5);
 -         _5 = &(*_1);
 -         StorageLive(_6);
@@ -76,8 +77,8 @@
   
       bb2: {
           StorageLive(_9);
-          _27 = const debuginfo::promoted[1];
-          _9 = &(((*_27) as Some).0: i32);
+          _28 = const debuginfo::promoted[1];
+          _9 = &(((*_28) as Some).0: i32);
 -         _6 = const ();
           StorageDead(_9);
           goto -> bb4;
@@ -92,41 +93,42 @@
           StorageDead(_7);
 -         StorageDead(_6);
 -         StorageLive(_10);
--         StorageLive(_11);
+          StorageLive(_11);
 -         StorageLive(_12);
           StorageLive(_13);
-          _26 = const debuginfo::promoted[0];
-          _13 = &(*_26);
+          _27 = const debuginfo::promoted[0];
+          _13 = &(*_27);
           StorageLive(_15);
           _15 = RangeFull;
-          _12 = <[i32; 10] as Index>::index(move _13, move _15) -> [return: bb5, unwind continue];
+-         _12 = <[i32; 10] as Index>::index(move _13, move _15) -> [return: bb5, unwind continue];
++         _12 = <[i32; 10] as Index>::index(copy _13, move _15) -> [return: bb5, unwind continue];
       }
   
       bb5: {
           StorageDead(_15);
           StorageDead(_13);
--         _11 = &(*_12);
--         _16 = Len((*_11));
-+         _16 = Len((*_12));
-          _17 = const 3_usize;
-          _18 = Ge(move _16, move _17);
-          switchInt(move _18) -> [0: bb7, otherwise: bb6];
+          _11 = &(*_12);
+          _17 = PtrMetadata(copy _11);
+          _16 = move _17;
+          _18 = const 3_usize;
+          _19 = Ge(move _16, move _18);
+          switchInt(move _19) -> [0: bb7, otherwise: bb6];
       }
   
       bb6: {
-          StorageLive(_19);
--         _19 = &(*_11)[1 of 3];
-+         _19 = &(*_12)[1 of 3];
           StorageLive(_20);
--         _20 = &(*_11)[2:-1];
-+         _20 = &(*_12)[2:-1];
+-         _20 = &(*_11)[1 of 3];
++         _20 = &(*_12)[1 of 3];
           StorageLive(_21);
--         _21 = &(*_11)[-1 of 3];
+-         _21 = &(*_11)[2:-1];
++         _21 = &(*_12)[2:-1];
+          StorageLive(_22);
+-         _22 = &(*_11)[-1 of 3];
 -         _10 = const ();
-+         _21 = &(*_12)[-1 of 3];
++         _22 = &(*_12)[-1 of 3];
+          StorageDead(_22);
           StorageDead(_21);
           StorageDead(_20);
-          StorageDead(_19);
           goto -> bb8;
       }
   
@@ -137,21 +139,21 @@
   
       bb8: {
 -         StorageDead(_12);
--         StorageDead(_11);
+          StorageDead(_11);
 -         StorageDead(_10);
-          StorageLive(_22);
           StorageLive(_23);
           StorageLive(_24);
           StorageLive(_25);
-          _25 = T(const 6_u8);
-          _24 = &mut (_25.0: u8);
+          StorageLive(_26);
+          _26 = T(const 6_u8);
+          _25 = &mut (_26.0: u8);
+          _24 = &_25;
           _23 = &_24;
-          _22 = &_23;
           _0 = const ();
+          StorageDead(_26);
           StorageDead(_25);
           StorageDead(_24);
           StorageDead(_23);
-          StorageDead(_22);
 -         StorageDead(_5);
           StorageDead(_3);
           StorageDead(_2);
diff --git a/tests/mir-opt/reference_prop.reference_propagation.ReferencePropagation.diff b/tests/mir-opt/reference_prop.reference_propagation.ReferencePropagation.diff
index 3c6a9a9614c3..0f90cc40a3d8 100644
--- a/tests/mir-opt/reference_prop.reference_propagation.ReferencePropagation.diff
+++ b/tests/mir-opt/reference_prop.reference_propagation.ReferencePropagation.diff
@@ -218,8 +218,9 @@
 -         StorageLive(_14);
 -         _14 = &_11;
 -         _13 = &(*_14);
+-         _12 = move _13;
 +         _13 = &_11;
-          _12 = move _13;
++         _12 = copy _13;
           StorageDead(_13);
 -         StorageDead(_14);
           StorageLive(_15);
@@ -252,7 +253,8 @@
           StorageLive(_23);
           StorageLive(_24);
           _24 = copy _21;
-          _23 = opaque::<&&usize>(move _24) -> [return: bb3, unwind continue];
+-         _23 = opaque::<&&usize>(move _24) -> [return: bb3, unwind continue];
++         _23 = opaque::<&&usize>(copy _24) -> [return: bb3, unwind continue];
       }
   
       bb3: {
@@ -276,7 +278,8 @@
           StorageLive(_30);
           StorageLive(_31);
           _31 = copy _28;
-          _30 = opaque::<*mut &usize>(move _31) -> [return: bb4, unwind continue];
+-         _30 = opaque::<*mut &usize>(move _31) -> [return: bb4, unwind continue];
++         _30 = opaque::<*mut &usize>(copy _31) -> [return: bb4, unwind continue];
       }
   
       bb4: {
@@ -299,7 +302,8 @@
           StorageLive(_36);
           StorageLive(_37);
           _37 = copy _34;
-          _36 = opaque::<&usize>(move _37) -> [return: bb5, unwind continue];
+-         _36 = opaque::<&usize>(move _37) -> [return: bb5, unwind continue];
++         _36 = opaque::<&usize>(copy _37) -> [return: bb5, unwind continue];
       }
   
       bb5: {
@@ -328,7 +332,8 @@
           StorageLive(_45);
           StorageLive(_46);
           _46 = copy _44;
-          _45 = opaque::<&usize>(move _46) -> [return: bb6, unwind continue];
+-         _45 = opaque::<&usize>(move _46) -> [return: bb6, unwind continue];
++         _45 = opaque::<&usize>(copy _46) -> [return: bb6, unwind continue];
       }
   
       bb6: {
@@ -368,8 +373,9 @@
 -         StorageLive(_55);
 -         _55 = &(*_1);
 -         _54 = &(*_55);
+-         _2 = move _54;
 +         _54 = &(*_1);
-          _2 = move _54;
++         _2 = copy _54;
           StorageDead(_54);
 -         StorageDead(_55);
           StorageLive(_56);
diff --git a/tests/mir-opt/reference_prop.reference_propagation_const_ptr.ReferencePropagation.diff b/tests/mir-opt/reference_prop.reference_propagation_const_ptr.ReferencePropagation.diff
index 75fe99de9381..99ef07a212cf 100644
--- a/tests/mir-opt/reference_prop.reference_propagation_const_ptr.ReferencePropagation.diff
+++ b/tests/mir-opt/reference_prop.reference_propagation_const_ptr.ReferencePropagation.diff
@@ -233,7 +233,8 @@
           _12 = &raw const _10;
           StorageLive(_13);
           _13 = &raw const _11;
-          _12 = move _13;
+-         _12 = move _13;
++         _12 = copy _13;
           StorageDead(_13);
           StorageLive(_14);
           _14 = copy (*_12);
@@ -265,7 +266,8 @@
           StorageLive(_22);
           StorageLive(_23);
           _23 = copy _20;
-          _22 = opaque::<&*const usize>(move _23) -> [return: bb3, unwind continue];
+-         _22 = opaque::<&*const usize>(move _23) -> [return: bb3, unwind continue];
++         _22 = opaque::<&*const usize>(copy _23) -> [return: bb3, unwind continue];
       }
   
       bb3: {
@@ -289,7 +291,8 @@
           StorageLive(_29);
           StorageLive(_30);
           _30 = copy _27;
-          _29 = opaque::<*mut *const usize>(move _30) -> [return: bb4, unwind continue];
+-         _29 = opaque::<*mut *const usize>(move _30) -> [return: bb4, unwind continue];
++         _29 = opaque::<*mut *const usize>(copy _30) -> [return: bb4, unwind continue];
       }
   
       bb4: {
@@ -312,7 +315,8 @@
           StorageLive(_35);
           StorageLive(_36);
           _36 = copy _33;
-          _35 = opaque::<*const usize>(move _36) -> [return: bb5, unwind continue];
+-         _35 = opaque::<*const usize>(move _36) -> [return: bb5, unwind continue];
++         _35 = opaque::<*const usize>(copy _36) -> [return: bb5, unwind continue];
       }
   
       bb5: {
@@ -341,7 +345,8 @@
           StorageLive(_44);
           StorageLive(_45);
           _45 = copy _43;
-          _44 = opaque::<*const usize>(move _45) -> [return: bb6, unwind continue];
+-         _44 = opaque::<*const usize>(move _45) -> [return: bb6, unwind continue];
++         _44 = opaque::<*const usize>(copy _45) -> [return: bb6, unwind continue];
       }
   
       bb6: {
@@ -379,7 +384,8 @@
           _52 = &raw const (*_2);
           StorageLive(_53);
           _53 = &raw const (*_1);
-          _2 = move _53;
+-         _2 = move _53;
++         _2 = copy _53;
           StorageDead(_53);
           StorageLive(_54);
           _54 = copy (*_52);
diff --git a/tests/mir-opt/reference_prop.reference_propagation_mut.ReferencePropagation.diff b/tests/mir-opt/reference_prop.reference_propagation_mut.ReferencePropagation.diff
index f35b4974d6eb..e2fab8a5f2e4 100644
--- a/tests/mir-opt/reference_prop.reference_propagation_mut.ReferencePropagation.diff
+++ b/tests/mir-opt/reference_prop.reference_propagation_mut.ReferencePropagation.diff
@@ -218,8 +218,9 @@
 -         StorageLive(_14);
 -         _14 = &mut _11;
 -         _13 = &mut (*_14);
+-         _12 = move _13;
 +         _13 = &mut _11;
-          _12 = move _13;
++         _12 = copy _13;
           StorageDead(_13);
 -         StorageDead(_14);
           StorageLive(_15);
@@ -251,7 +252,8 @@
           StorageLive(_23);
           StorageLive(_24);
           _24 = copy _21;
-          _23 = opaque::<&&mut usize>(move _24) -> [return: bb3, unwind continue];
+-         _23 = opaque::<&&mut usize>(move _24) -> [return: bb3, unwind continue];
++         _23 = opaque::<&&mut usize>(copy _24) -> [return: bb3, unwind continue];
       }
   
       bb3: {
@@ -275,7 +277,8 @@
           StorageLive(_30);
           StorageLive(_31);
           _31 = copy _28;
-          _30 = opaque::<*mut &mut usize>(move _31) -> [return: bb4, unwind continue];
+-         _30 = opaque::<*mut &mut usize>(move _31) -> [return: bb4, unwind continue];
++         _30 = opaque::<*mut &mut usize>(copy _31) -> [return: bb4, unwind continue];
       }
   
       bb4: {
@@ -296,8 +299,10 @@
           _35 = copy (*_34);
           StorageLive(_36);
           StorageLive(_37);
-          _37 = move _34;
-          _36 = opaque::<&mut usize>(move _37) -> [return: bb5, unwind continue];
+-         _37 = move _34;
+-         _36 = opaque::<&mut usize>(move _37) -> [return: bb5, unwind continue];
++         _37 = copy _34;
++         _36 = opaque::<&mut usize>(copy _37) -> [return: bb5, unwind continue];
       }
   
       bb5: {
@@ -316,15 +321,19 @@
           StorageLive(_41);
           _41 = copy (*_40);
           StorageLive(_42);
-          _42 = move _40;
+-         _42 = move _40;
++         _42 = copy _40;
           StorageLive(_43);
           _43 = copy (*_42);
           StorageLive(_44);
-          _44 = move _42;
+-         _44 = move _42;
++         _44 = copy _42;
           StorageLive(_45);
           StorageLive(_46);
-          _46 = move _44;
-          _45 = opaque::<&mut usize>(move _46) -> [return: bb6, unwind continue];
+-         _46 = move _44;
+-         _45 = opaque::<&mut usize>(move _46) -> [return: bb6, unwind continue];
++         _46 = copy _44;
++         _45 = opaque::<&mut usize>(copy _46) -> [return: bb6, unwind continue];
       }
   
       bb6: {
@@ -364,8 +373,9 @@
 -         StorageLive(_55);
 -         _55 = &mut (*_1);
 -         _54 = &mut (*_55);
+-         _2 = move _54;
 +         _54 = &mut (*_1);
-          _2 = move _54;
++         _2 = copy _54;
           StorageDead(_54);
 -         StorageDead(_55);
           StorageLive(_56);
diff --git a/tests/mir-opt/reference_prop.reference_propagation_mut_ptr.ReferencePropagation.diff b/tests/mir-opt/reference_prop.reference_propagation_mut_ptr.ReferencePropagation.diff
index 21b322b72187..c49254ee6c65 100644
--- a/tests/mir-opt/reference_prop.reference_propagation_mut_ptr.ReferencePropagation.diff
+++ b/tests/mir-opt/reference_prop.reference_propagation_mut_ptr.ReferencePropagation.diff
@@ -214,7 +214,8 @@
           _12 = &raw mut _10;
           StorageLive(_13);
           _13 = &raw mut _11;
-          _12 = move _13;
+-         _12 = move _13;
++         _12 = copy _13;
           StorageDead(_13);
           StorageLive(_14);
           _14 = copy (*_12);
@@ -245,7 +246,8 @@
           StorageLive(_22);
           StorageLive(_23);
           _23 = copy _20;
-          _22 = opaque::<&*mut usize>(move _23) -> [return: bb3, unwind continue];
+-         _22 = opaque::<&*mut usize>(move _23) -> [return: bb3, unwind continue];
++         _22 = opaque::<&*mut usize>(copy _23) -> [return: bb3, unwind continue];
       }
   
       bb3: {
@@ -269,7 +271,8 @@
           StorageLive(_29);
           StorageLive(_30);
           _30 = copy _27;
-          _29 = opaque::<*mut *mut usize>(move _30) -> [return: bb4, unwind continue];
+-         _29 = opaque::<*mut *mut usize>(move _30) -> [return: bb4, unwind continue];
++         _29 = opaque::<*mut *mut usize>(copy _30) -> [return: bb4, unwind continue];
       }
   
       bb4: {
@@ -291,7 +294,8 @@
           StorageLive(_35);
           StorageLive(_36);
           _36 = copy _33;
-          _35 = opaque::<*mut usize>(move _36) -> [return: bb5, unwind continue];
+-         _35 = opaque::<*mut usize>(move _36) -> [return: bb5, unwind continue];
++         _35 = opaque::<*mut usize>(copy _36) -> [return: bb5, unwind continue];
       }
   
       bb5: {
@@ -318,7 +322,8 @@
           StorageLive(_44);
           StorageLive(_45);
           _45 = copy _43;
-          _44 = opaque::<*mut usize>(move _45) -> [return: bb6, unwind continue];
+-         _44 = opaque::<*mut usize>(move _45) -> [return: bb6, unwind continue];
++         _44 = opaque::<*mut usize>(copy _45) -> [return: bb6, unwind continue];
       }
   
       bb6: {
@@ -356,7 +361,8 @@
           _52 = &raw mut (*_2);
           StorageLive(_53);
           _53 = &raw mut (*_1);
-          _2 = move _53;
+-         _2 = move _53;
++         _2 = copy _53;
           StorageDead(_53);
           StorageLive(_54);
           _54 = copy (*_52);
diff --git a/tests/mir-opt/reference_prop.rs b/tests/mir-opt/reference_prop.rs
index 00d489380716..c4b63b6313c2 100644
--- a/tests/mir-opt/reference_prop.rs
+++ b/tests/mir-opt/reference_prop.rs
@@ -30,7 +30,7 @@ fn reference_propagation<'a, T: Copy>(single: &'a T, mut multiple: &'a T) {
         // CHECK: [[a2:_.*]] = const 7_usize;
         // CHECK: [[b:_.*]] = &[[a]];
         // CHECK: [[btmp:_.*]] = &[[a2]];
-        // CHECK: [[b]] = move [[btmp]];
+        // CHECK: [[b]] = copy [[btmp]];
         // CHECK: [[c:_.*]] = copy (*[[b]]);
 
         let a = 5_usize;
@@ -122,7 +122,7 @@ fn reference_propagation<'a, T: Copy>(single: &'a T, mut multiple: &'a T) {
         // CHECK: bb7: {
         // CHECK: [[a:_.*]] = &(*_2);
         // CHECK: [[tmp:_.*]] = &(*_1);
-        // CHECK: _2 = move [[tmp]];
+        // CHECK: _2 = copy [[tmp]];
         // CHECK: [[b:_.*]] = copy (*[[a]]);
 
         let a = &*multiple;
@@ -186,7 +186,7 @@ fn reference_propagation_mut<'a, T: Copy>(single: &'a mut T, mut multiple: &'a m
         // CHECK: [[a2:_.*]] = const 7_usize;
         // CHECK: [[b:_.*]] = &mut [[a]];
         // CHECK: [[btmp:_.*]] = &mut [[a2]];
-        // CHECK: [[b]] = move [[btmp]];
+        // CHECK: [[b]] = copy [[btmp]];
         // CHECK: [[c:_.*]] = copy (*[[b]]);
 
         let mut a = 5_usize;
@@ -247,9 +247,9 @@ fn reference_propagation_mut<'a, T: Copy>(single: &'a mut T, mut multiple: &'a m
         // CHECK: [[a:_.*]] = const 7_usize;
         // CHECK: [[b1:_.*]] = &mut [[a]];
         // CHECK: [[c:_.*]] = copy (*[[b1]]);
-        // CHECK: [[b2:_.*]] = move [[b1]];
+        // CHECK: [[b2:_.*]] = copy [[b1]];
         // CHECK: [[c2:_.*]] = copy (*[[b2]]);
-        // CHECK: [[b3:_.*]] = move [[b2]];
+        // CHECK: [[b3:_.*]] = copy [[b2]];
 
         let mut a = 7_usize;
         let b1 = &mut a;
@@ -278,7 +278,7 @@ fn reference_propagation_mut<'a, T: Copy>(single: &'a mut T, mut multiple: &'a m
         // CHECK: bb7: {
         // CHECK: [[a:_.*]] = &mut (*_2);
         // CHECK: [[tmp:_.*]] = &mut (*_1);
-        // CHECK: _2 = move [[tmp]];
+        // CHECK: _2 = copy [[tmp]];
         // CHECK: [[b:_.*]] = copy (*[[a]]);
 
         let a = &mut *multiple;
@@ -343,7 +343,7 @@ fn reference_propagation_const_ptr(single: *const T, mut multiple: *con
         // CHECK: [[a2:_.*]] = const 7_usize;
         // CHECK: [[b:_.*]] = &raw const [[a]];
         // CHECK: [[btmp:_.*]] = &raw const [[a2]];
-        // CHECK: [[b]] = move [[btmp]];
+        // CHECK: [[b]] = copy [[btmp]];
         // CHECK: [[c:_.*]] = copy (*[[b]]);
 
         let a = 5_usize;
@@ -435,7 +435,7 @@ fn reference_propagation_const_ptr(single: *const T, mut multiple: *con
         // CHECK: bb7: {
         // CHECK: [[a:_.*]] = &raw const (*_2);
         // CHECK: [[tmp:_.*]] = &raw const (*_1);
-        // CHECK: _2 = move [[tmp]];
+        // CHECK: _2 = copy [[tmp]];
         // CHECK: [[b:_.*]] = copy (*[[a]]);
 
         let a = &raw const *multiple;
@@ -514,7 +514,7 @@ fn reference_propagation_mut_ptr(single: *mut T, mut multiple: *mut T)
         // CHECK: [[a2:_.*]] = const 7_usize;
         // CHECK: [[b:_.*]] = &raw mut [[a]];
         // CHECK: [[btmp:_.*]] = &raw mut [[a2]];
-        // CHECK: [[b]] = move [[btmp]];
+        // CHECK: [[b]] = copy [[btmp]];
         // CHECK: [[c:_.*]] = copy (*[[b]]);
 
         let mut a = 5_usize;
@@ -606,7 +606,7 @@ fn reference_propagation_mut_ptr(single: *mut T, mut multiple: *mut T)
         // CHECK: bb7: {
         // CHECK: [[a:_.*]] = &raw mut (*_2);
         // CHECK: [[tmp:_.*]] = &raw mut (*_1);
-        // CHECK: _2 = move [[tmp]];
+        // CHECK: _2 = copy [[tmp]];
         // CHECK: [[b:_.*]] = copy (*[[a]]);
 
         let a = &raw mut *multiple;
diff --git a/tests/mir-opt/reference_prop_do_not_reuse_move.rs b/tests/mir-opt/reference_prop_do_not_reuse_move.rs
new file mode 100644
index 000000000000..8859d30aed19
--- /dev/null
+++ b/tests/mir-opt/reference_prop_do_not_reuse_move.rs
@@ -0,0 +1,44 @@
+//@ test-mir-pass: ReferencePropagation
+
+#![feature(custom_mir, core_intrinsics)]
+#![allow(internal_features)]
+#![crate_type = "lib"]
+
+use std::intrinsics::mir::*;
+
+#[inline(never)]
+fn opaque(_: impl Sized, _: impl Sized) {}
+
+#[custom_mir(dialect = "runtime")]
+pub fn fn0() {
+    // CHECK-LABEL: fn0
+    // CHECK: _9 = opaque::<&u8, &u64>(copy (_2.1: &u8), copy _6) -> [return: bb1, unwind unreachable];
+    mir! {
+        let _1: (u8, u8);
+        let _2: (u64, &u8);
+        let _3: (u8, &&u64);
+        let _4: u64;
+        let _5: &u64;
+        let _6: &u64;
+        let _7: &u64;
+        let _8: u64;
+        let n: ();
+        {
+            _3.0 = 0;
+            _1 = (0, _3.0);
+            _4 = 0;
+            _2.1 = &_1.0;
+            _8 = 0;
+            _5 = &_8;
+            _5 = &_4;
+            _6 = _5;
+            _7 = _6;
+            _3.1 = &_6;
+            Call(n = opaque(_2.1, Move(_6)), ReturnTo(bb1), UnwindUnreachable())
+        }
+        bb1 = {
+            _2.0 = *_7;
+            Return()
+        }
+    }
+}
diff --git a/tests/mir-opt/remove_unneeded_drop_in_place.rs b/tests/mir-opt/remove_unneeded_drop_in_place.rs
new file mode 100644
index 000000000000..470c6499d169
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drop_in_place.rs
@@ -0,0 +1,17 @@
+//@ test-mir-pass: RemoveUnneededDrops
+//@ needs-unwind
+//@ compile-flags: -Z mir-opt-level=1
+
+// EMIT_MIR remove_unneeded_drop_in_place.slice_in_place.RemoveUnneededDrops.diff
+unsafe fn slice_in_place(ptr: *mut [char]) {
+    // CHECK-LABEL: fn slice_in_place(_1: *mut [char])
+    // CHECK: bb0: {
+    // CHECK-NEXT: return;
+    // CHECK-NEXT: }
+    std::ptr::drop_in_place(ptr)
+}
+
+fn main() {
+    let mut a = ['o', 'k'];
+    unsafe { slice_in_place(&raw mut a) };
+}
diff --git a/tests/mir-opt/remove_unneeded_drop_in_place.slice_in_place.RemoveUnneededDrops.diff b/tests/mir-opt/remove_unneeded_drop_in_place.slice_in_place.RemoveUnneededDrops.diff
new file mode 100644
index 000000000000..4d70e7151c39
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drop_in_place.slice_in_place.RemoveUnneededDrops.diff
@@ -0,0 +1,20 @@
+- // MIR for `slice_in_place` before RemoveUnneededDrops
++ // MIR for `slice_in_place` after RemoveUnneededDrops
+  
+  fn slice_in_place(_1: *mut [char]) -> () {
+      debug ptr => _1;
+      let mut _0: ();
+      let mut _2: *mut [char];
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = copy _1;
+-         _0 = drop_in_place::<[char]>(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.diff b/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.diff
new file mode 100644
index 000000000000..52832e739051
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.diff
@@ -0,0 +1,15 @@
+- // MIR for `cannot_opt_generic` before RemoveUnneededDrops
++ // MIR for `cannot_opt_generic` after RemoveUnneededDrops
+  
+  fn cannot_opt_generic(_1: T) -> () {
+      let mut _0: ();
+  
+      bb0: {
+          drop(_1) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-abort.diff b/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-abort.diff
deleted file mode 100644
index 0c73602bec84..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-abort.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `cannot_opt_generic` before RemoveUnneededDrops
-+ // MIR for `cannot_opt_generic` after RemoveUnneededDrops
-  
-  fn cannot_opt_generic(_1: T) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: T;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
-          nop;
-          StorageLive(_3);
-          _3 = move _1;
-          drop(_3) -> [return: bb1, unwind unreachable];
-      }
-  
-      bb1: {
-          StorageDead(_3);
-          nop;
-          nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-unwind.diff b/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-unwind.diff
deleted file mode 100644
index 59cce9fbcdd0..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.panic-unwind.diff
+++ /dev/null
@@ -1,30 +0,0 @@
-- // MIR for `cannot_opt_generic` before RemoveUnneededDrops
-+ // MIR for `cannot_opt_generic` after RemoveUnneededDrops
-  
-  fn cannot_opt_generic(_1: T) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: T;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
-          nop;
-          StorageLive(_3);
-          _3 = move _1;
-          drop(_3) -> [return: bb2, unwind: bb1];
-      }
-  
-      bb1 (cleanup): {
-          resume;
-      }
-  
-      bb2: {
-          StorageDead(_3);
-          nop;
-          nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.diff b/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.diff
new file mode 100644
index 000000000000..3d67cada5ddf
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.diff
@@ -0,0 +1,15 @@
+- // MIR for `dont_opt` before RemoveUnneededDrops
++ // MIR for `dont_opt` after RemoveUnneededDrops
+  
+  fn dont_opt(_1: Vec) -> () {
+      let mut _0: ();
+  
+      bb0: {
+          drop(_1) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-abort.diff b/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-abort.diff
deleted file mode 100644
index 428b366b5a68..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-abort.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `dont_opt` before RemoveUnneededDrops
-+ // MIR for `dont_opt` after RemoveUnneededDrops
-  
-  fn dont_opt(_1: Vec) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: std::vec::Vec;
-      scope 1 (inlined std::mem::drop::>) {
-      }
-  
-      bb0: {
-          nop;
-          StorageLive(_3);
-          _3 = move _1;
-          drop(_3) -> [return: bb1, unwind unreachable];
-      }
-  
-      bb1: {
-          StorageDead(_3);
-          nop;
-          nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-unwind.diff b/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-unwind.diff
deleted file mode 100644
index 445c1f82a96f..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.dont_opt.RemoveUnneededDrops.panic-unwind.diff
+++ /dev/null
@@ -1,30 +0,0 @@
-- // MIR for `dont_opt` before RemoveUnneededDrops
-+ // MIR for `dont_opt` after RemoveUnneededDrops
-  
-  fn dont_opt(_1: Vec) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: std::vec::Vec;
-      scope 1 (inlined std::mem::drop::>) {
-      }
-  
-      bb0: {
-          nop;
-          StorageLive(_3);
-          _3 = move _1;
-          drop(_3) -> [return: bb2, unwind: bb1];
-      }
-  
-      bb1 (cleanup): {
-          resume;
-      }
-  
-      bb2: {
-          StorageDead(_3);
-          nop;
-          nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.diff b/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.diff
new file mode 100644
index 000000000000..cb7e58ca1a1f
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.diff
@@ -0,0 +1,15 @@
+- // MIR for `opt` before RemoveUnneededDrops
++ // MIR for `opt` after RemoveUnneededDrops
+  
+  fn opt(_1: bool) -> () {
+      let mut _0: ();
+  
+      bb0: {
+-         drop(_1) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-abort.diff b/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-abort.diff
deleted file mode 100644
index 01eb6d4901f7..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-abort.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `opt` before RemoveUnneededDrops
-+ // MIR for `opt` after RemoveUnneededDrops
-  
-  fn opt(_1: bool) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: bool;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
--         nop;
-          StorageLive(_3);
-          _3 = copy _1;
--         drop(_3) -> [return: bb1, unwind unreachable];
--     }
-- 
--     bb1: {
-          StorageDead(_3);
--         nop;
--         nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-unwind.diff b/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-unwind.diff
deleted file mode 100644
index c2c3cb76e832..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.opt.RemoveUnneededDrops.panic-unwind.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `opt` before RemoveUnneededDrops
-+ // MIR for `opt` after RemoveUnneededDrops
-  
-  fn opt(_1: bool) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: bool;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
--         nop;
-          StorageLive(_3);
-          _3 = copy _1;
--         drop(_3) -> [return: bb1, unwind continue];
--     }
-- 
--     bb1: {
-          StorageDead(_3);
--         nop;
--         nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.diff b/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.diff
new file mode 100644
index 000000000000..1e166eee9fb0
--- /dev/null
+++ b/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.diff
@@ -0,0 +1,15 @@
+- // MIR for `opt_generic_copy` before RemoveUnneededDrops
++ // MIR for `opt_generic_copy` after RemoveUnneededDrops
+  
+  fn opt_generic_copy(_1: T) -> () {
+      let mut _0: ();
+  
+      bb0: {
+-         drop(_1) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-abort.diff b/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-abort.diff
deleted file mode 100644
index a82ede6196ed..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-abort.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `opt_generic_copy` before RemoveUnneededDrops
-+ // MIR for `opt_generic_copy` after RemoveUnneededDrops
-  
-  fn opt_generic_copy(_1: T) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: T;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
--         nop;
-          StorageLive(_3);
-          _3 = copy _1;
--         drop(_3) -> [return: bb1, unwind unreachable];
--     }
-- 
--     bb1: {
-          StorageDead(_3);
--         nop;
--         nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-unwind.diff b/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-unwind.diff
deleted file mode 100644
index 6e7c9ead740f..000000000000
--- a/tests/mir-opt/remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.panic-unwind.diff
+++ /dev/null
@@ -1,26 +0,0 @@
-- // MIR for `opt_generic_copy` before RemoveUnneededDrops
-+ // MIR for `opt_generic_copy` after RemoveUnneededDrops
-  
-  fn opt_generic_copy(_1: T) -> () {
-      debug x => _1;
-      let mut _0: ();
-      let _2: ();
-      let mut _3: T;
-      scope 1 (inlined std::mem::drop::) {
-      }
-  
-      bb0: {
--         nop;
-          StorageLive(_3);
-          _3 = copy _1;
--         drop(_3) -> [return: bb1, unwind continue];
--     }
-- 
--     bb1: {
-          StorageDead(_3);
--         nop;
--         nop;
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/remove_unneeded_drops.rs b/tests/mir-opt/remove_unneeded_drops.rs
index cad79e0aa0cb..49dc611838e3 100644
--- a/tests/mir-opt/remove_unneeded_drops.rs
+++ b/tests/mir-opt/remove_unneeded_drops.rs
@@ -1,28 +1,56 @@
-// skip-filecheck
-// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+//@ test-mir-pass: RemoveUnneededDrops
+
+#![feature(custom_mir, core_intrinsics)]
+use std::intrinsics::mir::*;
+
 // EMIT_MIR remove_unneeded_drops.opt.RemoveUnneededDrops.diff
+#[custom_mir(dialect = "runtime")]
 fn opt(x: bool) {
-    drop(x);
+    // CHECK-LABEL: fn opt(
+    // CHECK-NOT: drop(
+    mir! {
+        { Drop(x, ReturnTo(bb1), UnwindUnreachable()) }
+        bb1 = { Return() }
+    }
 }
 
 // EMIT_MIR remove_unneeded_drops.dont_opt.RemoveUnneededDrops.diff
+#[custom_mir(dialect = "runtime")]
 fn dont_opt(x: Vec) {
-    drop(x);
+    // CHECK-LABEL: fn dont_opt(
+    // CHECK: drop(
+    mir! {
+        { Drop(x, ReturnTo(bb1), UnwindUnreachable()) }
+        bb1 = { Return() }
+    }
 }
 
 // EMIT_MIR remove_unneeded_drops.opt_generic_copy.RemoveUnneededDrops.diff
+#[custom_mir(dialect = "runtime")]
 fn opt_generic_copy(x: T) {
-    drop(x);
+    // CHECK-LABEL: fn opt_generic_copy(
+    // CHECK-NOT: drop(
+    mir! {
+        { Drop(x, ReturnTo(bb1), UnwindUnreachable()) }
+        bb1 = { Return() }
+    }
 }
 
 // EMIT_MIR remove_unneeded_drops.cannot_opt_generic.RemoveUnneededDrops.diff
 // since the pass is not running on monomorphisized code,
 // we can't (but probably should) optimize this
+#[custom_mir(dialect = "runtime")]
 fn cannot_opt_generic(x: T) {
-    drop(x);
+    // CHECK-LABEL: fn cannot_opt_generic(
+    // CHECK: drop(
+    mir! {
+        { Drop(x, ReturnTo(bb1), UnwindUnreachable()) }
+        bb1 = { Return() }
+    }
 }
 
 fn main() {
+    // CHECK-LABEL: fn main(
     opt(true);
     opt_generic_copy(42);
     cannot_opt_generic(42);
diff --git a/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-abort.mir b/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-abort.mir
index da005d552e2b..4861abead2f3 100644
--- a/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-abort.mir
+++ b/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-abort.mir
@@ -8,7 +8,7 @@ fn box_to_raw_mut(_1: &mut Box) -> *mut i32 {
 
     bb0: {
         Retag([fn entry] _1);
-        _2 = deref_copy (*_1);
+        _2 = copy (*_1);
         _3 = copy ((_2.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
         _0 = &raw mut (*_3);
         Retag([raw] _0);
diff --git a/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-unwind.mir b/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
index da005d552e2b..4861abead2f3 100644
--- a/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
+++ b/tests/mir-opt/retag.box_to_raw_mut.SimplifyCfg-pre-optimizations.after.panic-unwind.mir
@@ -8,7 +8,7 @@ fn box_to_raw_mut(_1: &mut Box) -> *mut i32 {
 
     bb0: {
         Retag([fn entry] _1);
-        _2 = deref_copy (*_1);
+        _2 = copy (*_1);
         _3 = copy ((_2.0: std::ptr::Unique).0: std::ptr::NonNull) as *const i32 (Transmute);
         _0 = &raw mut (*_3);
         Retag([raw] _0);
diff --git a/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-abort.mir b/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-abort.mir
index 7be3ab8cbae9..8e47aabb9b9f 100644
--- a/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-abort.mir
+++ b/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-abort.mir
@@ -1,4 +1,4 @@
-// MIR for `drop_in_place` after SimplifyCfg-make_shim
+// MIR for `std::ptr::drop_in_place` after SimplifyCfg-make_shim
 
 fn drop_in_place(_1: *mut Test) -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-unwind.mir b/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-unwind.mir
index 6c3c1aaa2bd2..2457405d9969 100644
--- a/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-unwind.mir
+++ b/tests/mir-opt/retag.core.ptr-drop_in_place.Test.SimplifyCfg-make_shim.after.panic-unwind.mir
@@ -1,4 +1,4 @@
-// MIR for `drop_in_place` after SimplifyCfg-make_shim
+// MIR for `std::ptr::drop_in_place` after SimplifyCfg-make_shim
 
 fn drop_in_place(_1: *mut Test) -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff
deleted file mode 100644
index c3076fb67c23..000000000000
--- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff
+++ /dev/null
@@ -1,21 +0,0 @@
-- // MIR for `main` before SimplifyConstCondition-after-const-prop
-+ // MIR for `main` after SimplifyConstCondition-after-const-prop
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: ();
-  
-      bb0: {
--         switchInt(const false) -> [0: bb2, otherwise: bb1];
-+         goto -> bb2;
-      }
-  
-      bb1: {
-          _1 = noop() -> [return: bb2, unwind unreachable];
-      }
-  
-      bb2: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff
deleted file mode 100644
index 6c346e20e589..000000000000
--- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff
+++ /dev/null
@@ -1,21 +0,0 @@
-- // MIR for `main` before SimplifyConstCondition-after-const-prop
-+ // MIR for `main` after SimplifyConstCondition-after-const-prop
-  
-  fn main() -> () {
-      let mut _0: ();
-      let _1: ();
-  
-      bb0: {
--         switchInt(const false) -> [0: bb2, otherwise: bb1];
-+         goto -> bb2;
-      }
-  
-      bb1: {
-          _1 = noop() -> [return: bb2, unwind continue];
-      }
-  
-      bb2: {
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff
new file mode 100644
index 000000000000..c67fd69235b4
--- /dev/null
+++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-abort.diff
@@ -0,0 +1,25 @@
+- // MIR for `main` before SimplifyConstCondition-after-inst-simplify
++ // MIR for `main` after SimplifyConstCondition-after-inst-simplify
+  
+  fn main() -> () {
+      let mut _0: ();
+      let mut _1: bool;
+      let _2: ();
+  
+      bb0: {
+          StorageLive(_1);
+          _1 = const false;
+-         switchInt(move _1) -> [0: bb2, otherwise: bb1];
++         goto -> bb2;
+      }
+  
+      bb1: {
+          _2 = noop() -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff
new file mode 100644
index 000000000000..07f179dfdd45
--- /dev/null
+++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-inst-simplify.panic-unwind.diff
@@ -0,0 +1,25 @@
+- // MIR for `main` before SimplifyConstCondition-after-inst-simplify
++ // MIR for `main` after SimplifyConstCondition-after-inst-simplify
+  
+  fn main() -> () {
+      let mut _0: ();
+      let mut _1: bool;
+      let _2: ();
+  
+      bb0: {
+          StorageLive(_1);
+          _1 = const false;
+-         switchInt(move _1) -> [0: bb2, otherwise: bb1];
++         goto -> bb2;
+      }
+  
+      bb1: {
+          _2 = noop() -> [return: bb2, unwind continue];
+      }
+  
+      bb2: {
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/simplify_if.rs b/tests/mir-opt/simplify_if.rs
index f600c0595819..6aca23365554 100644
--- a/tests/mir-opt/simplify_if.rs
+++ b/tests/mir-opt/simplify_if.rs
@@ -2,7 +2,7 @@
 #[inline(never)]
 fn noop() {}
 
-// EMIT_MIR simplify_if.main.SimplifyConstCondition-after-const-prop.diff
+// EMIT_MIR simplify_if.main.SimplifyConstCondition-after-inst-simplify.diff
 fn main() {
     // CHECK-LABEL: fn main(
 
diff --git a/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String;42].AddMovesForPackedDrops.before.mir b/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String;42].AddMovesForPackedDrops.before.mir
index 9d5af8e84e4f..ed3f4788cea7 100644
--- a/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String;42].AddMovesForPackedDrops.before.mir
+++ b/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String;42].AddMovesForPackedDrops.before.mir
@@ -1,4 +1,4 @@
-// MIR for `drop_in_place` before AddMovesForPackedDrops
+// MIR for `std::ptr::drop_in_place` before AddMovesForPackedDrops
 
 fn drop_in_place(_1: *mut [String; 42]) -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String].AddMovesForPackedDrops.before.mir b/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String].AddMovesForPackedDrops.before.mir
index 144880d15989..bee671af6dfe 100644
--- a/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String].AddMovesForPackedDrops.before.mir
+++ b/tests/mir-opt/slice_drop_shim.core.ptr-drop_in_place.[String].AddMovesForPackedDrops.before.mir
@@ -1,4 +1,4 @@
-// MIR for `drop_in_place` before AddMovesForPackedDrops
+// MIR for `std::ptr::drop_in_place` before AddMovesForPackedDrops
 
 fn drop_in_place(_1: *mut [String]) -> () {
     let mut _0: ();
diff --git a/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
index 0d5fcf9ef143..056dc4c42c11 100644
--- a/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
+++ b/tests/mir-opt/sroa/lifetimes.foo.ScalarReplacementOfAggregates.diff
@@ -117,8 +117,8 @@
           StorageLive(_15);
           StorageLive(_16);
           StorageLive(_17);
--         _26 = deref_copy (_12.0: &std::boxed::Box);
-+         _26 = deref_copy _34;
+-         _26 = copy (_12.0: &std::boxed::Box);
++         _26 = copy _34;
           _17 = &(*_26);
           _16 = core::fmt::rt::Argument::<'_>::new_display::>(move _17) -> [return: bb3, unwind unreachable];
       }
@@ -127,8 +127,8 @@
           StorageDead(_17);
           StorageLive(_18);
           StorageLive(_19);
--         _27 = deref_copy (_12.1: &u32);
-+         _27 = deref_copy _35;
+-         _27 = copy (_12.1: &u32);
++         _27 = copy _35;
           _19 = &(*_27);
           _18 = core::fmt::rt::Argument::<'_>::new_display::(move _19) -> [return: bb4, unwind unreachable];
       }
diff --git a/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff b/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff
new file mode 100644
index 000000000000..744032471089
--- /dev/null
+++ b/tests/mir-opt/sroa/simd_sroa.foo.ScalarReplacementOfAggregates.diff
@@ -0,0 +1,32 @@
+- // MIR for `foo` before ScalarReplacementOfAggregates
++ // MIR for `foo` after ScalarReplacementOfAggregates
+  
+  fn foo(_1: &[Simd], _2: Simd) -> () {
+      debug simds => _1;
+      debug _unused => _2;
+      let mut _0: ();
+      let _3: std::simd::Simd;
+      let _4: usize;
+      let mut _5: usize;
+      let mut _6: bool;
+      scope 1 {
+          debug a => _3;
+      }
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          _4 = const 0_usize;
+          _5 = PtrMetadata(copy _1);
+          _6 = Lt(copy _4, copy _5);
+          assert(move _6, "index out of bounds: the length is {} but the index is {}", move _5, copy _4) -> [success: bb1, unwind continue];
+      }
+  
+      bb1: {
+          _3 = copy (*_1)[_4];
+          StorageDead(_4);
+          StorageDead(_3);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/sroa/simd_sroa.rs b/tests/mir-opt/sroa/simd_sroa.rs
new file mode 100644
index 000000000000..1ae84d3f975b
--- /dev/null
+++ b/tests/mir-opt/sroa/simd_sroa.rs
@@ -0,0 +1,18 @@
+//@ needs-unwind
+#![feature(portable_simd)]
+
+// SRoA expands things even if they're unused
+// 
+
+use std::simd::Simd;
+
+// EMIT_MIR simd_sroa.foo.ScalarReplacementOfAggregates.diff
+pub(crate) fn foo(simds: &[Simd], _unused: Simd) {
+    // CHECK-LABEL: fn foo
+    // CHECK-NOT: [u8; 16]
+    // CHECK: let [[SIMD:_.+]]: std::simd::Simd;
+    // CHECK-NOT: [u8; 16]
+    // CHECK: [[SIMD]] = copy (*_1)[0 of 1];
+    // CHECK-NOT: [u8; 16]
+    let a = simds[0];
+}
diff --git a/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-abort.diff b/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-abort.diff
index a8c57d2cfe00..4fba0032729e 100644
--- a/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-abort.diff
+++ b/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-abort.diff
@@ -93,7 +93,7 @@
       }
   
       bb11: {
-          tailcall g_with_arg(Spanned { node: move _10, span: $DIR/tail_call_drops.rs:36:23: 36:36 (#0) }, Spanned { node: move _11, span: $DIR/tail_call_drops.rs:36:38: 36:51 (#0) });
+          tailcall g_with_arg(move _10, move _11);
       }
   
       bb12 (cleanup): {
diff --git a/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-unwind.diff b/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-unwind.diff
index a8c57d2cfe00..4fba0032729e 100644
--- a/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-unwind.diff
+++ b/tests/mir-opt/tail_call_drops.f_with_arg.ElaborateDrops.panic-unwind.diff
@@ -93,7 +93,7 @@
       }
   
       bb11: {
-          tailcall g_with_arg(Spanned { node: move _10, span: $DIR/tail_call_drops.rs:36:23: 36:36 (#0) }, Spanned { node: move _11, span: $DIR/tail_call_drops.rs:36:38: 36:51 (#0) });
+          tailcall g_with_arg(move _10, move _11);
       }
   
       bb12 (cleanup): {
diff --git a/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-abort.mir b/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-abort.mir
index f89b98a32053..9ec358ec1893 100644
--- a/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-abort.mir
+++ b/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-abort.mir
@@ -90,7 +90,7 @@ fn f_with_arg(_1: String, _2: String) -> () {
     }
 
     bb11: {
-        tailcall g_with_arg(Spanned { node: move _10, span: $DIR/tail_call_drops.rs:36:23: 36:36 (#0) }, Spanned { node: move _11, span: $DIR/tail_call_drops.rs:36:38: 36:51 (#0) });
+        tailcall g_with_arg(move _10, move _11);
     }
 
     bb12: {
diff --git a/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-unwind.mir b/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-unwind.mir
index f89b98a32053..9ec358ec1893 100644
--- a/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-unwind.mir
+++ b/tests/mir-opt/tail_call_drops.f_with_arg.built.after.panic-unwind.mir
@@ -90,7 +90,7 @@ fn f_with_arg(_1: String, _2: String) -> () {
     }
 
     bb11: {
-        tailcall g_with_arg(Spanned { node: move _10, span: $DIR/tail_call_drops.rs:36:23: 36:36 (#0) }, Spanned { node: move _11, span: $DIR/tail_call_drops.rs:36:38: 36:51 (#0) });
+        tailcall g_with_arg(move _10, move _11);
     }
 
     bb12: {
diff --git a/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir b/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir
index 6bf4be652bef..89f4cbc2edea 100644
--- a/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir
+++ b/tests/mir-opt/uninhabited_not_read.main.SimplifyLocals-final.after.mir
@@ -31,7 +31,7 @@ fn main() -> () {
         StorageLive(_2);
         StorageLive(_3);
         _3 = &raw const _1;
-        _2 = move _3 as *const ! (PtrToPtr);
+        _2 = copy _3 as *const ! (PtrToPtr);
         StorageDead(_3);
         StorageDead(_2);
         StorageDead(_1);
@@ -40,7 +40,7 @@ fn main() -> () {
         StorageLive(_5);
         StorageLive(_6);
         _6 = &raw const _4;
-        _5 = move _6 as *const ! (PtrToPtr);
+        _5 = copy _6 as *const ! (PtrToPtr);
         StorageDead(_6);
         StorageDead(_5);
         StorageDead(_4);
diff --git a/tests/mir-opt/unusual_item_types.core.ptr-drop_in_place.Vec_i32_.AddMovesForPackedDrops.before.mir b/tests/mir-opt/unusual_item_types.core.ptr-drop_in_place.Vec_i32_.AddMovesForPackedDrops.before.mir
index 51ef9f7c068e..1bdb1c1debdb 100644
--- a/tests/mir-opt/unusual_item_types.core.ptr-drop_in_place.Vec_i32_.AddMovesForPackedDrops.before.mir
+++ b/tests/mir-opt/unusual_item_types.core.ptr-drop_in_place.Vec_i32_.AddMovesForPackedDrops.before.mir
@@ -1,4 +1,4 @@
-// MIR for `drop_in_place` before AddMovesForPackedDrops
+// MIR for `std::ptr::drop_in_place` before AddMovesForPackedDrops
 
 fn drop_in_place(_1: *mut Vec) -> () {
     let mut _0: ();
diff --git a/tests/pretty/autodiff/autodiff_forward.pp b/tests/pretty/autodiff/autodiff_forward.pp
index a2525abc8320..6eddb5669c7a 100644
--- a/tests/pretty/autodiff/autodiff_forward.pp
+++ b/tests/pretty/autodiff/autodiff_forward.pp
@@ -3,10 +3,10 @@
 //@ needs-enzyme
 
 #![feature(autodiff)]
-#[prelude_import]
-use ::std::prelude::rust_2015::*;
 #[macro_use]
 extern crate std;
+#[prelude_import]
+use ::std::prelude::rust_2015::*;
 //@ pretty-mode:expanded
 //@ pretty-compare-only
 //@ pp-exact:autodiff_forward.pp
@@ -16,7 +16,6 @@ extern crate std;
 use std::autodiff::{autodiff_forward, autodiff_reverse};
 
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f1(x: &[f64], y: f64) -> f64 {
 
 
@@ -36,163 +35,96 @@ pub fn f1(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Forward, 1, Dual, Const, Dual)]
-#[inline(never)]
 pub fn df1(x: &[f64], bx_0: &[f64], y: f64) -> (f64, f64) {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f1(x, y));
-    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box(<(f64, f64)>::default())
+    ::core::intrinsics::autodiff(f1::<>, df1::<>, (x, bx_0, y))
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f2(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Forward, 1, Dual, Const, Const)]
-#[inline(never)]
 pub fn df2(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f2(x, y));
-    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box(f2(x, y))
+    ::core::intrinsics::autodiff(f2::<>, df2::<>, (x, bx_0, y))
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f3(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Forward, 1, Dual, Const, Const)]
-#[inline(never)]
 pub fn df3(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f3(x, y));
-    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box(f3(x, y))
+    ::core::intrinsics::autodiff(f3::<>, df3::<>, (x, bx_0, y))
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f4() {}
 #[rustc_autodiff(Forward, 1, None)]
-#[inline(never)]
-pub fn df4() -> () {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f4());
-    ::core::hint::black_box(());
-}
+pub fn df4() -> () { ::core::intrinsics::autodiff(f4::<>, df4::<>, ()) }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f5(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Forward, 1, Const, Dual, Const)]
-#[inline(never)]
 pub fn df5_y(x: &[f64], y: f64, by_0: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((by_0,));
-    ::core::hint::black_box(f5(x, y))
+    ::core::intrinsics::autodiff(f5::<>, df5_y::<>, (x, y, by_0))
 }
 #[rustc_autodiff(Forward, 1, Dual, Const, Const)]
-#[inline(never)]
 pub fn df5_x(x: &[f64], bx_0: &[f64], y: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box(f5(x, y))
+    ::core::intrinsics::autodiff(f5::<>, df5_x::<>, (x, bx_0, y))
 }
 #[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
-#[inline(never)]
 pub fn df5_rev(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((dx_0, dret));
-    ::core::hint::black_box(f5(x, y))
+    ::core::intrinsics::autodiff(f5::<>, df5_rev::<>, (x, dx_0, y, dret))
 }
 struct DoesNotImplDefault;
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f6() -> DoesNotImplDefault {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Forward, 1, Const)]
-#[inline(never)]
 pub fn df6() -> DoesNotImplDefault {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f6());
-    ::core::hint::black_box(());
-    ::core::hint::black_box(f6())
+    ::core::intrinsics::autodiff(f6::<>, df6::<>, ())
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f7(x: f32) -> () {}
 #[rustc_autodiff(Forward, 1, Const, None)]
-#[inline(never)]
 pub fn df7(x: f32) -> () {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f7(x));
-    ::core::hint::black_box(());
+    ::core::intrinsics::autodiff(f7::<>, df7::<>, (x,))
 }
 #[no_mangle]
 #[rustc_autodiff]
-#[inline(never)]
 fn f8(x: &f32) -> f32 { ::core::panicking::panic("not implemented") }
 #[rustc_autodiff(Forward, 4, Dual, Dual)]
-#[inline(never)]
 fn f8_3(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
     -> [f32; 5usize] {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f8(x));
-    ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
-    ::core::hint::black_box(<[f32; 5usize]>::default())
+    ::core::intrinsics::autodiff(f8::<>, f8_3::<>,
+        (x, bx_0, bx_1, bx_2, bx_3))
 }
 #[rustc_autodiff(Forward, 4, Dual, DualOnly)]
-#[inline(never)]
 fn f8_2(x: &f32, bx_0: &f32, bx_1: &f32, bx_2: &f32, bx_3: &f32)
     -> [f32; 4usize] {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f8(x));
-    ::core::hint::black_box((bx_0, bx_1, bx_2, bx_3));
-    ::core::hint::black_box(<[f32; 4usize]>::default())
+    ::core::intrinsics::autodiff(f8::<>, f8_2::<>,
+        (x, bx_0, bx_1, bx_2, bx_3))
 }
 #[rustc_autodiff(Forward, 1, Dual, DualOnly)]
-#[inline(never)]
 fn f8_1(x: &f32, bx_0: &f32) -> f32 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f8(x));
-    ::core::hint::black_box((bx_0,));
-    ::core::hint::black_box(::default())
+    ::core::intrinsics::autodiff(f8::<>, f8_1::<>, (x, bx_0))
 }
 pub fn f9() {
     #[rustc_autodiff]
-    #[inline(never)]
     fn inner(x: f32) -> f32 { x * x }
     #[rustc_autodiff(Forward, 1, Dual, Dual)]
-    #[inline(never)]
     fn d_inner_2(x: f32, bx_0: f32) -> (f32, f32) {
-        unsafe { asm!("NOP", options(pure, nomem)); };
-        ::core::hint::black_box(inner(x));
-        ::core::hint::black_box((bx_0,));
-        ::core::hint::black_box(<(f32, f32)>::default())
+        ::core::intrinsics::autodiff(inner::<>, d_inner_2::<>, (x, bx_0))
     }
     #[rustc_autodiff(Forward, 1, Dual, DualOnly)]
-    #[inline(never)]
     fn d_inner_1(x: f32, bx_0: f32) -> f32 {
-        unsafe { asm!("NOP", options(pure, nomem)); };
-        ::core::hint::black_box(inner(x));
-        ::core::hint::black_box((bx_0,));
-        ::core::hint::black_box(::default())
+        ::core::intrinsics::autodiff(inner::<>, d_inner_1::<>, (x, bx_0))
     }
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f10 + Copy>(x: &T) -> T { *x * *x }
 #[rustc_autodiff(Reverse, 1, Duplicated, Active)]
-#[inline(never)]
 pub fn d_square +
     Copy>(x: &T, dx_0: &mut T, dret: T) -> T {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f10::(x));
-    ::core::hint::black_box((dx_0, dret));
-    ::core::hint::black_box(f10::(x))
+    ::core::intrinsics::autodiff(f10::, d_square::, (x, dx_0, dret))
 }
 fn main() {}
diff --git a/tests/pretty/autodiff/autodiff_reverse.pp b/tests/pretty/autodiff/autodiff_reverse.pp
index e67c3443ddef..8f598b865c7b 100644
--- a/tests/pretty/autodiff/autodiff_reverse.pp
+++ b/tests/pretty/autodiff/autodiff_reverse.pp
@@ -3,10 +3,10 @@
 //@ needs-enzyme
 
 #![feature(autodiff)]
-#[prelude_import]
-use ::std::prelude::rust_2015::*;
 #[macro_use]
 extern crate std;
+#[prelude_import]
+use ::std::prelude::rust_2015::*;
 //@ pretty-mode:expanded
 //@ pretty-compare-only
 //@ pp-exact:autodiff_reverse.pp
@@ -16,7 +16,6 @@ extern crate std;
 use std::autodiff::autodiff_reverse;
 
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f1(x: &[f64], y: f64) -> f64 {
 
     // Not the most interesting derivative, but who are we to judge
@@ -29,58 +28,33 @@ pub fn f1(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
-#[inline(never)]
 pub fn df1(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f1(x, y));
-    ::core::hint::black_box((dx_0, dret));
-    ::core::hint::black_box(f1(x, y))
+    ::core::intrinsics::autodiff(f1::<>, df1::<>, (x, dx_0, y, dret))
 }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f2() {}
 #[rustc_autodiff(Reverse, 1, None)]
-#[inline(never)]
-pub fn df2() {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f2());
-    ::core::hint::black_box(());
-}
+pub fn df2() { ::core::intrinsics::autodiff(f2::<>, df2::<>, ()) }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f3(x: &[f64], y: f64) -> f64 {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Reverse, 1, Duplicated, Const, Active)]
-#[inline(never)]
 pub fn df3(x: &[f64], dx_0: &mut [f64], y: f64, dret: f64) -> f64 {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f3(x, y));
-    ::core::hint::black_box((dx_0, dret));
-    ::core::hint::black_box(f3(x, y))
+    ::core::intrinsics::autodiff(f3::<>, df3::<>, (x, dx_0, y, dret))
 }
 enum Foo { Reverse, }
 use Foo::Reverse;
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f4(x: f32) { ::core::panicking::panic("not implemented") }
 #[rustc_autodiff(Reverse, 1, Const, None)]
-#[inline(never)]
-pub fn df4(x: f32) {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f4(x));
-    ::core::hint::black_box(());
-}
+pub fn df4(x: f32) { ::core::intrinsics::autodiff(f4::<>, df4::<>, (x,)) }
 #[rustc_autodiff]
-#[inline(never)]
 pub fn f5(x: *const f32, y: &f32) {
     ::core::panicking::panic("not implemented")
 }
 #[rustc_autodiff(Reverse, 1, DuplicatedOnly, Duplicated, None)]
-#[inline(never)]
 pub unsafe fn df5(x: *const f32, dx_0: *mut f32, y: &f32, dy_0: &mut f32) {
-    unsafe { asm!("NOP", options(pure, nomem)); };
-    ::core::hint::black_box(f5(x, y));
-    ::core::hint::black_box((dx_0, dy_0));
+    ::core::intrinsics::autodiff(f5::<>, df5::<>, (x, dx_0, y, dy_0))
 }
 fn main() {}
diff --git a/tests/pretty/autodiff/autodiff_reverse.rs b/tests/pretty/autodiff/autodiff_reverse.rs
index d37e5e3eb4ce..c50b81d7780d 100644
--- a/tests/pretty/autodiff/autodiff_reverse.rs
+++ b/tests/pretty/autodiff/autodiff_reverse.rs
@@ -23,7 +23,9 @@ pub fn f3(x: &[f64], y: f64) -> f64 {
     unimplemented!()
 }
 
-enum Foo { Reverse }
+enum Foo {
+    Reverse,
+}
 use Foo::Reverse;
 // What happens if we already have Reverse in type (enum variant decl) and value (enum variant
 // constructor) namespace? > It's expected to work normally.
diff --git a/tests/pretty/autodiff/inherent_impl.pp b/tests/pretty/autodiff/inherent_impl.pp
index d18061b2dbde..36a9222640ae 100644
--- a/tests/pretty/autodiff/inherent_impl.pp
+++ b/tests/pretty/autodiff/inherent_impl.pp
@@ -3,10 +3,10 @@
 //@ needs-enzyme
 
 #![feature(autodiff)]
-#[prelude_import]
-use ::std::prelude::rust_2015::*;
 #[macro_use]
 extern crate std;
+#[prelude_import]
+use ::std::prelude::rust_2015::*;
 //@ pretty-mode:expanded
 //@ pretty-compare-only
 //@ pp-exact:inherent_impl.pp
@@ -26,16 +26,12 @@ trait MyTrait {
 
 impl MyTrait for Foo {
     #[rustc_autodiff]
-    #[inline(never)]
     fn f(&self, x: f64) -> f64 {
         self.a * 0.25 * (x * x - 1.0 - 2.0 * x.ln())
     }
     #[rustc_autodiff(Reverse, 1, Const, Active, Active)]
-    #[inline(never)]
     fn df(&self, x: f64, dret: f64) -> (f64, f64) {
-        unsafe { asm!("NOP", options(pure, nomem)); };
-        ::core::hint::black_box(self.f(x));
-        ::core::hint::black_box((dret,));
-        ::core::hint::black_box((self.f(x), f64::default()))
+        ::core::intrinsics::autodiff(Self::f::<>, Self::df::<>,
+            (self, x, dret))
     }
 }
diff --git a/tests/pretty/hir-lifetimes.pp b/tests/pretty/hir-lifetimes.pp
index 00c052d3f798..e8b174189130 100644
--- a/tests/pretty/hir-lifetimes.pp
+++ b/tests/pretty/hir-lifetimes.pp
@@ -29,11 +29,9 @@ impl  Foo<'_> {
     fn d<'a>(&self, x: &'a u32) { }
 
     // FIXME: impl Traits printed as just `/*impl Trait*/`, ugh
-    fn iter1<'a>(&self)
-        -> /*impl Trait*/ { #[lang = "Range"] { start: 0, end: 1 } }
+    fn iter1<'a>(&self) -> /*impl Trait*/ { Range { start: 0, end: 1 } }
 
-    fn iter2(&self)
-        -> /*impl Trait*/ { #[lang = "Range"] { start: 0, end: 1 } }
+    fn iter2(&self) -> /*impl Trait*/ { Range { start: 0, end: 1 } }
 }
 
 fn a(x: Foo<'_>) { }
diff --git a/tests/run-make/apple-slow-tls/rmake.rs b/tests/run-make-cargo/apple-slow-tls/rmake.rs
similarity index 100%
rename from tests/run-make/apple-slow-tls/rmake.rs
rename to tests/run-make-cargo/apple-slow-tls/rmake.rs
diff --git a/tests/run-make/apple-slow-tls/tls_test/Cargo.toml b/tests/run-make-cargo/apple-slow-tls/tls_test/Cargo.toml
similarity index 100%
rename from tests/run-make/apple-slow-tls/tls_test/Cargo.toml
rename to tests/run-make-cargo/apple-slow-tls/tls_test/Cargo.toml
diff --git a/tests/run-make/apple-slow-tls/tls_test/src/main.rs b/tests/run-make-cargo/apple-slow-tls/tls_test/src/main.rs
similarity index 100%
rename from tests/run-make/apple-slow-tls/tls_test/src/main.rs
rename to tests/run-make-cargo/apple-slow-tls/tls_test/src/main.rs
diff --git a/tests/run-make/compiler-builtins/Cargo.toml b/tests/run-make-cargo/compiler-builtins/Cargo.toml
similarity index 100%
rename from tests/run-make/compiler-builtins/Cargo.toml
rename to tests/run-make-cargo/compiler-builtins/Cargo.toml
diff --git a/tests/run-make/compiler-builtins/lib.rs b/tests/run-make-cargo/compiler-builtins/lib.rs
similarity index 100%
rename from tests/run-make/compiler-builtins/lib.rs
rename to tests/run-make-cargo/compiler-builtins/lib.rs
diff --git a/tests/run-make/compiler-builtins/rmake.rs b/tests/run-make-cargo/compiler-builtins/rmake.rs
similarity index 100%
rename from tests/run-make/compiler-builtins/rmake.rs
rename to tests/run-make-cargo/compiler-builtins/rmake.rs
diff --git a/tests/run-make-cargo/panic-immediate-abort-codegen/Cargo.toml b/tests/run-make-cargo/panic-immediate-abort-codegen/Cargo.toml
new file mode 100644
index 000000000000..3c61c12a84ed
--- /dev/null
+++ b/tests/run-make-cargo/panic-immediate-abort-codegen/Cargo.toml
@@ -0,0 +1,12 @@
+cargo-features = ["profile-rustflags"]
+
+[package]
+name = "panic_scenarios"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+path = "lib.rs"
+
+[profile.release]
+rustflags = ["-Zmerge-functions=disabled", "-Zcodegen-source-order", "--emit=llvm-ir"]
diff --git a/tests/run-make-cargo/panic-immediate-abort-codegen/lib.rs b/tests/run-make-cargo/panic-immediate-abort-codegen/lib.rs
new file mode 100644
index 000000000000..1e20da93ba80
--- /dev/null
+++ b/tests/run-make-cargo/panic-immediate-abort-codegen/lib.rs
@@ -0,0 +1,65 @@
+#![no_std]
+
+#[unsafe(no_mangle)]
+pub fn panic_noarg() {
+    // CHECK-LABEL: @panic_noarg(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: tail call void @llvm.trap()
+    panic!();
+}
+
+#[unsafe(no_mangle)]
+pub fn panic_str() {
+    // CHECK-LABEL: @panic_str(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: tail call void @llvm.trap()
+    panic!("ouch");
+}
+
+#[unsafe(no_mangle)]
+pub fn bounds_check(x: &[u8], idx: usize) -> &u8 {
+    // CHECK-LABEL: @bounds_check(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: icmp ult
+    // CHECK-NEXT: br i1
+    // CHECK: bb1:
+    // CHECK-NEXT: getelementptr inbounds nuw i8
+    // CHECK-NEXT: ret ptr
+    // CHECK: panic:
+    // CHECK-NEXT: tail call void @llvm.trap()
+    &x[idx]
+}
+
+#[unsafe(no_mangle)]
+pub fn str_bounds_check(x: &str, idx: usize) -> &str {
+    // CHECK-LABEL: @str_bounds_check(
+    // CHECK-NOT: call
+    // CHECK: tail call void @llvm.trap()
+    // CHECK-NOT: call
+    &x[idx..]
+}
+
+#[unsafe(no_mangle)]
+pub fn unsigned_integer_div(x: u16, y: u16) -> u16 {
+    // CHECK-LABEL: @unsigned_integer_div(
+    // CHECK-NEXT: start:
+    // CHECK-NEXT: icmp eq i16
+    // CHECK-NEXT: br i1
+    // CHECK: bb1:
+    // CHECK-NEXT: udiv i16
+    // CHECK-NEXT: ret i16
+    // CHECK: panic:
+    // CHECK-NEXT: tail call void @llvm.trap()
+    x / y
+}
+
+#[unsafe(no_mangle)]
+pub fn refcell_already_borrowed() {
+    // CHECK-LABEL: @refcell_already_borrowed(
+    // CHECK-NOT: call
+    // CHECK: tail call void @llvm.trap()
+    // CHECK-NOT: call
+    let r = core::cell::RefCell::new(0u8);
+    let _guard = r.borrow_mut();
+    r.borrow_mut();
+}
diff --git a/tests/run-make-cargo/panic-immediate-abort-codegen/rmake.rs b/tests/run-make-cargo/panic-immediate-abort-codegen/rmake.rs
new file mode 100644
index 000000000000..d7a7a8bfd8c3
--- /dev/null
+++ b/tests/run-make-cargo/panic-immediate-abort-codegen/rmake.rs
@@ -0,0 +1,46 @@
+// This is a codegen test which checks that when code is compiled with panic=immediate-abort,
+// we get a `tail call void @llvm.trap()` in user code instead of a call into the standard
+// library's panic formatting code (such as panic_fmt) or one of the numerous panic outlining shims
+// (such as slice_index_fail).
+
+#![deny(warnings)]
+
+use run_make_support::{cargo, llvm_filecheck, path, rfs, target};
+
+fn main() {
+    let target_dir = path("target");
+
+    cargo()
+        .args(&[
+            "build",
+            "--release",
+            "--lib",
+            "--manifest-path",
+            "Cargo.toml",
+            "-Zbuild-std=core",
+            "--target",
+            &target(),
+        ])
+        .env("RUSTFLAGS", "-Zunstable-options -Cpanic=immediate-abort")
+        .env("CARGO_TARGET_DIR", &target_dir)
+        .env("RUSTC_BOOTSTRAP", "1")
+        // Visual Studio 2022 requires that the LIB env var be set so it can
+        // find the Windows SDK.
+        .env("LIB", std::env::var("LIB").unwrap_or_default())
+        .run();
+
+    let out_dir = target_dir.join(target()).join("release").join("deps");
+    let ir_file = rfs::read_dir(out_dir)
+        .find_map(|e| {
+            let path = e.unwrap().path();
+            let file_name = path.file_name().unwrap().to_str().unwrap();
+            if file_name.starts_with("panic_scenarios") && file_name.ends_with(".ll") {
+                Some(path)
+            } else {
+                None
+            }
+        })
+        .unwrap();
+
+    llvm_filecheck().patterns("lib.rs").input_file(ir_file).run();
+}
diff --git a/tests/run-make-cargo/panic-immediate-abort-works/hello/Cargo.toml b/tests/run-make-cargo/panic-immediate-abort-works/hello/Cargo.toml
new file mode 100644
index 000000000000..1e278d557c08
--- /dev/null
+++ b/tests/run-make-cargo/panic-immediate-abort-works/hello/Cargo.toml
@@ -0,0 +1,4 @@
+[package]
+name = "hello"
+version = "0.1.0"
+edition = "2024"
diff --git a/tests/run-make/pdb-buildinfo-cl-cmd/main.rs b/tests/run-make-cargo/panic-immediate-abort-works/hello/src/main.rs
similarity index 100%
rename from tests/run-make/pdb-buildinfo-cl-cmd/main.rs
rename to tests/run-make-cargo/panic-immediate-abort-works/hello/src/main.rs
diff --git a/tests/run-make-cargo/panic-immediate-abort-works/rmake.rs b/tests/run-make-cargo/panic-immediate-abort-works/rmake.rs
new file mode 100644
index 000000000000..3eeef38c962d
--- /dev/null
+++ b/tests/run-make-cargo/panic-immediate-abort-works/rmake.rs
@@ -0,0 +1,39 @@
+// This test ensures we are able to compile and link a simple binary with panic=immediate-abort.
+// The test panic-immediate-abort-codegen checks that panic strategy produces the desired codegen,
+// but is based on compiling a library crate (which is the norm for codegen tests because it is
+// cleaner and more portable). So this test ensures that we didn't mix up a cfg or a compiler
+// implementation detail in a way that makes panic=immediate-abort encounter errors at link time.
+
+// Ideally this test would be run for most targets, but unfortunately:
+// This test is currently written using `fn main() {}` which requires std.
+// And since the default linker is only a linker for the host, we can't handle cross-compilation.
+// Both of these shortcomings could be addressed at the cost of making the test more complicated.
+//@ needs-target-std
+//@ ignore-cross-compile
+
+#![deny(warnings)]
+
+use run_make_support::{cargo, path, target};
+
+fn main() {
+    let target_dir = path("target");
+
+    cargo()
+        .current_dir("hello")
+        .args(&[
+            "build",
+            "--release",
+            "--manifest-path",
+            "Cargo.toml",
+            "-Zbuild-std",
+            "--target",
+            &target(),
+        ])
+        .env("RUSTFLAGS", "-Zunstable-options -Cpanic=immediate-abort")
+        .env("CARGO_TARGET_DIR", &target_dir)
+        .env("RUSTC_BOOTSTRAP", "1")
+        // Visual Studio 2022 requires that the LIB env var be set so it can
+        // find the Windows SDK.
+        .env("LIB", std::env::var("LIB").unwrap_or_default())
+        .run();
+}
diff --git a/tests/run-make/rustc-crates-on-stable/rmake.rs b/tests/run-make-cargo/rustc-crates-on-stable/rmake.rs
similarity index 100%
rename from tests/run-make/rustc-crates-on-stable/rmake.rs
rename to tests/run-make-cargo/rustc-crates-on-stable/rmake.rs
diff --git a/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/Cargo.toml b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/Cargo.toml
new file mode 100644
index 000000000000..6962028375bb
--- /dev/null
+++ b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "foo"
+version = "0.0.1"
+edition = "2024"
+
+[[example]]
+name = "complex"
+doc-scrape-examples = true
diff --git a/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/complex.rs b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/complex.rs
new file mode 100644
index 000000000000..1cda7e098f45
--- /dev/null
+++ b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/complex.rs
@@ -0,0 +1,3 @@
+fn main() {
+    let mut x = foo::X::new();
+}
diff --git a/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/tester.rs b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/tester.rs
new file mode 100644
index 000000000000..fbd1906ec099
--- /dev/null
+++ b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/examples/tester.rs
@@ -0,0 +1 @@
+// This file MUST exist to trigger the original bug.
diff --git a/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/src/lib.rs b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/src/lib.rs
new file mode 100644
index 000000000000..a9bb0272c787
--- /dev/null
+++ b/tests/run-make-cargo/rustdoc-scrape-examples-paths/foo/src/lib.rs
@@ -0,0 +1,7 @@
+pub struct X;
+
+impl X {
+    pub fn new() -> Self {
+        X
+    }
+}
diff --git a/tests/run-make-cargo/rustdoc-scrape-examples-paths/rmake.rs b/tests/run-make-cargo/rustdoc-scrape-examples-paths/rmake.rs
new file mode 100644
index 000000000000..6784e438762c
--- /dev/null
+++ b/tests/run-make-cargo/rustdoc-scrape-examples-paths/rmake.rs
@@ -0,0 +1,16 @@
+//! Test to ensure that the rustdoc `scrape-examples` feature is not panicking.
+//! Regression test for .
+
+use run_make_support::cargo;
+use run_make_support::scoped_run::run_in_tmpdir;
+
+fn main() {
+    // We copy the crate to be documented "outside" to prevent documenting
+    // the whole compiler.
+    std::env::set_current_dir("foo").unwrap();
+    run_in_tmpdir(|| {
+        // The `scrape-examples` feature is also implemented in `cargo` so instead of reproducing
+        // what `cargo` does, better to just let `cargo` do it.
+        cargo().args(["doc", "-p", "foo", "-Zrustdoc-scrape-examples"]).run();
+    })
+}
diff --git a/tests/run-make-cargo/thumb-none-cortex-m/rmake.rs b/tests/run-make-cargo/thumb-none-cortex-m/rmake.rs
new file mode 100644
index 000000000000..6158b9c36da2
--- /dev/null
+++ b/tests/run-make-cargo/thumb-none-cortex-m/rmake.rs
@@ -0,0 +1,45 @@
+//! Test building of the `cortex-m` crate, a foundational crate in the embedded ecosystem
+//! for a collection of thumb targets. This is a smoke test that verifies that both cargo
+//! and rustc work in this case.
+//!
+//! How to run this
+//! $ ./x.py clean
+//! $ ./x.py test --target thumbv6m-none-eabi,thumbv7m-none-eabi tests/run-make-cargo
+//!
+//! Supported targets:
+//! - thumbv6m-none-eabi (Bare Cortex-M0, M0+, M1)
+//! - thumbv7em-none-eabi (Bare Cortex-M4, M7)
+//! - thumbv7em-none-eabihf (Bare Cortex-M4F, M7F, FPU, hardfloat)
+//! - thumbv7m-none-eabi (Bare Cortex-M3)
+
+//@ only-thumb
+
+use run_make_support::{cargo, cmd, env, env_var, target};
+
+const CRATE: &str = "cortex-m";
+const CRATE_URL: &str = "https://github.com/rust-embedded/cortex-m";
+const CRATE_SHA1: &str = "a448e9156e2cb1e556e5441fd65426952ef4b927"; // v0.5.0
+
+fn main() {
+    // FIXME: requires an internet connection https://github.com/rust-lang/rust/issues/128733
+    // See below link for git usage:
+    // https://stackoverflow.com/questions/3489173#14091182
+    cmd("git").args(["clone", CRATE_URL, CRATE]).run();
+    env::set_current_dir(CRATE);
+    cmd("git").args(["reset", "--hard", CRATE_SHA1]).run();
+
+    cargo()
+        .args(&[
+            "build",
+            "--manifest-path",
+            "Cargo.toml",
+            "-Zbuild-std=core",
+            "--target",
+            &target(),
+        ])
+        .env("CARGO_TARGET_DIR", "target")
+        // Don't make lints fatal, but they need to at least warn
+        // or they break Cargo's target info parsing.
+        .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes --cap-lints=warn")
+        .run();
+}
diff --git a/tests/run-make/thumb-none-qemu/example/.cargo/config.toml b/tests/run-make-cargo/thumb-none-qemu/example/.cargo/config.toml
similarity index 100%
rename from tests/run-make/thumb-none-qemu/example/.cargo/config.toml
rename to tests/run-make-cargo/thumb-none-qemu/example/.cargo/config.toml
diff --git a/tests/run-make/thumb-none-qemu/example/Cargo.lock b/tests/run-make-cargo/thumb-none-qemu/example/Cargo.lock
similarity index 100%
rename from tests/run-make/thumb-none-qemu/example/Cargo.lock
rename to tests/run-make-cargo/thumb-none-qemu/example/Cargo.lock
diff --git a/tests/run-make/thumb-none-qemu/example/Cargo.toml b/tests/run-make-cargo/thumb-none-qemu/example/Cargo.toml
similarity index 100%
rename from tests/run-make/thumb-none-qemu/example/Cargo.toml
rename to tests/run-make-cargo/thumb-none-qemu/example/Cargo.toml
diff --git a/tests/run-make/thumb-none-qemu/example/memory.x b/tests/run-make-cargo/thumb-none-qemu/example/memory.x
similarity index 100%
rename from tests/run-make/thumb-none-qemu/example/memory.x
rename to tests/run-make-cargo/thumb-none-qemu/example/memory.x
diff --git a/tests/run-make/thumb-none-qemu/example/src/main.rs b/tests/run-make-cargo/thumb-none-qemu/example/src/main.rs
similarity index 100%
rename from tests/run-make/thumb-none-qemu/example/src/main.rs
rename to tests/run-make-cargo/thumb-none-qemu/example/src/main.rs
diff --git a/tests/run-make/thumb-none-qemu/rmake.rs b/tests/run-make-cargo/thumb-none-qemu/rmake.rs
similarity index 100%
rename from tests/run-make/thumb-none-qemu/rmake.rs
rename to tests/run-make-cargo/thumb-none-qemu/rmake.rs
diff --git a/tests/run-make/uefi-qemu/rmake.rs b/tests/run-make-cargo/uefi-qemu/rmake.rs
similarity index 100%
rename from tests/run-make/uefi-qemu/rmake.rs
rename to tests/run-make-cargo/uefi-qemu/rmake.rs
diff --git a/tests/run-make/uefi-qemu/uefi_qemu_test/Cargo.lock b/tests/run-make-cargo/uefi-qemu/uefi_qemu_test/Cargo.lock
similarity index 100%
rename from tests/run-make/uefi-qemu/uefi_qemu_test/Cargo.lock
rename to tests/run-make-cargo/uefi-qemu/uefi_qemu_test/Cargo.lock
diff --git a/tests/run-make/uefi-qemu/uefi_qemu_test/Cargo.toml b/tests/run-make-cargo/uefi-qemu/uefi_qemu_test/Cargo.toml
similarity index 100%
rename from tests/run-make/uefi-qemu/uefi_qemu_test/Cargo.toml
rename to tests/run-make-cargo/uefi-qemu/uefi_qemu_test/Cargo.toml
diff --git a/tests/run-make/uefi-qemu/uefi_qemu_test/src/main.rs b/tests/run-make-cargo/uefi-qemu/uefi_qemu_test/src/main.rs
similarity index 100%
rename from tests/run-make/uefi-qemu/uefi_qemu_test/src/main.rs
rename to tests/run-make-cargo/uefi-qemu/uefi_qemu_test/src/main.rs
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_c_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cc_plus_one_cxx_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_global_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_global_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_global_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_c_global_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_global_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_global_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_global_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/cmake_plus_one_cxx_global_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/Cargo.toml b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/Cargo.toml
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/Cargo.toml
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/Cargo.toml
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/build.rs b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/build.rs
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/build.rs
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/build.rs
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo.c b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo.c
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo.c
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo.c
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_asm.s b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_asm.s
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_asm.s
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_asm.s
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_cxx.cpp b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_cxx.cpp
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_cxx.cpp
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/foo_cxx.cpp
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/CMakeLists.txt b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/CMakeLists.txt
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/CMakeLists.txt
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/CMakeLists.txt
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo.c b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo.c
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo.c
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo.c
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_asm.s b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_asm.s
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_asm.s
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_asm.s
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_cxx.cpp b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_cxx.cpp
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_cxx.cpp
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/libcmake_foo/src/foo_cxx.cpp
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/src/main.rs b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/src/main.rs
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/enclave/src/main.rs
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/enclave/src/main.rs
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/jumpto.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/jumpto.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/jumpto.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/jumpto.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/print.with_frame_pointers.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/print.with_frame_pointers.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/print.with_frame_pointers.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/print.with_frame_pointers.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/print.without_frame_pointers.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/print.without_frame_pointers.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/print.without_frame_pointers.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/print.without_frame_pointers.checks
diff --git a/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/rmake.rs b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/rmake.rs
new file mode 100644
index 000000000000..89754cdaf905
--- /dev/null
+++ b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/rmake.rs
@@ -0,0 +1,113 @@
+// ignore-tidy-linelength
+// Reason: intel.com link
+
+// This security test checks that the disassembled form of certain symbols
+// is "hardened" - that means, the assembly instructions match a pattern that
+// mitigate potential Load Value Injection vulnerabilities.
+// To do so, a test crate is compiled, and certain symbols are found, disassembled
+// and checked one by one.
+// See https://github.com/rust-lang/rust/pull/77008
+
+// On load value injection:
+// https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/load-value-injection.html
+
+//@ only-x86_64-fortanix-unknown-sgx
+
+use run_make_support::{
+    cargo, cwd, llvm_filecheck, llvm_objdump, regex, run, set_current_dir, target,
+};
+
+fn main() {
+    cargo()
+        .arg("-v")
+        .arg("build")
+        .arg("--target")
+        .arg(target())
+        .current_dir("enclave")
+        .env("CC_x86_64_fortanix_unknown_sgx", "clang")
+        .env(
+            "CFLAGS_x86_64_fortanix_unknown_sgx",
+            "-D__ELF__ -isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening",
+        )
+        .env("CXX_x86_64_fortanix_unknown_sgx", "clang++")
+        .env(
+            "CXXFLAGS_x86_64_fortanix_unknown_sgx",
+            "-D__ELF__ -isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening",
+        )
+        .run();
+
+    // Rust has several ways of including machine code into a binary:
+    //
+    // - Rust code
+    // - Inline assembly
+    // - Global assembly
+    // - C/C++ code compiled as part of Rust crates
+    //
+    // For each of those, check that the mitigations are applied. Mostly we check
+    // that ret instructions are no longer present.
+
+    // Check that normal rust code has the right mitigations.
+    check("unw_getcontext", "unw_getcontext.checks");
+    check("__libunwind_Registers_x86_64_jumpto", "jumpto.checks");
+
+    check("std::io::stdio::_print::[[:alnum:]]+", "print.with_frame_pointers.checks");
+
+    // Check that rust global assembly has the right mitigations.
+    check("rust_plus_one_global_asm", "rust_plus_one_global_asm.checks");
+
+    // Check that C code compiled using the `cc` crate has the right mitigations.
+    check("cc_plus_one_c", "cc_plus_one_c.checks");
+    check("cc_plus_one_c_asm", "cc_plus_one_c_asm.checks");
+    check("cc_plus_one_cxx", "cc_plus_one_cxx.checks");
+    check("cc_plus_one_cxx_asm", "cc_plus_one_cxx_asm.checks");
+    check("cc_plus_one_asm", "cc_plus_one_asm.checks");
+
+    // Check that C++ code compiled using the `cc` crate has the right mitigations.
+    check("cmake_plus_one_c", "cmake_plus_one_c.checks");
+    check("cmake_plus_one_c_asm", "cmake_plus_one_c_asm.checks");
+    check("cmake_plus_one_c_global_asm", "cmake_plus_one_c_global_asm.checks");
+    check("cmake_plus_one_cxx", "cmake_plus_one_cxx.checks");
+    check("cmake_plus_one_cxx_asm", "cmake_plus_one_cxx_asm.checks");
+    check("cmake_plus_one_cxx_global_asm", "cmake_plus_one_cxx_global_asm.checks");
+    check("cmake_plus_one_asm", "cmake_plus_one_asm.checks");
+}
+
+fn check(func_re: &str, mut checks: &str) {
+    let dump = llvm_objdump()
+        .input("enclave/target/x86_64-fortanix-unknown-sgx/debug/enclave")
+        .args(&["--syms", "--demangle"])
+        .run()
+        .stdout_utf8();
+    let re = regex::Regex::new(&format!("[[:blank:]]+{func_re}")).unwrap();
+    let func = re.find_iter(&dump).map(|m| m.as_str().trim()).collect::>().join(",");
+    assert!(!func.is_empty());
+    let dump = llvm_objdump()
+        .input("enclave/target/x86_64-fortanix-unknown-sgx/debug/enclave")
+        .args(&["--demangle", &format!("--disassemble-symbols={func}")])
+        .run()
+        .stdout();
+
+    // Unique case, must succeed at one of two possible tests.
+    // This is because frame pointers are optional, and them being enabled requires
+    // an additional `popq` in the pattern checking file.
+    if func_re == "std::io::stdio::_print::[[:alnum:]]+" {
+        let output = llvm_filecheck().stdin_buf(&dump).patterns(checks).run_unchecked();
+        if !output.status().success() {
+            checks = "print.without_frame_pointers.checks";
+            llvm_filecheck().stdin_buf(&dump).patterns(checks).run();
+        }
+    } else {
+        llvm_filecheck().stdin_buf(&dump).patterns(checks).run();
+    }
+    if !["rust_plus_one_global_asm", "cmake_plus_one_c_global_asm", "cmake_plus_one_cxx_global_asm"]
+        .contains(&func_re)
+    {
+        // The assembler cannot avoid explicit `ret` instructions. Sequences
+        // of `shlq $0x0, (%rsp); lfence; retq` are used instead.
+        llvm_filecheck()
+            .args(&["--implicit-check-not", "ret"])
+            .stdin_buf(dump)
+            .patterns(checks)
+            .run();
+    }
+}
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/rust_plus_one_global_asm.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/rust_plus_one_global_asm.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/rust_plus_one_global_asm.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/rust_plus_one_global_asm.checks
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/unw_getcontext.checks b/tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/unw_getcontext.checks
similarity index 100%
rename from tests/run-make/x86_64-fortanix-unknown-sgx-lvi/unw_getcontext.checks
rename to tests/run-make-cargo/x86_64-fortanix-unknown-sgx-lvi/unw_getcontext.checks
diff --git a/tests/run-make/apple-c-available-links/foo.c b/tests/run-make/apple-c-available-links/foo.c
new file mode 100644
index 000000000000..eff99a8b12a7
--- /dev/null
+++ b/tests/run-make/apple-c-available-links/foo.c
@@ -0,0 +1,22 @@
+int foo(void) {
+    // Act as if using some API that's a lot newer than the deployment target.
+    //
+    // This forces Clang to insert a call to __isPlatformVersionAtLeast,
+    // and linking will fail if that is not present.
+    if (__builtin_available(
+        macos 1000.0,
+        ios 1000.0,
+        tvos 1000.0,
+        watchos 1000.0,
+        // CI runs below Xcode 15, where `visionos` wasn't a valid key in
+        // `__builtin_available`.
+#ifdef TARGET_OS_VISION
+        visionos 1000.0,
+#endif
+        *
+    )) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
diff --git a/tests/run-make/apple-c-available-links/main.rs b/tests/run-make/apple-c-available-links/main.rs
new file mode 100644
index 000000000000..4ffada43c1b2
--- /dev/null
+++ b/tests/run-make/apple-c-available-links/main.rs
@@ -0,0 +1,7 @@
+unsafe extern "C" {
+    safe fn foo() -> core::ffi::c_int;
+}
+
+fn main() {
+    assert_eq!(foo(), 0);
+}
diff --git a/tests/run-make/apple-c-available-links/rmake.rs b/tests/run-make/apple-c-available-links/rmake.rs
new file mode 100644
index 000000000000..44a5ee94d577
--- /dev/null
+++ b/tests/run-make/apple-c-available-links/rmake.rs
@@ -0,0 +1,14 @@
+//! Test that using `__builtin_available` in C (`@available` in Objective-C)
+//! successfully links (because `std` provides the required symbols).
+
+//@ only-apple __builtin_available is (mostly) specific to Apple platforms.
+
+use run_make_support::{cc, rustc, target};
+
+fn main() {
+    // Invoke the C compiler to generate an object file.
+    cc().arg("-c").input("foo.c").output("foo.o").run();
+
+    // Link the object file together with a Rust program.
+    rustc().target(target()).input("main.rs").link_arg("foo.o").run();
+}
diff --git a/tests/run-make/apple-deployment-target/rmake.rs b/tests/run-make/apple-deployment-target/rmake.rs
index 7297a8622240..5d4512843d53 100644
--- a/tests/run-make/apple-deployment-target/rmake.rs
+++ b/tests/run-make/apple-deployment-target/rmake.rs
@@ -33,6 +33,7 @@ fn main() {
         // armv7s-apple-ios and i386-apple-ios only supports iOS 10.0
         "ios" if target() == "armv7s-apple-ios" || target() == "i386-apple-ios" => ("10.0", "10.0"),
         "ios" => ("15.0", "16.0"),
+        "watchos" if target() == "aarch64-apple-watchos" => ("28.0", "30.0"),
         "watchos" => ("7.0", "9.0"),
         "tvos" => ("14.0", "15.0"),
         "visionos" => ("1.1", "1.2"),
diff --git a/tests/run-make/atomic-lock-free/atomic_lock_free.rs b/tests/run-make/atomic-lock-free/atomic_lock_free.rs
index f5c3b360ee81..92ffd111ce8b 100644
--- a/tests/run-make/atomic-lock-free/atomic_lock_free.rs
+++ b/tests/run-make/atomic-lock-free/atomic_lock_free.rs
@@ -14,7 +14,7 @@ pub enum AtomicOrdering {
 }
 
 #[rustc_intrinsic]
-unsafe fn atomic_xadd(dst: *mut T, src: T) -> T;
+unsafe fn atomic_xadd(dst: *mut T, src: U) -> T;
 
 #[lang = "pointee_sized"]
 pub trait PointeeSized {}
@@ -35,51 +35,62 @@ impl Copy for *mut T {}
 impl ConstParamTy for AtomicOrdering {}
 
 #[cfg(target_has_atomic = "8")]
+#[unsafe(no_mangle)] // let's make sure we actually generate a symbol to check
 pub unsafe fn atomic_u8(x: *mut u8) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u8);
 }
 #[cfg(target_has_atomic = "8")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_i8(x: *mut i8) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i8);
 }
 #[cfg(target_has_atomic = "16")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_u16(x: *mut u16) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u16);
 }
 #[cfg(target_has_atomic = "16")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_i16(x: *mut i16) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i16);
 }
 #[cfg(target_has_atomic = "32")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_u32(x: *mut u32) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u32);
 }
 #[cfg(target_has_atomic = "32")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_i32(x: *mut i32) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i32);
 }
 #[cfg(target_has_atomic = "64")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_u64(x: *mut u64) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u64);
 }
 #[cfg(target_has_atomic = "64")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_i64(x: *mut i64) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i64);
 }
 #[cfg(target_has_atomic = "128")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_u128(x: *mut u128) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1u128);
 }
 #[cfg(target_has_atomic = "128")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_i128(x: *mut i128) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1i128);
 }
 #[cfg(target_has_atomic = "ptr")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_usize(x: *mut usize) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1usize);
 }
 #[cfg(target_has_atomic = "ptr")]
+#[unsafe(no_mangle)]
 pub unsafe fn atomic_isize(x: *mut isize) {
-    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
+    atomic_xadd::<_, _, { AtomicOrdering::SeqCst }>(x, 1isize);
 }
diff --git a/tests/run-make/autodiff/type-trees/array-typetree/array.check b/tests/run-make/autodiff/type-trees/array-typetree/array.check
new file mode 100644
index 000000000000..0d38bdec17e8
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/array-typetree/array.check
@@ -0,0 +1,4 @@
+; Check that array TypeTree metadata is correctly generated  
+; Should show Float@double at each array element offset (0, 8, 16, 24, 32 bytes)
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_array{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,-1]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/array-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/array-typetree/rmake.rs
new file mode 100644
index 000000000000..20b6a0669062
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/array-typetree/rmake.rs
@@ -0,0 +1,9 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+    llvm_filecheck().patterns("array.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/array-typetree/test.rs b/tests/run-make/autodiff/type-trees/array-typetree/test.rs
new file mode 100644
index 000000000000..f54ebf5a4c7b
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/array-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_array(arr: &[f64; 5]) -> f64 {
+    arr[0] + arr[1] + arr[2] + arr[3] + arr[4]
+}
+
+fn main() {
+    let arr = [1.0, 2.0, 3.0, 4.0, 5.0];
+    let mut d_arr = [0.0; 5];
+    let _result = d_test(&arr, &mut d_arr, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy-ir.check b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy-ir.check
new file mode 100644
index 000000000000..0e6351ac4d39
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy-ir.check
@@ -0,0 +1,8 @@
+; Check that enzyme_type attributes are present in the LLVM IR function definition
+; This verifies our TypeTree system correctly attaches metadata for Enzyme
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_memcpy({{.*}}"enzyme_type"="{[-1]:Pointer, [-1,-1]:Float@double}"
+
+; Check that llvm.memcpy exists (either call or declare)
+CHECK: {{(call|declare).*}}@llvm.memcpy
+
diff --git a/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.check b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.check
new file mode 100644
index 000000000000..ae70830297a7
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.check
@@ -0,0 +1,13 @@
+CHECK: force_memcpy
+
+CHECK: @llvm.memcpy.p0.p0.i64
+
+CHECK: test_memcpy - {[-1]:Float@double} |{[-1]:Pointer}:{}
+
+CHECK-DAG: ptr %{{[0-9]+}}: {[-1]:Pointer, [-1,0]:Float@double, [-1,8]:Float@double, [-1,16]:Float@double, [-1,24]:Float@double}
+
+CHECK-DAG: load double{{.*}}: {[-1]:Float@double}
+
+CHECK-DAG: fmul double{{.*}}: {[-1]:Float@double}
+
+CHECK-DAG: fadd double{{.*}}: {[-1]:Float@double}
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.rs b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.rs
new file mode 100644
index 000000000000..3c1029190c88
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/memcpy-typetree/memcpy.rs
@@ -0,0 +1,36 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+use std::ptr;
+
+#[inline(never)]
+fn force_memcpy(src: *const f64, dst: *mut f64, count: usize) {
+    unsafe {
+        ptr::copy_nonoverlapping(src, dst, count);
+    }
+}
+
+#[autodiff_reverse(d_test_memcpy, Duplicated, Active)]
+#[no_mangle]
+fn test_memcpy(input: &[f64; 128]) -> f64 {
+    let mut local_data = [0.0f64; 128];
+
+    // Use a separate function to prevent inlining and optimization
+    force_memcpy(input.as_ptr(), local_data.as_mut_ptr(), 128);
+
+    // Sum only first few elements to keep the computation simple
+    local_data[0] * local_data[0]
+        + local_data[1] * local_data[1]
+        + local_data[2] * local_data[2]
+        + local_data[3] * local_data[3]
+}
+
+fn main() {
+    let input = [1.0; 128];
+    let mut d_input = [0.0; 128];
+    let result = test_memcpy(&input);
+    let result_d = d_test_memcpy(&input, &mut d_input, 1.0);
+
+    assert_eq!(result, result_d);
+    println!("Memcpy test passed: result = {}", result);
+}
diff --git a/tests/run-make/autodiff/type-trees/memcpy-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/memcpy-typetree/rmake.rs
new file mode 100644
index 000000000000..b4c650330fe9
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/memcpy-typetree/rmake.rs
@@ -0,0 +1,39 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // First, compile to LLVM IR to check for enzyme_type attributes
+    let _ir_output = rustc()
+        .input("memcpy.rs")
+        .arg("-Zautodiff=Enable")
+        .arg("-Zautodiff=NoPostopt")
+        .opt_level("0")
+        .arg("--emit=llvm-ir")
+        .arg("-o")
+        .arg("main.ll")
+        .run();
+
+    // Then compile with TypeTree analysis output for the existing checks
+    let output = rustc()
+        .input("memcpy.rs")
+        .arg("-Zautodiff=Enable,PrintTAFn=test_memcpy")
+        .arg("-Zautodiff=NoPostopt")
+        .opt_level("3")
+        .arg("-Clto=fat")
+        .arg("-g")
+        .run();
+
+    let stdout = output.stdout_utf8();
+    let stderr = output.stderr_utf8();
+    let ir_content = rfs::read_to_string("main.ll");
+
+    rfs::write("memcpy.stdout", &stdout);
+    rfs::write("memcpy.stderr", &stderr);
+    rfs::write("main.ir", &ir_content);
+
+    llvm_filecheck().patterns("memcpy.check").stdin_buf(stdout).run();
+
+    llvm_filecheck().patterns("memcpy-ir.check").stdin_buf(ir_content).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/mixed-struct-typetree/mixed.check b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/mixed.check
new file mode 100644
index 000000000000..584f58408435
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/mixed.check
@@ -0,0 +1,2 @@
+; Check that mixed struct with large array generates correct detailed type tree
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@float}"{{.*}}@test_mixed_struct{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Integer, [-1,8]:Float@float}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/mixed-struct-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/rmake.rs
new file mode 100644
index 000000000000..1c19963bc361
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/rmake.rs
@@ -0,0 +1,16 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc()
+        .input("test.rs")
+        .arg("-Zautodiff=Enable")
+        .arg("-Zautodiff=NoPostopt")
+        .opt_level("0")
+        .emit("llvm-ir")
+        .run();
+
+    llvm_filecheck().patterns("mixed.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/mixed-struct-typetree/test.rs b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/test.rs
new file mode 100644
index 000000000000..7a734980e617
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/mixed-struct-typetree/test.rs
@@ -0,0 +1,23 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[repr(C)]
+struct Container {
+    header: i64,
+    data: [f32; 1000],
+}
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+#[inline(never)]
+fn test_mixed_struct(container: &Container) -> f32 {
+    container.data[0] + container.data[999]
+}
+
+fn main() {
+    let container = Container { header: 42, data: [1.0; 1000] };
+    let mut d_container = Container { header: 0, data: [0.0; 1000] };
+    let result = d_test(&container, &mut d_container, 1.0);
+    std::hint::black_box(result);
+}
diff --git a/tests/run-make/autodiff/type-trees/nott-flag/nott.check b/tests/run-make/autodiff/type-trees/nott-flag/nott.check
new file mode 100644
index 000000000000..8d23e2ee3195
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/nott-flag/nott.check
@@ -0,0 +1,5 @@
+// Check that enzyme_type attributes are NOT present when NoTT flag is used
+// This verifies the NoTT flag correctly disables TypeTree metadata
+
+CHECK: define{{.*}}@square
+CHECK-NOT: "enzyme_type"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/nott-flag/rmake.rs b/tests/run-make/autodiff/type-trees/nott-flag/rmake.rs
new file mode 100644
index 000000000000..de540b990cab
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/nott-flag/rmake.rs
@@ -0,0 +1,30 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Test with NoTT flag - should not generate TypeTree metadata
+    rustc()
+        .input("test.rs")
+        .arg("-Zautodiff=Enable,NoTT")
+        .emit("llvm-ir")
+        .arg("-o")
+        .arg("nott.ll")
+        .run();
+
+    // Test without NoTT flag - should generate TypeTree metadata
+    rustc()
+        .input("test.rs")
+        .arg("-Zautodiff=Enable")
+        .emit("llvm-ir")
+        .arg("-o")
+        .arg("with_tt.ll")
+        .run();
+
+    // Verify NoTT version does NOT have enzyme_type attributes
+    llvm_filecheck().patterns("nott.check").stdin_buf(rfs::read("nott.ll")).run();
+
+    // Verify TypeTree version DOES have enzyme_type attributes
+    llvm_filecheck().patterns("with_tt.check").stdin_buf(rfs::read("with_tt.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/nott-flag/test.rs b/tests/run-make/autodiff/type-trees/nott-flag/test.rs
new file mode 100644
index 000000000000..de3549c37c67
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/nott-flag/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_square, Duplicated, Active)]
+#[no_mangle]
+fn square(x: &f64) -> f64 {
+    x * x
+}
+
+fn main() {
+    let x = 2.0;
+    let mut dx = 0.0;
+    let _result = d_square(&x, &mut dx, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/nott-flag/with_tt.check b/tests/run-make/autodiff/type-trees/nott-flag/with_tt.check
new file mode 100644
index 000000000000..0b4c91191798
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/nott-flag/with_tt.check
@@ -0,0 +1,4 @@
+// Check that enzyme_type attributes are present when TypeTree is enabled
+// This verifies our TypeTree metadata attachment is working
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@square{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/recursion-typetree/recursion.check b/tests/run-make/autodiff/type-trees/recursion-typetree/recursion.check
new file mode 100644
index 000000000000..1960e7b816c5
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/recursion-typetree/recursion.check
@@ -0,0 +1,3 @@
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_deep{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double}"
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_graph{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Integer, [-1,8]:Integer, [-1,16]:Integer, [-1,24]:Float@double}"
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_node{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/recursion-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/recursion-typetree/rmake.rs
new file mode 100644
index 000000000000..78718f3a2159
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/recursion-typetree/rmake.rs
@@ -0,0 +1,9 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+    llvm_filecheck().patterns("recursion.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/recursion-typetree/test.rs b/tests/run-make/autodiff/type-trees/recursion-typetree/test.rs
new file mode 100644
index 000000000000..9d40bec1bf1d
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/recursion-typetree/test.rs
@@ -0,0 +1,100 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+// Self-referential struct to test recursion detection
+#[derive(Clone)]
+struct Node {
+    value: f64,
+    next: Option>,
+}
+
+// Mutually recursive structs to test cycle detection
+#[derive(Clone)]
+struct GraphNodeA {
+    value: f64,
+    connections: Vec,
+}
+
+#[derive(Clone)]
+struct GraphNodeB {
+    weight: f64,
+    target: Option>,
+}
+
+#[autodiff_reverse(d_test_node, Duplicated, Active)]
+#[no_mangle]
+fn test_node(node: &Node) -> f64 {
+    node.value * 2.0
+}
+
+#[autodiff_reverse(d_test_graph, Duplicated, Active)]
+#[no_mangle]
+fn test_graph(a: &GraphNodeA) -> f64 {
+    a.value * 3.0
+}
+
+// Simple depth test - deeply nested but not circular
+#[derive(Clone)]
+struct Level1 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level2 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level3 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level4 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level5 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level6 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level7 {
+    val: f64,
+    next: Option>,
+}
+#[derive(Clone)]
+struct Level8 {
+    val: f64,
+}
+
+#[autodiff_reverse(d_test_deep, Duplicated, Active)]
+#[no_mangle]
+fn test_deep(deep: &Level1) -> f64 {
+    deep.val * 4.0
+}
+
+fn main() {
+    let node = Node { value: 1.0, next: None };
+
+    let graph = GraphNodeA { value: 2.0, connections: vec![] };
+
+    let deep = Level1 { val: 5.0, next: None };
+
+    let mut d_node = Node { value: 0.0, next: None };
+
+    let mut d_graph = GraphNodeA { value: 0.0, connections: vec![] };
+
+    let mut d_deep = Level1 { val: 0.0, next: None };
+
+    let _result1 = d_test_node(&node, &mut d_node, 1.0);
+    let _result2 = d_test_graph(&graph, &mut d_graph, 1.0);
+    let _result3 = d_test_deep(&deep, &mut d_deep, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/f128.check b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/f128.check
new file mode 100644
index 000000000000..23db64eea52a
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/f128.check
@@ -0,0 +1,4 @@
+; Check that f128 TypeTree metadata is correctly generated
+; Should show Float@fp128 for f128 values and Pointer for references
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@fp128}"{{.*}}@test_f128{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@fp128}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/rmake.rs
new file mode 100644
index 000000000000..44320ecdd571
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/rmake.rs
@@ -0,0 +1,12 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Compile with TypeTree enabled and emit LLVM IR
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+
+    // Check that f128 TypeTree metadata is correctly generated
+    llvm_filecheck().patterns("f128.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/test.rs b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/test.rs
new file mode 100644
index 000000000000..5c71baa3e699
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f128-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff, f128)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_f128(x: &f128) -> f128 {
+    *x * *x
+}
+
+fn main() {
+    let x = 2.0_f128;
+    let mut dx = 0.0_f128;
+    let _result = d_test(&x, &mut dx, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/f16.check b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/f16.check
new file mode 100644
index 000000000000..9adff68d36f3
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/f16.check
@@ -0,0 +1,4 @@
+; Check that f16 TypeTree metadata is correctly generated
+; Should show Float@half for f16 values and Pointer for references
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@half}"{{.*}}@test_f16{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@half}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/rmake.rs
new file mode 100644
index 000000000000..0aebdbf55209
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/rmake.rs
@@ -0,0 +1,12 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Compile with TypeTree enabled and emit LLVM IR
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+
+    // Check that f16 TypeTree metadata is correctly generated
+    llvm_filecheck().patterns("f16.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/test.rs b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/test.rs
new file mode 100644
index 000000000000..6b68e8252f4c
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f16-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff, f16)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_f16(x: &f16) -> f16 {
+    *x * *x
+}
+
+fn main() {
+    let x = 2.0_f16;
+    let mut dx = 0.0_f16;
+    let _result = d_test(&x, &mut dx, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/f32.check b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/f32.check
new file mode 100644
index 000000000000..176630f57e8f
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/f32.check
@@ -0,0 +1,4 @@
+; Check that f32 TypeTree metadata is correctly generated
+; Should show Float@float for f32 values and Pointer for references
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@float}"{{.*}}@test_f32{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@float}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/rmake.rs
new file mode 100644
index 000000000000..ee3ab753bf50
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/rmake.rs
@@ -0,0 +1,12 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Compile with TypeTree enabled and emit LLVM IR
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+
+    // Check that f32 TypeTree metadata is correctly generated
+    llvm_filecheck().patterns("f32.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/test.rs b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/test.rs
new file mode 100644
index 000000000000..56c118399ee4
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f32-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_f32(x: &f32) -> f32 {
+    x * x
+}
+
+fn main() {
+    let x = 2.0_f32;
+    let mut dx = 0.0_f32;
+    let _result = d_test(&x, &mut dx, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/f64.check b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/f64.check
new file mode 100644
index 000000000000..929cd379694a
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/f64.check
@@ -0,0 +1,4 @@
+; Check that f64 TypeTree metadata is correctly generated  
+; Should show Float@double for f64 values and Pointer for references
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_f64{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/rmake.rs
new file mode 100644
index 000000000000..5fac9b23bc80
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/rmake.rs
@@ -0,0 +1,12 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Compile with TypeTree enabled and emit LLVM IR
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+
+    // Check that f64 TypeTree metadata is correctly generated
+    llvm_filecheck().patterns("f64.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/test.rs b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/test.rs
new file mode 100644
index 000000000000..235360b76b23
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/f64-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_f64(x: &f64) -> f64 {
+    x * x
+}
+
+fn main() {
+    let x = 2.0_f64;
+    let mut dx = 0.0_f64;
+    let _result = d_test(&x, &mut dx, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/i32.check b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/i32.check
new file mode 100644
index 000000000000..dee4aa5bbb6b
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/i32.check
@@ -0,0 +1,4 @@
+; Check that i32 TypeTree metadata is correctly generated
+; Should show Integer for i32 values and Pointer for references
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Integer}"{{.*}}@test_i32{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Integer}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/rmake.rs
new file mode 100644
index 000000000000..a40fd55d88ad
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/rmake.rs
@@ -0,0 +1,12 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    // Compile with TypeTree enabled and emit LLVM IR
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+
+    // Check that i32 TypeTree metadata is correctly generated
+    llvm_filecheck().patterns("i32.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/test.rs b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/test.rs
new file mode 100644
index 000000000000..249803c5d9f7
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/scalar-types/i32-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_i32(x: &i32) -> i32 {
+    x * x
+}
+
+fn main() {
+    let x = 5_i32;
+    let mut dx = 0_i32;
+    let _result = d_test(&x, &mut dx, 1);
+}
diff --git a/tests/run-make/autodiff/type-trees/slice-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/slice-typetree/rmake.rs
new file mode 100644
index 000000000000..b81fb50bf1a7
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/slice-typetree/rmake.rs
@@ -0,0 +1,9 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+    llvm_filecheck().patterns("slice.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/slice-typetree/slice.check b/tests/run-make/autodiff/type-trees/slice-typetree/slice.check
new file mode 100644
index 000000000000..6543b6161153
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/slice-typetree/slice.check
@@ -0,0 +1,4 @@
+; Check that slice TypeTree metadata is correctly generated
+; Should show Float@double for slice elements
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_slice{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,-1]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/slice-typetree/test.rs b/tests/run-make/autodiff/type-trees/slice-typetree/test.rs
new file mode 100644
index 000000000000..7117fa3844f5
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/slice-typetree/test.rs
@@ -0,0 +1,16 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_slice(slice: &[f64]) -> f64 {
+    slice.iter().sum()
+}
+
+fn main() {
+    let arr = [1.0, 2.0, 3.0, 4.0, 5.0];
+    let slice = &arr[..];
+    let mut d_slice = [0.0; 5];
+    let _result = d_test(slice, &mut d_slice[..], 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/struct-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/struct-typetree/rmake.rs
new file mode 100644
index 000000000000..0af1b65ee181
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/struct-typetree/rmake.rs
@@ -0,0 +1,9 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+    llvm_filecheck().patterns("struct.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/struct-typetree/struct.check b/tests/run-make/autodiff/type-trees/struct-typetree/struct.check
new file mode 100644
index 000000000000..54956317e1e9
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/struct-typetree/struct.check
@@ -0,0 +1,4 @@
+; Check that struct TypeTree metadata is correctly generated
+; Should show Float@double at offsets 0, 8, 16 for Point struct fields
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_struct{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double, [-1,8]:Float@double, [-1,16]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/struct-typetree/test.rs b/tests/run-make/autodiff/type-trees/struct-typetree/test.rs
new file mode 100644
index 000000000000..cbe7b10e4097
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/struct-typetree/test.rs
@@ -0,0 +1,22 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[repr(C)]
+struct Point {
+    x: f64,
+    y: f64,
+    z: f64,
+}
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_struct(point: &Point) -> f64 {
+    point.x + point.y * 2.0 + point.z * 3.0
+}
+
+fn main() {
+    let point = Point { x: 1.0, y: 2.0, z: 3.0 };
+    let mut d_point = Point { x: 0.0, y: 0.0, z: 0.0 };
+    let _result = d_test(&point, &mut d_point, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/tuple-typetree/rmake.rs b/tests/run-make/autodiff/type-trees/tuple-typetree/rmake.rs
new file mode 100644
index 000000000000..76913828901c
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/tuple-typetree/rmake.rs
@@ -0,0 +1,9 @@
+//@ needs-enzyme
+//@ ignore-cross-compile
+
+use run_make_support::{llvm_filecheck, rfs, rustc};
+
+fn main() {
+    rustc().input("test.rs").arg("-Zautodiff=Enable").emit("llvm-ir").run();
+    llvm_filecheck().patterns("tuple.check").stdin_buf(rfs::read("test.ll")).run();
+}
diff --git a/tests/run-make/autodiff/type-trees/tuple-typetree/test.rs b/tests/run-make/autodiff/type-trees/tuple-typetree/test.rs
new file mode 100644
index 000000000000..32187b587a38
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/tuple-typetree/test.rs
@@ -0,0 +1,15 @@
+#![feature(autodiff)]
+
+use std::autodiff::autodiff_reverse;
+
+#[autodiff_reverse(d_test, Duplicated, Active)]
+#[no_mangle]
+fn test_tuple(tuple: &(f64, f64, f64)) -> f64 {
+    tuple.0 + tuple.1 * 2.0 + tuple.2 * 3.0
+}
+
+fn main() {
+    let tuple = (1.0, 2.0, 3.0);
+    let mut d_tuple = (0.0, 0.0, 0.0);
+    let _result = d_test(&tuple, &mut d_tuple, 1.0);
+}
diff --git a/tests/run-make/autodiff/type-trees/tuple-typetree/tuple.check b/tests/run-make/autodiff/type-trees/tuple-typetree/tuple.check
new file mode 100644
index 000000000000..47647e78cc35
--- /dev/null
+++ b/tests/run-make/autodiff/type-trees/tuple-typetree/tuple.check
@@ -0,0 +1,4 @@
+; Check that tuple TypeTree metadata is correctly generated
+; Should show Float@double at offsets 0, 8, 16 for (f64, f64, f64)
+
+CHECK: define{{.*}}"enzyme_type"="{[-1]:Float@double}"{{.*}}@test_tuple{{.*}}"enzyme_type"="{[-1]:Pointer, [-1,0]:Float@double, [-1,8]:Float@double, [-1,16]:Float@double}"
\ No newline at end of file
diff --git a/tests/run-make/autodiff/type-trees/type-analysis/vec/vec.check b/tests/run-make/autodiff/type-trees/type-analysis/vec/vec.check
index dcf9508b69d6..cdb70eb83fc1 100644
--- a/tests/run-make/autodiff/type-trees/type-analysis/vec/vec.check
+++ b/tests/run-make/autodiff/type-trees/type-analysis/vec/vec.check
@@ -1,7 +1,7 @@
 // CHECK: callee - {[-1]:Float@float} |{[-1]:Pointer}:{}
 // CHECK: ptr %{{[0-9]+}}: {[-1]:Pointer}
 // CHECK-DAG: %{{[0-9]+}} = getelementptr inbounds nuw i8, ptr %{{[0-9]+}}, i64 8, !dbg !{{[0-9]+}}: {[-1]:Pointer}
-// CHECK-DAG: %{{[0-9]+}} = load ptr, ptr %{{[0-9]+}}, align 8, !dbg !{{[0-9]+}}, !nonnull !102, !noundef !{{[0-9]+}}: {}
+// CHECK-DAG: %{{[0-9]+}} = load ptr, ptr %{{[0-9]+}}, align 8, !dbg !{{[0-9]+}}, !nonnull !{{[0-9]+}}, !noundef !{{[0-9]+}}: {}
 // CHECK-DAG: %{{[0-9]+}} = getelementptr inbounds nuw i8, ptr %{{[0-9]+}}, i64 16, !dbg !{{[0-9]+}}: {[-1]:Pointer}
 // CHECK-DAG: %{{[0-9]+}} = load i64, ptr %{{[0-9]+}}, align 8, !dbg !{{[0-9]+}}, !noundef !{{[0-9]+}}: {}
 // CHECK-DAG: %{{[0-9]+}} = icmp eq i64 %{{[0-9]+}}, 0, !dbg !{{[0-9]+}}: {[-1]:Integer}
diff --git a/tests/run-make/bin-emit-no-symbols/rmake.rs b/tests/run-make/bin-emit-no-symbols/rmake.rs
index 2faeb20025bb..0822bd6dfd6a 100644
--- a/tests/run-make/bin-emit-no-symbols/rmake.rs
+++ b/tests/run-make/bin-emit-no-symbols/rmake.rs
@@ -14,5 +14,5 @@ fn main() {
     let out = llvm_readobj().input("app.o").arg("--symbols").run();
     out.assert_stdout_contains("rust_begin_unwind");
     out.assert_stdout_contains("rust_eh_personality");
-    out.assert_stdout_contains("__rg_oom");
+    out.assert_stdout_contains("__rust_alloc_error_handler");
 }
diff --git a/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-disabled.stderr b/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-disabled.stderr
index 464208f989e3..895558f3b0fc 100644
--- a/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-disabled.stderr
+++ b/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-disabled.stderr
@@ -1,20 +1,20 @@
 error: `[const]` is not allowed here
   --> const-super-trait.rs:7:12
    |
-LL | trait Bar: ~const Foo {}
-   |            ^^^^^^
+LL | trait Bar: [const] Foo {}
+   |            ^^^^^^^
    |
 note: this trait is not `const`, so it cannot have `[const]` trait bounds
   --> const-super-trait.rs:7:1
    |
-LL | trait Bar: ~const Foo {}
-   | ^^^^^^^^^^^^^^^^^^^^^^^^
+LL | trait Bar: [const] Foo {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0658]: const trait impls are experimental
   --> const-super-trait.rs:7:12
    |
-LL | trait Bar: ~const Foo {}
-   |            ^^^^^^
+LL | trait Bar: [const] Foo {}
+   |            ^^^^^^^
    |
    = note: see issue #143874  for more information
    = help: add `#![feature(const_trait_impl)]` to the crate attributes to enable
@@ -23,8 +23,8 @@ LL | trait Bar: ~const Foo {}
 error[E0658]: const trait impls are experimental
   --> const-super-trait.rs:9:17
    |
-LL | const fn foo(x: &T) {
-   |                 ^^^^^^
+LL | const fn foo(x: &T) {
+   |                 ^^^^^^^
    |
    = note: see issue #143874  for more information
    = help: add `#![feature(const_trait_impl)]` to the crate attributes to enable
@@ -33,8 +33,8 @@ LL | const fn foo(x: &T) {
 error: `[const]` can only be applied to `const` traits
   --> const-super-trait.rs:7:12
    |
-LL | trait Bar: ~const Foo {}
-   |            ^^^^^^ can't be applied to `Foo`
+LL | trait Bar: [const] Foo {}
+   |            ^^^^^^^ can't be applied to `Foo`
    |
 help: enable `#![feature(const_trait_impl)]` in your crate and mark `Foo` as `const` to allow it to have `const` implementations
    |
@@ -44,12 +44,12 @@ LL | #[const_trait] trait Foo {
 error: `[const]` can only be applied to `const` traits
   --> const-super-trait.rs:9:17
    |
-LL | const fn foo(x: &T) {
-   |                 ^^^^^^ can't be applied to `Bar`
+LL | const fn foo(x: &T) {
+   |                 ^^^^^^^ can't be applied to `Bar`
    |
 help: enable `#![feature(const_trait_impl)]` in your crate and mark `Bar` as `const` to allow it to have `const` implementations
    |
-LL | #[const_trait] trait Bar: ~const Foo {}
+LL | #[const_trait] trait Bar: [const] Foo {}
    | ++++++++++++++
 
 error[E0015]: cannot call non-const method `::a` in constant functions
diff --git a/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-enabled.stderr b/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-enabled.stderr
index 569e559186f7..821ab6fa57cd 100644
--- a/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-enabled.stderr
+++ b/tests/run-make/const-trait-stable-toolchain/const-super-trait-nightly-enabled.stderr
@@ -1,20 +1,20 @@
 error: `[const]` is not allowed here
   --> const-super-trait.rs:7:12
    |
-LL | trait Bar: ~const Foo {}
-   |            ^^^^^^
+LL | trait Bar: [const] Foo {}
+   |            ^^^^^^^
    |
 note: this trait is not `const`, so it cannot have `[const]` trait bounds
   --> const-super-trait.rs:7:1
    |
-LL | trait Bar: ~const Foo {}
-   | ^^^^^^^^^^^^^^^^^^^^^^^^
+LL | trait Bar: [const] Foo {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: `[const]` can only be applied to `const` traits
   --> const-super-trait.rs:7:12
    |
-LL | trait Bar: ~const Foo {}
-   |            ^^^^^^ can't be applied to `Foo`
+LL | trait Bar: [const] Foo {}
+   |            ^^^^^^^ can't be applied to `Foo`
    |
 help: mark `Foo` as `const` to allow it to have `const` implementations
    |
@@ -24,12 +24,12 @@ LL | #[const_trait] trait Foo {
 error: `[const]` can only be applied to `const` traits
   --> const-super-trait.rs:9:17
    |
-LL | const fn foo(x: &T) {
-   |                 ^^^^^^ can't be applied to `Bar`
+LL | const fn foo(x: &T) {
+   |                 ^^^^^^^ can't be applied to `Bar`
    |
 help: mark `Bar` as `const` to allow it to have `const` implementations
    |
-LL | #[const_trait] trait Bar: ~const Foo {}
+LL | #[const_trait] trait Bar: [const] Foo {}
    | ++++++++++++++
 
 error[E0015]: cannot call non-const method `::a` in constant functions
diff --git a/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-disabled.stderr b/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-disabled.stderr
index 694e06fb6ea2..b39be78e4384 100644
--- a/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-disabled.stderr
+++ b/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-disabled.stderr
@@ -1,36 +1,36 @@
 error: `[const]` is not allowed here
  --> const-super-trait.rs:7:12
   |
-7 | trait Bar: ~const Foo {}
-  |            ^^^^^^
+7 | trait Bar: [const] Foo {}
+  |            ^^^^^^^
   |
 note: this trait is not `const`, so it cannot have `[const]` trait bounds
  --> const-super-trait.rs:7:1
   |
-7 | trait Bar: ~const Foo {}
-  | ^^^^^^^^^^^^^^^^^^^^^^^^
+7 | trait Bar: [const] Foo {}
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0658]: const trait impls are experimental
  --> const-super-trait.rs:7:12
   |
-7 | trait Bar: ~const Foo {}
-  |            ^^^^^^
+7 | trait Bar: [const] Foo {}
+  |            ^^^^^^^
   |
   = note: see issue #143874  for more information
 
 error[E0658]: const trait impls are experimental
  --> const-super-trait.rs:9:17
   |
-9 | const fn foo(x: &T) {
-  |                 ^^^^^^
+9 | const fn foo(x: &T) {
+  |                 ^^^^^^^
   |
   = note: see issue #143874  for more information
 
 error: `[const]` can only be applied to `const` traits
  --> const-super-trait.rs:7:12
   |
-7 | trait Bar: ~const Foo {}
-  |            ^^^^^^ can't be applied to `Foo`
+7 | trait Bar: [const] Foo {}
+  |            ^^^^^^^ can't be applied to `Foo`
   |
 note: `Foo` can't be used with `[const]` because it isn't `const`
  --> const-super-trait.rs:3:1
@@ -41,14 +41,14 @@ note: `Foo` can't be used with `[const]` because it isn't `const`
 error: `[const]` can only be applied to `const` traits
  --> const-super-trait.rs:9:17
   |
-9 | const fn foo(x: &T) {
-  |                 ^^^^^^ can't be applied to `Bar`
+9 | const fn foo(x: &T) {
+  |                 ^^^^^^^ can't be applied to `Bar`
   |
 note: `Bar` can't be used with `[const]` because it isn't `const`
  --> const-super-trait.rs:7:1
   |
-7 | trait Bar: ~const Foo {}
-  | ^^^^^^^^^^^^^^^^^^^^^
+7 | trait Bar: [const] Foo {}
+  | ^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0015]: cannot call non-const method `::a` in constant functions
   --> const-super-trait.rs:10:7
diff --git a/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-enabled.stderr b/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-enabled.stderr
index 2109f0deb729..30ee5cf6f4b0 100644
--- a/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-enabled.stderr
+++ b/tests/run-make/const-trait-stable-toolchain/const-super-trait-stable-enabled.stderr
@@ -1,14 +1,14 @@
 error: `[const]` is not allowed here
  --> const-super-trait.rs:7:12
   |
-7 | trait Bar: ~const Foo {}
-  |            ^^^^^^
+7 | trait Bar: [const] Foo {}
+  |            ^^^^^^^
   |
 note: this trait is not `const`, so it cannot have `[const]` trait bounds
  --> const-super-trait.rs:7:1
   |
-7 | trait Bar: ~const Foo {}
-  | ^^^^^^^^^^^^^^^^^^^^^^^^
+7 | trait Bar: [const] Foo {}
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0554]: `#![feature]` may not be used on the NIGHTLY release channel
  --> const-super-trait.rs:1:30
@@ -19,8 +19,8 @@ error[E0554]: `#![feature]` may not be used on the NIGHTLY release channel
 error: `[const]` can only be applied to `const` traits
  --> const-super-trait.rs:7:12
   |
-7 | trait Bar: ~const Foo {}
-  |            ^^^^^^ can't be applied to `Foo`
+7 | trait Bar: [const] Foo {}
+  |            ^^^^^^^ can't be applied to `Foo`
   |
 note: `Foo` can't be used with `[const]` because it isn't `const`
  --> const-super-trait.rs:3:1
@@ -31,14 +31,14 @@ note: `Foo` can't be used with `[const]` because it isn't `const`
 error: `[const]` can only be applied to `const` traits
  --> const-super-trait.rs:9:17
   |
-9 | const fn foo(x: &T) {
-  |                 ^^^^^^ can't be applied to `Bar`
+9 | const fn foo(x: &T) {
+  |                 ^^^^^^^ can't be applied to `Bar`
   |
 note: `Bar` can't be used with `[const]` because it isn't `const`
  --> const-super-trait.rs:7:1
   |
-7 | trait Bar: ~const Foo {}
-  | ^^^^^^^^^^^^^^^^^^^^^
+7 | trait Bar: [const] Foo {}
+  | ^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0015]: cannot call non-const method `::a` in constant functions
   --> const-super-trait.rs:10:7
diff --git a/tests/run-make/const-trait-stable-toolchain/const-super-trait.rs b/tests/run-make/const-trait-stable-toolchain/const-super-trait.rs
index b2ee96d79f70..2371dfc0e6d0 100644
--- a/tests/run-make/const-trait-stable-toolchain/const-super-trait.rs
+++ b/tests/run-make/const-trait-stable-toolchain/const-super-trait.rs
@@ -4,9 +4,9 @@ trait Foo {
     fn a(&self);
 }
 
-trait Bar: ~const Foo {}
+trait Bar: [const] Foo {}
 
-const fn foo(x: &T) {
+const fn foo(x: &T) {
     x.a();
 }
 
diff --git a/tests/run-make/crate-loading/multiple-dep-versions-3.rs b/tests/run-make/crate-loading/dep-2-reexport.rs
similarity index 100%
rename from tests/run-make/crate-loading/multiple-dep-versions-3.rs
rename to tests/run-make/crate-loading/dep-2-reexport.rs
diff --git a/tests/run-make/crate-loading/multiple-dep-versions-1.rs b/tests/run-make/crate-loading/dependency-1.rs
similarity index 100%
rename from tests/run-make/crate-loading/multiple-dep-versions-1.rs
rename to tests/run-make/crate-loading/dependency-1.rs
diff --git a/tests/run-make/crate-loading/multiple-dep-versions-2.rs b/tests/run-make/crate-loading/dependency-2.rs
similarity index 100%
rename from tests/run-make/crate-loading/multiple-dep-versions-2.rs
rename to tests/run-make/crate-loading/dependency-2.rs
diff --git a/tests/run-make/crate-loading/rmake.rs b/tests/run-make/crate-loading/rmake.rs
index 6ad456e3e3e5..8f2577861239 100644
--- a/tests/run-make/crate-loading/rmake.rs
+++ b/tests/run-make/crate-loading/rmake.rs
@@ -6,12 +6,9 @@
 use run_make_support::{diff, rust_lib_name, rustc};
 
 fn main() {
-    rustc().input("multiple-dep-versions-1.rs").run();
-    rustc().input("multiple-dep-versions-2.rs").extra_filename("2").metadata("2").run();
-    rustc()
-        .input("multiple-dep-versions-3.rs")
-        .extern_("dependency", rust_lib_name("dependency2"))
-        .run();
+    rustc().input("dependency-1.rs").run();
+    rustc().input("dependency-2.rs").extra_filename("2").metadata("2").run();
+    rustc().input("dep-2-reexport.rs").extern_("dependency", rust_lib_name("dependency2")).run();
 
     let out = rustc()
         .input("multiple-dep-versions.rs")
diff --git a/tests/run-make/cross-lang-lto-clang/rmake.rs b/tests/run-make/cross-lang-lto-clang/rmake.rs
index 3fed6ea20667..f209318abbcc 100644
--- a/tests/run-make/cross-lang-lto-clang/rmake.rs
+++ b/tests/run-make/cross-lang-lto-clang/rmake.rs
@@ -28,7 +28,17 @@ static C_NEVER_INLINED_PATTERN: &'static str = "bl.*";
 static C_NEVER_INLINED_PATTERN: &'static str = "call.*c_never_inlined";
 
 fn main() {
+    test_lto(false);
+    test_lto(true);
+}
+
+fn test_lto(fat_lto: bool) {
+    let lto = if fat_lto { "fat" } else { "thin" };
+    let clang_lto = if fat_lto { "full" } else { "thin" };
+    println!("Running {lto} lto");
+
     rustc()
+        .lto(lto)
         .linker_plugin_lto("on")
         .output(static_lib_name("rustlib-xlto"))
         .opt_level("2")
@@ -36,30 +46,36 @@ fn main() {
         .input("rustlib.rs")
         .run();
     clang()
-        .lto("thin")
+        .lto(clang_lto)
         .use_ld("lld")
         .arg("-lrustlib-xlto")
         .out_exe("cmain")
         .input("cmain.c")
         .arg("-O3")
         .run();
+
+    let dump = llvm_objdump().disassemble().input("cmain").run();
     // Make sure we don't find a call instruction to the function we expect to
     // always be inlined.
-    llvm_objdump()
-        .disassemble()
-        .input("cmain")
-        .run()
-        .assert_stdout_not_contains_regex(RUST_ALWAYS_INLINED_PATTERN);
+    dump.assert_stdout_not_contains_regex(RUST_ALWAYS_INLINED_PATTERN);
     // As a sanity check, make sure we do find a call instruction to a
     // non-inlined function
-    llvm_objdump()
-        .disassemble()
-        .input("cmain")
-        .run()
-        .assert_stdout_contains_regex(RUST_NEVER_INLINED_PATTERN);
-    clang().input("clib.c").lto("thin").arg("-c").out_exe("clib.o").arg("-O2").run();
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+    dump.assert_stdout_contains_regex(RUST_NEVER_INLINED_PATTERN);
+    #[cfg(any(target_arch = "aarch64", target_arch = "arm"))]
+    {
+        if fat_lto {
+            // fat lto inlines this anyway
+            dump.assert_stdout_not_contains_regex(RUST_NEVER_INLINED_PATTERN);
+        } else {
+            dump.assert_stdout_contains_regex(RUST_NEVER_INLINED_PATTERN);
+        }
+    }
+
+    clang().input("clib.c").lto(clang_lto).arg("-c").out_exe("clib.o").arg("-O2").run();
     llvm_ar().obj_to_ar().output_input(static_lib_name("xyz"), "clib.o").run();
     rustc()
+        .lto(lto)
         .linker_plugin_lto("on")
         .opt_level("2")
         .linker(&env_var("CLANG"))
@@ -67,14 +83,13 @@ fn main() {
         .input("main.rs")
         .output("rsmain")
         .run();
-    llvm_objdump()
-        .disassemble()
-        .input("rsmain")
-        .run()
-        .assert_stdout_not_contains_regex(C_ALWAYS_INLINED_PATTERN);
-    llvm_objdump()
-        .disassemble()
-        .input("rsmain")
-        .run()
-        .assert_stdout_contains_regex(C_NEVER_INLINED_PATTERN);
+
+    let dump = llvm_objdump().disassemble().input("rsmain").run();
+    dump.assert_stdout_not_contains_regex(C_ALWAYS_INLINED_PATTERN);
+    if fat_lto {
+        // fat lto inlines this anyway
+        dump.assert_stdout_not_contains_regex(C_NEVER_INLINED_PATTERN);
+    } else {
+        dump.assert_stdout_contains_regex(C_NEVER_INLINED_PATTERN);
+    }
 }
diff --git a/tests/run-make/doctests-compilation-time-info/rmake.rs b/tests/run-make/doctests-compilation-time-info/rmake.rs
new file mode 100644
index 000000000000..2bcf664923f2
--- /dev/null
+++ b/tests/run-make/doctests-compilation-time-info/rmake.rs
@@ -0,0 +1,119 @@
+//@ ignore-cross-compile (needs to run doctests)
+
+use run_make_support::rfs::write;
+use run_make_support::{cwd, rustdoc};
+
+fn assert_presence_of_compilation_time_report(
+    content: &str,
+    success: bool,
+    should_contain_compile_time: bool,
+) {
+    let mut cmd = rustdoc();
+    let file = cwd().join("foo.rs");
+
+    write(&file, content);
+    cmd.input(&file).arg("--test").edition("2024").env("RUST_BACKTRACE", "0");
+    let output = if success { cmd.run() } else { cmd.run_fail() };
+
+    assert_eq!(
+        output
+            .stdout_utf8()
+            .split("all doctests ran in ")
+            .last()
+            .is_some_and(|s| s.contains("; merged doctests compilation took")),
+        should_contain_compile_time,
+    );
+}
+
+fn main() {
+    // Checking with only successful merged doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! let x = 12;
+//! ```",
+        true,
+        true,
+    );
+    // Checking with only failing merged doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! panic!();
+//! ```",
+        false,
+        true,
+    );
+    // Checking with mix of successful doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! let x = 12;
+//! ```
+//!
+//! ```compile_fail
+//! let x
+//! ```",
+        true,
+        true,
+    );
+    // Checking with mix of failing doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! panic!();
+//! ```
+//!
+//! ```compile_fail
+//! let x
+//! ```",
+        false,
+        true,
+    );
+    // Checking with mix of failing doctests (v2).
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! let x = 12;
+//! ```
+//!
+//! ```compile_fail
+//! let x = 12;
+//! ```",
+        false,
+        true,
+    );
+    // Checking with mix of failing doctests (v3).
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```
+//! panic!();
+//! ```
+//!
+//! ```compile_fail
+//! let x = 12;
+//! ```",
+        false,
+        true,
+    );
+    // Checking with successful non-merged doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```compile_fail
+//! let x
+//! ```",
+        true,
+        // If there is no merged doctests, then we should not display compilation time.
+        false,
+    );
+    // Checking with failing non-merged doctests.
+    assert_presence_of_compilation_time_report(
+        "\
+//! ```compile_fail
+//! let x = 12;
+//! ```",
+        false,
+        // If there is no merged doctests, then we should not display compilation time.
+        false,
+    );
+}
diff --git a/tests/run-make/doctests-merge/doctest-2024.stdout b/tests/run-make/doctests-merge/doctest-2024.stdout
index 7da08d68faae..a7e139bbd23d 100644
--- a/tests/run-make/doctests-merge/doctest-2024.stdout
+++ b/tests/run-make/doctests-merge/doctest-2024.stdout
@@ -5,3 +5,4 @@ test doctest.rs - init (line 8) ... ok
 
 test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME
 
+all doctests ran in $TIME; merged doctests compilation took $TIME
diff --git a/tests/run-make/doctests-merge/rmake.rs b/tests/run-make/doctests-merge/rmake.rs
index 7893d4988ebb..f2a1e8e13ddf 100644
--- a/tests/run-make/doctests-merge/rmake.rs
+++ b/tests/run-make/doctests-merge/rmake.rs
@@ -20,6 +20,8 @@ fn test_and_compare(input_file: &str, stdout_file: &str, edition: &str, dep: &Pa
         .expected_file(stdout_file)
         .actual_text("output", output.stdout_utf8())
         .normalize(r#"finished in \d+\.\d+s"#, "finished in $$TIME")
+        .normalize(r#"ran in \d+\.\d+s"#, "ran in $$TIME")
+        .normalize(r#"compilation took \d+\.\d+s"#, "compilation took $$TIME")
         .run();
 }
 
diff --git a/tests/run-make/duplicate-dependency/foo-v1.rs b/tests/run-make/duplicate-dependency/foo-v1.rs
new file mode 100644
index 000000000000..4a835673a596
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/foo-v1.rs
@@ -0,0 +1 @@
+pub struct Foo;
diff --git a/tests/run-make/duplicate-dependency/foo-v2.rs b/tests/run-make/duplicate-dependency/foo-v2.rs
new file mode 100644
index 000000000000..4a835673a596
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/foo-v2.rs
@@ -0,0 +1 @@
+pub struct Foo;
diff --git a/tests/run-make/duplicate-dependency/main.rs b/tests/run-make/duplicate-dependency/main.rs
new file mode 100644
index 000000000000..b22d9581c9a4
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/main.rs
@@ -0,0 +1,15 @@
+struct Bar;
+
+impl From for foo::Foo {
+    fn from(_: Bar) -> Self {
+        foo::Foo
+    }
+}
+
+fn main() {
+    // The user might wrongly expect this to work since From for Foo
+    // implies Into for Bar. What the user missed is that different
+    // versions of Foo exist in the dependency graph, and the impl is for the
+    // wrong version.
+    re_export_foo::into_foo(Bar);
+}
diff --git a/tests/run-make/duplicate-dependency/main.stderr b/tests/run-make/duplicate-dependency/main.stderr
new file mode 100644
index 000000000000..36d54988788f
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/main.stderr
@@ -0,0 +1,24 @@
+error[E0277]: the trait bound `re_export_foo::foo::Foo: From` is not satisfied
+  --> main.rs:14:29
+   |
+LL |     re_export_foo::into_foo(Bar);
+   |     ----------------------- ^^^ the trait `From` is not implemented for `re_export_foo::foo::Foo`
+   |     |
+   |     required by a bound introduced by this call
+   |
+help: item with same name found
+  --> $DIR/foo-v1.rs:1:1
+   |
+LL | pub struct Foo;
+   | ^^^^^^^^^^^^^^
+   = note: perhaps two different versions of crate `foo` are being used?
+   = note: required for `Bar` to implement `Into`
+note: required by a bound in `into_foo`
+  --> $DIR/re-export-foo.rs:3:25
+   |
+LL | pub fn into_foo(_: impl Into) {}
+   |                         ^^^^^^^^^^^^^^ required by this bound in `into_foo`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/run-make/duplicate-dependency/re-export-foo.rs b/tests/run-make/duplicate-dependency/re-export-foo.rs
new file mode 100644
index 000000000000..b346cfcee9b1
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/re-export-foo.rs
@@ -0,0 +1,3 @@
+pub use foo;
+
+pub fn into_foo(_: impl Into) {}
diff --git a/tests/run-make/duplicate-dependency/rmake.rs b/tests/run-make/duplicate-dependency/rmake.rs
new file mode 100644
index 000000000000..762d97e4311f
--- /dev/null
+++ b/tests/run-make/duplicate-dependency/rmake.rs
@@ -0,0 +1,45 @@
+//@ needs-target-std
+
+use run_make_support::{Rustc, cwd, diff, rust_lib_name, rustc};
+
+fn rustc_with_common_args() -> Rustc {
+    let mut rustc = rustc();
+    rustc.remap_path_prefix(cwd(), "$DIR");
+    rustc.edition("2018"); // Don't require `extern crate`
+    rustc
+}
+
+fn main() {
+    rustc_with_common_args()
+        .input("foo-v1.rs")
+        .crate_type("rlib")
+        .crate_name("foo")
+        .extra_filename("-v1")
+        .metadata("-v1")
+        .run();
+
+    rustc_with_common_args()
+        .input("foo-v2.rs")
+        .crate_type("rlib")
+        .crate_name("foo")
+        .extra_filename("-v2")
+        .metadata("-v2")
+        .run();
+
+    rustc_with_common_args()
+        .input("re-export-foo.rs")
+        .crate_type("rlib")
+        .extern_("foo", rust_lib_name("foo-v2"))
+        .run();
+
+    let stderr = rustc_with_common_args()
+        .input("main.rs")
+        .extern_("foo", rust_lib_name("foo-v1"))
+        .extern_("re_export_foo", rust_lib_name("re_export_foo"))
+        .library_search_path(cwd())
+        .ui_testing()
+        .run_fail()
+        .stderr_utf8();
+
+    diff().expected_file("main.stderr").normalize(r"\\", "/").actual_text("(rustc)", &stderr).run();
+}
diff --git a/tests/run-make/emit-shared-files/rmake.rs b/tests/run-make/emit-shared-files/rmake.rs
index f88fe69aa9cd..bd868d4fd19e 100644
--- a/tests/run-make/emit-shared-files/rmake.rs
+++ b/tests/run-make/emit-shared-files/rmake.rs
@@ -19,7 +19,7 @@ fn main() {
         .args(&["--extend-css", "z.css"])
         .input("x.rs")
         .run();
-    assert!(path("invocation-only/search-index-xxx.js").exists());
+    assert!(path("invocation-only/search.index/root-xxx.js").exists());
     assert!(path("invocation-only/crates-xxx.js").exists());
     assert!(path("invocation-only/settings.html").exists());
     assert!(path("invocation-only/x/all.html").exists());
@@ -68,7 +68,7 @@ fn main() {
 
     rustdoc()
         .arg("-Zunstable-options")
-        .arg("--emit=toolchain-shared-resources,unversioned-shared-resources")
+        .arg("--emit=toolchain-shared-resources")
         .out_dir("all-shared")
         .arg("--resource-suffix=-xxx")
         .args(&["--extend-css", "z.css"])
diff --git a/tests/run-make/fat-then-thin-lto/lib.rs b/tests/run-make/fat-then-thin-lto/lib.rs
new file mode 100644
index 000000000000..c675dcb6e8a8
--- /dev/null
+++ b/tests/run-make/fat-then-thin-lto/lib.rs
@@ -0,0 +1,13 @@
+#![allow(internal_features)]
+#![feature(no_core, lang_items)]
+#![no_core]
+#![crate_type = "rlib"]
+
+#[lang = "pointee_sized"]
+trait PointeeSized {}
+#[lang = "meta_sized"]
+trait MetaSized: PointeeSized {}
+#[lang = "sized"]
+trait Sized: MetaSized {}
+
+pub fn foo() {}
diff --git a/tests/run-make/fat-then-thin-lto/main.rs b/tests/run-make/fat-then-thin-lto/main.rs
new file mode 100644
index 000000000000..a3f2e18158bc
--- /dev/null
+++ b/tests/run-make/fat-then-thin-lto/main.rs
@@ -0,0 +1,11 @@
+#![allow(internal_features)]
+#![feature(no_core, lang_items)]
+#![no_core]
+#![crate_type = "cdylib"]
+
+extern crate lib;
+
+#[unsafe(no_mangle)]
+pub fn bar() {
+    lib::foo();
+}
diff --git a/tests/run-make/fat-then-thin-lto/rmake.rs b/tests/run-make/fat-then-thin-lto/rmake.rs
new file mode 100644
index 000000000000..ef4f26689d4e
--- /dev/null
+++ b/tests/run-make/fat-then-thin-lto/rmake.rs
@@ -0,0 +1,25 @@
+// Compile a library with lto=fat, then compile a binary with lto=thin
+// and check that lto is applied with the library.
+// The goal is to mimic the standard library being build with lto=fat
+// and allowing users to build with lto=thin.
+
+//@ only-x86_64-unknown-linux-gnu
+
+use run_make_support::{dynamic_lib_name, llvm_objdump, rustc};
+
+fn main() {
+    rustc().input("lib.rs").opt_level("3").lto("fat").run();
+    rustc().input("main.rs").panic("abort").opt_level("3").lto("thin").run();
+
+    llvm_objdump()
+        .input(dynamic_lib_name("main"))
+        .arg("--disassemble-symbols=bar")
+        .run()
+        // The called function should be inlined.
+        // Check that we have a ret (to detect tail
+        // calls with a jmp) and no call.
+        .assert_stdout_contains("bar")
+        .assert_stdout_contains("ret")
+        .assert_stdout_not_contains("foo")
+        .assert_stdout_not_contains("call");
+}
diff --git a/tests/run-make/libtest-json/output-default.json b/tests/run-make/libtest-json/output-default.json
index a6a8a9f3b477..5371715d17c7 100644
--- a/tests/run-make/libtest-json/output-default.json
+++ b/tests/run-make/libtest-json/output-default.json
@@ -2,7 +2,7 @@
 { "type": "test", "event": "started", "name": "a" }
 { "type": "test", "name": "a", "event": "ok" }
 { "type": "test", "event": "started", "name": "b" }
-{ "type": "test", "name": "b", "event": "failed", "stdout": "\nthread 'b' panicked at f.rs:9:5:\nassertion failed: false\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n" }
+{ "type": "test", "name": "b", "event": "failed", "stdout": "\nthread 'b' ($TID) panicked at f.rs:9:5:\nassertion failed: false\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n" }
 { "type": "test", "event": "started", "name": "c" }
 { "type": "test", "name": "c", "event": "ok" }
 { "type": "test", "event": "started", "name": "d" }
diff --git a/tests/run-make/libtest-json/output-stdout-success.json b/tests/run-make/libtest-json/output-stdout-success.json
index a6c36e746b38..5caadcf56cf3 100644
--- a/tests/run-make/libtest-json/output-stdout-success.json
+++ b/tests/run-make/libtest-json/output-stdout-success.json
@@ -2,9 +2,9 @@
 { "type": "test", "event": "started", "name": "a" }
 { "type": "test", "name": "a", "event": "ok", "stdout": "print from successful test\n" }
 { "type": "test", "event": "started", "name": "b" }
-{ "type": "test", "name": "b", "event": "failed", "stdout": "\nthread 'b' panicked at f.rs:9:5:\nassertion failed: false\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n" }
+{ "type": "test", "name": "b", "event": "failed", "stdout": "\nthread 'b' ($TID) panicked at f.rs:9:5:\nassertion failed: false\nnote: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n" }
 { "type": "test", "event": "started", "name": "c" }
-{ "type": "test", "name": "c", "event": "ok", "stdout": "\nthread 'c' panicked at f.rs:15:5:\nassertion failed: false\n" }
+{ "type": "test", "name": "c", "event": "ok", "stdout": "\nthread 'c' ($TID) panicked at f.rs:15:5:\nassertion failed: false\n" }
 { "type": "test", "event": "started", "name": "d" }
 { "type": "test", "name": "d", "event": "ignored", "message": "msg" }
 { "type": "suite", "event": "failed", "passed": 2, "failed": 1, "ignored": 1, "measured": 0, "filtered_out": 0, "exec_time": "$EXEC_TIME" }
diff --git a/tests/run-make/libtest-json/rmake.rs b/tests/run-make/libtest-json/rmake.rs
index c31f4a79b641..034ba6246f1d 100644
--- a/tests/run-make/libtest-json/rmake.rs
+++ b/tests/run-make/libtest-json/rmake.rs
@@ -38,5 +38,6 @@ fn run_tests(extra_args: &[&str], expected_file: &str) {
         .expected_file(expected_file)
         .actual_text("stdout", test_stdout)
         .normalize(r#"(?"exec_time": )[0-9.]+"#, r#"${prefix}"$$EXEC_TIME""#)
+        .normalize(r"thread '(?P.*?)' \(\d+\) panicked", "thread '$name' ($$TID) panicked")
         .run();
 }
diff --git a/tests/run-make/libtest-junit/output-default.xml b/tests/run-make/libtest-junit/output-default.xml
index aa1b8c855aad..2467d8d940a4 100644
--- a/tests/run-make/libtest-junit/output-default.xml
+++ b/tests/run-make/libtest-junit/output-default.xml
@@ -1 +1 @@
-





+





diff --git a/tests/run-make/libtest-junit/output-stdout-success.xml b/tests/run-make/libtest-junit/output-stdout-success.xml
index 2592ec7efb16..6bf1d7008a44 100644
--- a/tests/run-make/libtest-junit/output-stdout-success.xml
+++ b/tests/run-make/libtest-junit/output-stdout-success.xml
@@ -1 +1 @@
-









+









diff --git a/tests/run-make/libtest-junit/rmake.rs b/tests/run-make/libtest-junit/rmake.rs
index 5917660b6c7f..6961be215138 100644
--- a/tests/run-make/libtest-junit/rmake.rs
+++ b/tests/run-make/libtest-junit/rmake.rs
@@ -27,5 +27,6 @@ fn run_tests(extra_args: &[&str], expected_file: &str) {
         .expected_file(expected_file)
         .actual_text("stdout", test_stdout)
         .normalize(r#"\btime="[0-9.]+""#, r#"time="$$TIME""#)
+        .normalize(r"thread '(?P.*?)' \(\d+\) panicked", "thread '$name' ($$TID) panicked")
         .run();
 }
diff --git a/tests/run-make/link-under-xcode/foo.rs b/tests/run-make/link-under-xcode/foo.rs
new file mode 100644
index 000000000000..f328e4d9d04c
--- /dev/null
+++ b/tests/run-make/link-under-xcode/foo.rs
@@ -0,0 +1 @@
+fn main() {}
diff --git a/tests/run-make/link-under-xcode/rmake.rs b/tests/run-make/link-under-xcode/rmake.rs
new file mode 100644
index 000000000000..c9394feb000a
--- /dev/null
+++ b/tests/run-make/link-under-xcode/rmake.rs
@@ -0,0 +1,32 @@
+//! Test that linking works under an environment similar to what Xcode sets up.
+//!
+//! Regression test for https://github.com/rust-lang/rust/issues/80817.
+
+//@ only-apple
+
+use run_make_support::{cmd, rustc, target};
+
+fn main() {
+    // Fetch toolchain `/usr/bin` directory. Usually:
+    // /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin
+    let clang_bin = cmd("xcrun").arg("--find").arg("clang").run().stdout_utf8();
+    let toolchain_bin = clang_bin.trim().strip_suffix("/clang").unwrap();
+
+    // Put toolchain directory at the front of PATH.
+    let path = format!("{}:{}", toolchain_bin, std::env::var("PATH").unwrap());
+
+    // Check that compiling and linking still works.
+    //
+    // Removing `SDKROOT` is necessary for the test to excercise what we want, since bootstrap runs
+    // under `/usr/bin/python3`, which will set SDKROOT for us.
+    rustc().target(target()).env_remove("SDKROOT").env("PATH", &path).input("foo.rs").run();
+
+    // Also check linking directly with the system linker.
+    rustc()
+        .target(target())
+        .env_remove("SDKROOT")
+        .env("PATH", &path)
+        .input("foo.rs")
+        .arg("-Clinker-flavor=ld")
+        .run();
+}
diff --git a/tests/run-make/linker-plugin-lto-fat/ir.ll b/tests/run-make/linker-plugin-lto-fat/ir.ll
new file mode 100644
index 000000000000..fa3dbdd4e088
--- /dev/null
+++ b/tests/run-make/linker-plugin-lto-fat/ir.ll
@@ -0,0 +1,6 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @ir_callee() {
+  ret void
+}
diff --git a/tests/run-make/linker-plugin-lto-fat/main.rs b/tests/run-make/linker-plugin-lto-fat/main.rs
new file mode 100644
index 000000000000..ad2a90bc8019
--- /dev/null
+++ b/tests/run-make/linker-plugin-lto-fat/main.rs
@@ -0,0 +1,22 @@
+#![allow(internal_features)]
+#![feature(no_core, lang_items)]
+#![no_core]
+#![crate_type = "cdylib"]
+
+#[lang = "pointee_sized"]
+trait PointeeSized {}
+#[lang = "meta_sized"]
+trait MetaSized: PointeeSized {}
+#[lang = "sized"]
+trait Sized: MetaSized {}
+
+extern "C" {
+    fn ir_callee();
+}
+
+#[no_mangle]
+extern "C" fn rs_foo() {
+    unsafe {
+        ir_callee();
+    }
+}
diff --git a/tests/run-make/linker-plugin-lto-fat/rmake.rs b/tests/run-make/linker-plugin-lto-fat/rmake.rs
new file mode 100644
index 000000000000..ff5b647a5941
--- /dev/null
+++ b/tests/run-make/linker-plugin-lto-fat/rmake.rs
@@ -0,0 +1,33 @@
+// Check that -C lto=fat with -C linker-plugin-lto actually works and can inline functions.
+// A library is created from LLVM IR, defining a single function. Then a dylib is compiled,
+// linking to the library and calling the function from the library.
+// The function from the library should end up inlined and disappear from the output.
+
+//@ only-x86_64-unknown-linux-gnu
+//@ needs-rust-lld
+
+use run_make_support::{dynamic_lib_name, llvm_as, llvm_objdump, rustc};
+
+fn main() {
+    llvm_as().input("ir.ll").run();
+    rustc()
+        .input("main.rs")
+        .opt_level("3")
+        .lto("fat")
+        .linker_plugin_lto("on")
+        .link_arg("ir.bc")
+        .arg("-Zunstable-options")
+        .arg("-Clinker-features=+lld")
+        .run();
+
+    llvm_objdump()
+        .input(dynamic_lib_name("main"))
+        .arg("--disassemble-symbols=rs_foo")
+        .run()
+        // The called function should be inlined.
+        // Check that we have a ret (to detect tail
+        // calls with a jmp) and no call.
+        .assert_stdout_contains("foo")
+        .assert_stdout_contains("ret")
+        .assert_stdout_not_contains("call");
+}
diff --git a/tests/run-make/linker-warning/rmake.rs b/tests/run-make/linker-warning/rmake.rs
index 9ea706af5035..a31b08d6c695 100644
--- a/tests/run-make/linker-warning/rmake.rs
+++ b/tests/run-make/linker-warning/rmake.rs
@@ -61,14 +61,13 @@ fn main() {
         diff()
             .expected_file("short-error.txt")
             .actual_text("(linker error)", out.stderr())
-            .normalize(r#"/rustc[^/_-]*/"#, "/rustc/")
-            .normalize("libpanic_abort", "libpanic_unwind")
             .normalize(
                 regex::escape(
                     run_make_support::build_root().canonicalize().unwrap().to_str().unwrap(),
                 ),
                 "/build-root",
             )
+            .normalize("libpanic_abort", "libpanic_unwind")
             .normalize(r#""[^"]*\/symbols.o""#, "\"/symbols.o\"")
             .normalize(r#""[^"]*\/raw-dylibs""#, "\"/raw-dylibs\"")
             .run();
diff --git a/tests/run-make/linker-warning/short-error.txt b/tests/run-make/linker-warning/short-error.txt
index 5b7c040bc50e..e5861b732c5f 100644
--- a/tests/run-make/linker-warning/short-error.txt
+++ b/tests/run-make/linker-warning/short-error.txt
@@ -1,6 +1,6 @@
 error: linking with `./fake-linker` failed: exit status: 1
   |
-  = note:  "./fake-linker" "-m64" "/symbols.o" "<2 object files omitted>" "-Wl,--as-needed" "-Wl,-Bstatic" "/build-root/test/run-make/linker-warning/rmake_out/{libfoo,libbar}.rlib" "/lib/rustlib/x86_64-unknown-linux-gnu/lib/{libstd-*,libpanic_unwind-*,libobject-*,libmemchr-*,libaddr2line-*,libgimli-*,librustc_demangle-*,libstd_detect-*,libhashbrown-*,librustc_std_workspace_alloc-*,libminiz_oxide-*,libadler2-*,libunwind-*,libcfg_if-*,liblibc-*,librustc_std_workspace_core-*,liballoc-*,libcore-*,libcompiler_builtins-*}.rlib" "-Wl,-Bdynamic" "-lgcc_s" "-lutil" "-lrt" "-lpthread" "-lm" "-ldl" "-lc" "-L" "/raw-dylibs" "-Wl,--eh-frame-hdr" "-Wl,-z,noexecstack" "-L" "/build-root/test/run-make/linker-warning/rmake_out" "-L" "/lib/rustlib/x86_64-unknown-linux-gnu/lib" "-o" "main" "-Wl,--gc-sections" "-pie" "-Wl,-z,relro,-z,now" "-nodefaultlibs" "run_make_error"
+  = note:  "./fake-linker" "-m64" "/symbols.o" "<2 object files omitted>" "-Wl,--as-needed" "-Wl,-Bstatic" "/build-root/test/run-make/linker-warning/rmake_out/{libfoo,libbar}.rlib" "/lib/rustlib/x86_64-unknown-linux-gnu/lib/{libstd-*,libpanic_unwind-*,libobject-*,libmemchr-*,libaddr2line-*,libgimli-*,libcfg_if-*,librustc_demangle-*,libstd_detect-*,libhashbrown-*,librustc_std_workspace_alloc-*,libminiz_oxide-*,libadler2-*,libunwind-*,liblibc-*,librustc_std_workspace_core-*,liballoc-*,libcore-*,libcompiler_builtins-*}.rlib" "-Wl,-Bdynamic" "-lgcc_s" "-lutil" "-lrt" "-lpthread" "-lm" "-ldl" "-lc" "-L" "/raw-dylibs" "-Wl,--eh-frame-hdr" "-Wl,-z,noexecstack" "-L" "/build-root/test/run-make/linker-warning/rmake_out" "-L" "/lib/rustlib/x86_64-unknown-linux-gnu/lib" "-o" "main" "-Wl,--gc-sections" "-pie" "-Wl,-z,relro,-z,now" "-nodefaultlibs" "run_make_error"
   = note: some arguments are omitted. use `--verbose` to show all linker arguments
   = note: error: baz
           
diff --git a/tests/run-make/lto-long-filenames/main.rs b/tests/run-make/lto-long-filenames/main.rs
new file mode 100644
index 000000000000..daedff5c05fc
--- /dev/null
+++ b/tests/run-make/lto-long-filenames/main.rs
@@ -0,0 +1,7 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+extern crate generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name;
+
+fn main() {}
diff --git a/tests/run-make/lto-long-filenames/rmake.rs b/tests/run-make/lto-long-filenames/rmake.rs
new file mode 100644
index 000000000000..98ce54e5efc8
--- /dev/null
+++ b/tests/run-make/lto-long-filenames/rmake.rs
@@ -0,0 +1,30 @@
+// This file has very long lines, but there is no way to avoid it as we are testing
+// long crate names. so:
+// ignore-tidy-linelength
+
+// A variant of the smoke test to check that link time optimization
+// (LTO) is accepted by the compiler, and that
+// passing its various flags still results in successful compilation, even for very long crate names.
+// See https://github.com/rust-lang/rust/issues/49914
+
+//@ ignore-cross-compile
+
+use run_make_support::{rfs, rustc};
+
+// This test make sure we don't get such following error:
+// error: could not write output to generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.9384edb61bfd127c-cgu.0.rcgu.o: File name too long
+// as reported in issue #49914
+fn main() {
+    let lto_flags = ["-Clto", "-Clto=yes", "-Clto=off", "-Clto=thin", "-Clto=fat"];
+    let aux_file = "generated_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_large_crate_name.rs";
+    // The auxiliary file is used to test long crate names.
+    // The file name is intentionally long to test the handling of long filenames.
+    // We don't commit it to avoid issues with Windows paths which have known limitations for the full path length.
+    // Posix usually only have a limit for the length of the file name.
+    rfs::write(aux_file, "#![crate_type = \"rlib\"]\n");
+
+    for flag in lto_flags {
+        rustc().input(aux_file).arg(flag).run();
+        rustc().input("main.rs").arg(flag).run();
+    }
+}
diff --git a/tests/run-make/lto-long-filenames_cn/rmake.rs b/tests/run-make/lto-long-filenames_cn/rmake.rs
new file mode 100644
index 000000000000..fc25a2ac166b
--- /dev/null
+++ b/tests/run-make/lto-long-filenames_cn/rmake.rs
@@ -0,0 +1,25 @@
+//@ ignore-cross-compile
+// gnu ld is confused with intermediate files having multibytes characters in their names:
+// = note: ld.exe: cannot find f0d5ff18d6510ebc-???_???_??????????_?_?????_?_???????.d50c2 \
+// 4c0c4ea93cc-cgu.0.rcgu.o: Invalid argument
+// as this is not something rustc can fix by itself,
+// we just skip the test on windows-gnu for now. Hence:
+//@ ignore-windows-gnu
+
+use run_make_support::{rfs, rustc};
+
+// This test make sure we don't crash when lto creates output files with long names.
+// cn characters can be multi-byte and thus trigger the long filename reduction code more easily.
+// we need to make sure that the code is properly generating names at char boundaries.
+// as reported in issue #147975
+fn main() {
+    let lto_flags = ["-Clto", "-Clto=yes", "-Clto=off", "-Clto=thin", "-Clto=fat"];
+    for prefix_len in 0..4 {
+        let prefix: String = std::iter::repeat("_").take(prefix_len).collect();
+        let main_file = format!("{}ⵅⴻⵎⵎⴻⵎ_ⴷⵉⵎⴰ_ⵖⴻⴼ_ⵢⵉⵙⴻⴽⴽⵉⵍⴻⵏ_ⵏ_ⵡⴰⵟⴰⵙ_ⵏ_ⵢⵉⴱⵢⵜⴻⵏ.rs", prefix);
+        rfs::write(&main_file, "fn main() {}\n");
+        for flag in lto_flags {
+            rustc().input(&main_file).arg(flag).run();
+        }
+    }
+}
diff --git a/tests/run-make/msvc-wholearchive/rmake.rs b/tests/run-make/msvc-wholearchive/rmake.rs
index 98586fd8cc8b..aec9391a420d 100644
--- a/tests/run-make/msvc-wholearchive/rmake.rs
+++ b/tests/run-make/msvc-wholearchive/rmake.rs
@@ -7,7 +7,7 @@
 
 use std::path::PathBuf;
 
-use run_make_support::{cc, cmd, env_var, extra_c_flags, rustc};
+use run_make_support::{cc, cmd, env_var, extra_linker_flags, rustc};
 
 fn main() {
     // Build the staticlib
@@ -31,7 +31,7 @@ fn main() {
     // Otherwise the actual test failure may be caused by something else.
     cmd(&linker)
         .args(["c.obj", "./static.lib", "-dll", "-def:dll.def", "-out:dll.dll"])
-        .args(extra_c_flags())
+        .args(extra_linker_flags())
         .run();
 
     // FIXME(@ChrisDenton): this doesn't currently work with llvm's lld-link for other reasons.
@@ -46,7 +46,7 @@ fn main() {
                 "-def:dll.def",
                 "-out:dll_whole_archive.dll",
             ])
-            .args(extra_c_flags())
+            .args(extra_linker_flags())
             .run();
     }
 }
diff --git a/tests/run-make/multiline-args-value/cfg-frontmatter.stderr b/tests/run-make/multiline-args-value/cfg-frontmatter.stderr
new file mode 100644
index 000000000000..9b06f84be488
--- /dev/null
+++ b/tests/run-make/multiline-args-value/cfg-frontmatter.stderr
@@ -0,0 +1,4 @@
+error: invalid `--cfg` argument: `---
+       ---
+       key` (expected `key` or `key="value"`)
+
diff --git a/tests/run-make/multiline-args-value/cfg-shebang.stderr b/tests/run-make/multiline-args-value/cfg-shebang.stderr
new file mode 100644
index 000000000000..09baf3b51239
--- /dev/null
+++ b/tests/run-make/multiline-args-value/cfg-shebang.stderr
@@ -0,0 +1,3 @@
+error: invalid `--cfg` argument: `#!/usr/bin/shebang
+       key` (expected `key` or `key="value"`)
+
diff --git a/tests/run-make/multiline-args-value/check-cfg-frontmatter.stderr b/tests/run-make/multiline-args-value/check-cfg-frontmatter.stderr
new file mode 100644
index 000000000000..4a499cc0a1e2
--- /dev/null
+++ b/tests/run-make/multiline-args-value/check-cfg-frontmatter.stderr
@@ -0,0 +1,7 @@
+error: invalid `--check-cfg` argument: `---
+       ---
+       cfg(key)`
+  |
+  = note: expected `cfg(name, values("value1", "value2", ... "valueN"))`
+  = note: visit  for more details
+
diff --git a/tests/run-make/multiline-args-value/check-cfg-shebang.stderr b/tests/run-make/multiline-args-value/check-cfg-shebang.stderr
new file mode 100644
index 000000000000..5bf18dc2b14c
--- /dev/null
+++ b/tests/run-make/multiline-args-value/check-cfg-shebang.stderr
@@ -0,0 +1,6 @@
+error: invalid `--check-cfg` argument: `#!/usr/bin/shebang
+       cfg(key)`
+  |
+  = note: expected `cfg(name, values("value1", "value2", ... "valueN"))`
+  = note: visit  for more details
+
diff --git a/tests/run-make/multiline-args-value/rmake.rs b/tests/run-make/multiline-args-value/rmake.rs
new file mode 100644
index 000000000000..3964cbbc1e60
--- /dev/null
+++ b/tests/run-make/multiline-args-value/rmake.rs
@@ -0,0 +1,51 @@
+use run_make_support::{cwd, diff, rustc};
+
+fn test_and_compare(test_name: &str, flag: &str, val: &str) {
+    let mut cmd = rustc();
+
+    let output = cmd.input("").arg("--crate-type=lib").arg(flag).arg(val).run_fail();
+
+    assert_eq!(output.stdout_utf8(), "");
+    diff()
+        .expected_file(format!("{test_name}.stderr"))
+        .actual_text("stderr", output.stderr_utf8())
+        .run();
+}
+
+fn main() {
+    // Verify that frontmatter isn't allowed in `--cfg` arguments.
+    // https://github.com/rust-lang/rust/issues/146130
+    test_and_compare(
+        "cfg-frontmatter",
+        "--cfg",
+        r#"---
+---
+key"#,
+    );
+
+    // Verify that frontmatter isn't allowed in `--check-cfg` arguments.
+    // https://github.com/rust-lang/rust/issues/146130
+    test_and_compare(
+        "check-cfg-frontmatter",
+        "--check-cfg",
+        r#"---
+---
+cfg(key)"#,
+    );
+
+    // Verify that shebang isn't allowed in `--cfg` arguments.
+    test_and_compare(
+        "cfg-shebang",
+        "--cfg",
+        r#"#!/usr/bin/shebang
+key"#,
+    );
+
+    // Verify that shebang isn't allowed in `--check-cfg` arguments.
+    test_and_compare(
+        "check-cfg-shebang",
+        "--check-cfg",
+        r#"#!/usr/bin/shebang
+cfg(key)"#,
+    );
+}
diff --git a/tests/run-make/musl-default-linking/rmake.rs b/tests/run-make/musl-default-linking/rmake.rs
index 7bb54e2739c9..e9d09e359c68 100644
--- a/tests/run-make/musl-default-linking/rmake.rs
+++ b/tests/run-make/musl-default-linking/rmake.rs
@@ -4,7 +4,7 @@ use run_make_support::{rustc, serde_json};
 // Per https://github.com/rust-lang/compiler-team/issues/422,
 // we should be trying to move these targets to dynamically link
 // musl libc by default.
-//@ needs-llvm-components: aarch64 arm mips powerpc riscv systemz x86
+//@ needs-llvm-components: aarch64 arm powerpc x86
 static LEGACY_STATIC_LINKING_TARGETS: &[&'static str] = &[
     "aarch64-unknown-linux-musl",
     "arm-unknown-linux-musleabi",
@@ -14,16 +14,7 @@ static LEGACY_STATIC_LINKING_TARGETS: &[&'static str] = &[
     "armv7-unknown-linux-musleabihf",
     "i586-unknown-linux-musl",
     "i686-unknown-linux-musl",
-    "mips64-unknown-linux-musl",
-    "mips64-unknown-linux-muslabi64",
-    "mips64el-unknown-linux-muslabi64",
-    "powerpc-unknown-linux-musl",
-    "powerpc-unknown-linux-muslspe",
-    "powerpc64-unknown-linux-musl",
     "powerpc64le-unknown-linux-musl",
-    "riscv32gc-unknown-linux-musl",
-    "s390x-unknown-linux-musl",
-    "thumbv7neon-unknown-linux-musleabihf",
     "x86_64-unknown-linux-musl",
 ];
 
diff --git a/tests/run-make/panic-abort-eh_frame/rmake.rs b/tests/run-make/panic-abort-eh_frame/rmake.rs
index 23d95dc57749..2eccde627955 100644
--- a/tests/run-make/panic-abort-eh_frame/rmake.rs
+++ b/tests/run-make/panic-abort-eh_frame/rmake.rs
@@ -1,9 +1,11 @@
 // An `.eh_frame` section in an object file is a symptom of an UnwindAction::Terminate
 // being inserted, useful for determining whether or not unwinding is necessary.
-// This is useless when panics would NEVER unwind due to -C panic=abort. This section should
-// therefore never appear in the emit file of a -C panic=abort compilation, and this test
-// checks that this is respected.
-// See https://github.com/rust-lang/rust/pull/112403
+// This is useless when panics would NEVER unwind due to -C panic=abort and when we don't need
+// being able to generate backtraces (which depend on unwind tables on linux). This section should
+// therefore never appear in the emit file of a -C panic=abort compilation
+// with -C force-unwind-tables=no, and this test checks that this is respected.
+// See https://github.com/rust-lang/rust/pull/112403 and
+// https://github.com/rust-lang/rust/pull/143613.
 
 //@ only-linux
 // FIXME(Oneirical): the DW_CFA symbol appears on Windows-gnu, because uwtable
@@ -19,6 +21,7 @@ fn main() {
         .panic("abort")
         .edition("2021")
         .arg("-Zvalidate-mir")
+        .arg("-Cforce-unwind-tables=no")
         .run();
     llvm_objdump().arg("--dwarf=frames").input("foo.o").run().assert_stdout_not_contains("DW_CFA");
 }
diff --git a/tests/run-make/pdb-buildinfo-cl-cmd/filecheck.txt b/tests/run-make/pdb-buildinfo-cl-cmd/filecheck.txt
deleted file mode 100644
index a01999d5bdf7..000000000000
--- a/tests/run-make/pdb-buildinfo-cl-cmd/filecheck.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-CHECK: LF_BUILDINFO
-CHECK: rustc.exe
-CHECK: main.rs
-CHECK: "-g" "--crate-name" "my_crate_name" "--crate-type" "bin" "-Cmetadata=dc9ef878b0a48666"
diff --git a/tests/run-make/pdb-buildinfo-cl-cmd/rmake.rs b/tests/run-make/pdb-buildinfo-cl-cmd/rmake.rs
deleted file mode 100644
index 9418f4f8d84d..000000000000
--- a/tests/run-make/pdb-buildinfo-cl-cmd/rmake.rs
+++ /dev/null
@@ -1,25 +0,0 @@
-// Check if the pdb file contains the following information in the LF_BUILDINFO:
-// 1. full path to the compiler (cl)
-// 2. the commandline args to compile it (cmd)
-// This is because these used to be missing in #96475.
-// See https://github.com/rust-lang/rust/pull/113492
-
-//@ only-windows-msvc
-// Reason: pdb files are unique to this architecture
-
-use run_make_support::{llvm, rustc};
-
-fn main() {
-    rustc()
-        .input("main.rs")
-        .arg("-g")
-        .crate_name("my_crate_name")
-        .crate_type("bin")
-        .metadata("dc9ef878b0a48666")
-        .run();
-
-    let pdbutil_result =
-        llvm::llvm_pdbutil().arg("dump").arg("-ids").input("my_crate_name.pdb").run();
-
-    llvm::llvm_filecheck().patterns("filecheck.txt").stdin_buf(pdbutil_result.stdout_utf8()).run();
-}
diff --git a/tests/run-make/pgo-branch-weights/rmake.rs b/tests/run-make/pgo-branch-weights/rmake.rs
index 1893248e3077..e74eabc18754 100644
--- a/tests/run-make/pgo-branch-weights/rmake.rs
+++ b/tests/run-make/pgo-branch-weights/rmake.rs
@@ -17,15 +17,21 @@ use run_make_support::{llvm_filecheck, llvm_profdata, rfs, run_with_args, rustc}
 fn main() {
     let path_prof_data_dir = Path::new("prof_data_dir");
     let path_merged_profdata = path_prof_data_dir.join("merged.profdata");
-    rustc().input("opaque.rs").run();
+    rustc().input("opaque.rs").codegen_source_order().run();
     rfs::create_dir_all(&path_prof_data_dir);
     rustc()
         .input("interesting.rs")
         .profile_generate(&path_prof_data_dir)
         .opt()
         .codegen_units(1)
+        .codegen_source_order()
+        .run();
+    rustc()
+        .input("main.rs")
+        .profile_generate(&path_prof_data_dir)
+        .opt()
+        .codegen_source_order()
         .run();
-    rustc().input("main.rs").profile_generate(&path_prof_data_dir).opt().run();
     run_with_args("main", &["aaaaaaaaaaaa2bbbbbbbbbbbb2bbbbbbbbbbbbbbbbcc"]);
     llvm_profdata().merge().output(&path_merged_profdata).input(path_prof_data_dir).run();
     rustc()
@@ -34,6 +40,7 @@ fn main() {
         .opt()
         .codegen_units(1)
         .emit("llvm-ir")
+        .codegen_source_order()
         .run();
     llvm_filecheck()
         .patterns("filecheck-patterns.txt")
diff --git a/tests/run-make/pgo-indirect-call-promotion/rmake.rs b/tests/run-make/pgo-indirect-call-promotion/rmake.rs
index ce9754f13b93..ee09141912b1 100644
--- a/tests/run-make/pgo-indirect-call-promotion/rmake.rs
+++ b/tests/run-make/pgo-indirect-call-promotion/rmake.rs
@@ -14,11 +14,17 @@ use run_make_support::{llvm_filecheck, llvm_profdata, rfs, run, rustc};
 
 fn main() {
     // We don't compile `opaque` with either optimizations or instrumentation.
-    rustc().input("opaque.rs").run();
+    rustc().input("opaque.rs").codegen_source_order().run();
     // Compile the test program with instrumentation
     rfs::create_dir("prof_data_dir");
-    rustc().input("interesting.rs").profile_generate("prof_data_dir").opt().codegen_units(1).run();
-    rustc().input("main.rs").profile_generate("prof_data_dir").opt().run();
+    rustc()
+        .input("interesting.rs")
+        .profile_generate("prof_data_dir")
+        .opt()
+        .codegen_units(1)
+        .codegen_source_order()
+        .run();
+    rustc().input("main.rs").profile_generate("prof_data_dir").opt().codegen_source_order().run();
     // The argument below generates to the expected branch weights
     run("main");
     llvm_profdata().merge().output("prof_data_dir/merged.profdata").input("prof_data_dir").run();
@@ -28,6 +34,7 @@ fn main() {
         .opt()
         .codegen_units(1)
         .emit("llvm-ir")
+        .codegen_source_order()
         .run();
     llvm_filecheck()
         .patterns("filecheck-patterns.txt")
diff --git a/tests/run-make/pgo-use/rmake.rs b/tests/run-make/pgo-use/rmake.rs
index c09a82353b9b..137b0b859a04 100644
--- a/tests/run-make/pgo-use/rmake.rs
+++ b/tests/run-make/pgo-use/rmake.rs
@@ -22,6 +22,7 @@ fn main() {
         .opt_level("2")
         .codegen_units(1)
         .arg("-Cllvm-args=-disable-preinline")
+        .codegen_source_order()
         .profile_generate(cwd())
         .input("main.rs")
         .run();
@@ -40,6 +41,7 @@ fn main() {
         .arg("-Cllvm-args=-disable-preinline")
         .profile_use("merged.profdata")
         .emit("llvm-ir")
+        .codegen_source_order()
         .input("main.rs")
         .run();
     // Check that the generate IR contains some things that we expect.
diff --git a/tests/run-make/pointer-auth-link-with-c-lto-clang/rmake.rs b/tests/run-make/pointer-auth-link-with-c-lto-clang/rmake.rs
index 0a2186b09538..2ac5fdee063c 100644
--- a/tests/run-make/pointer-auth-link-with-c-lto-clang/rmake.rs
+++ b/tests/run-make/pointer-auth-link-with-c-lto-clang/rmake.rs
@@ -1,12 +1,14 @@
 // `-Z branch protection` is an unstable compiler feature which adds pointer-authentication
 // code (PAC), a useful hashing measure for verifying that pointers have not been modified.
 // This test checks that compilation and execution is successful when this feature is activated,
-// with some of its possible extra arguments (bti, pac-ret, leaf) when doing LTO.
+// with some of its possible extra arguments (bti, gcs, pac-ret, leaf) when doing LTO.
 // See https://github.com/rust-lang/rust/pull/88354
 
 //@ needs-force-clang-based-tests
 //@ only-aarch64
 // Reason: branch protection is not supported on other architectures
+//@ ignore-apple
+// Reason: XCode needs updating to support gcs
 //@ ignore-cross-compile
 // Reason: the compiled binary is executed
 
@@ -19,7 +21,7 @@ fn main() {
     clang()
         .arg("-v")
         .lto("thin")
-        .arg("-mbranch-protection=bti+pac-ret+b-key+leaf")
+        .arg("-mbranch-protection=bti+gcs+pac-ret+b-key+leaf")
         .arg("-c")
         .out_exe("test.o")
         .input("test.c")
@@ -30,7 +32,7 @@ fn main() {
         .opt_level("2")
         .linker(&env_var("CLANG"))
         .link_arg("-fuse-ld=lld")
-        .arg("-Zbranch-protection=bti,pac-ret,leaf")
+        .arg("-Zbranch-protection=bti,gcs,pac-ret,leaf")
         .input("test.rs")
         .output("test.bin")
         .run();
diff --git a/tests/run-make/pointer-auth-link-with-c/rmake.rs b/tests/run-make/pointer-auth-link-with-c/rmake.rs
index a4d7454e5755..1ddcb79d64ff 100644
--- a/tests/run-make/pointer-auth-link-with-c/rmake.rs
+++ b/tests/run-make/pointer-auth-link-with-c/rmake.rs
@@ -1,11 +1,13 @@
 // `-Z branch protection` is an unstable compiler feature which adds pointer-authentication
 // code (PAC), a useful hashing measure for verifying that pointers have not been modified.
 // This test checks that compilation and execution is successful when this feature is activated,
-// with some of its possible extra arguments (bti, pac-ret, pc, leaf, b-key).
+// with some of its possible extra arguments (bti, gcs, pac-ret, pc, leaf, b-key).
 // See https://github.com/rust-lang/rust/pull/88354
 
 //@ only-aarch64
 // Reason: branch protection is not supported on other architectures
+//@ ignore-apple
+// Reason: XCode needs updating to support gcs
 //@ ignore-cross-compile
 // Reason: the compiled binary is executed
 
@@ -13,17 +15,17 @@ use run_make_support::{build_native_static_lib, cc, is_windows_msvc, llvm_ar, ru
 
 fn main() {
     build_native_static_lib("test");
-    rustc().arg("-Zbranch-protection=bti,pac-ret,leaf").input("test.rs").run();
+    rustc().arg("-Zbranch-protection=bti,gcs,pac-ret,leaf").input("test.rs").run();
     run("test");
     cc().arg("-v")
         .arg("-c")
         .out_exe("test")
         .input("test.c")
-        .arg("-mbranch-protection=bti+pac-ret+leaf")
+        .arg("-mbranch-protection=bti+gcs+pac-ret+leaf")
         .run();
     let obj_file = if is_windows_msvc() { "test.obj" } else { "test" };
     llvm_ar().obj_to_ar().output_input("libtest.a", &obj_file).run();
-    rustc().arg("-Zbranch-protection=bti,pac-ret,leaf").input("test.rs").run();
+    rustc().arg("-Zbranch-protection=bti,gcs,pac-ret,leaf").input("test.rs").run();
     run("test");
 
     // FIXME: +pc was only recently added to LLVM
diff --git a/tests/run-make/print-request-help-stable-unstable/help-diff.diff b/tests/run-make/print-request-help-stable-unstable/help-diff.diff
index 07eafca32710..044302a19a01 100644
--- a/tests/run-make/print-request-help-stable-unstable/help-diff.diff
+++ b/tests/run-make/print-request-help-stable-unstable/help-diff.diff
@@ -2,6 +2,6 @@
  error: unknown print request: `xxx`
    |
 -  = help: valid print requests are: `calling-conventions`, `cfg`, `code-models`, `crate-name`, `deployment-target`, `file-names`, `host-tuple`, `link-args`, `native-static-libs`, `relocation-models`, `split-debuginfo`, `stack-protector-strategies`, `sysroot`, `target-cpus`, `target-features`, `target-libdir`, `target-list`, `tls-models`
-+  = help: valid print requests are: `all-target-specs-json`, `calling-conventions`, `cfg`, `check-cfg`, `code-models`, `crate-name`, `crate-root-lint-levels`, `deployment-target`, `file-names`, `host-tuple`, `link-args`, `native-static-libs`, `relocation-models`, `split-debuginfo`, `stack-protector-strategies`, `supported-crate-types`, `sysroot`, `target-cpus`, `target-features`, `target-libdir`, `target-list`, `target-spec-json`, `tls-models`
++  = help: valid print requests are: `all-target-specs-json`, `calling-conventions`, `cfg`, `check-cfg`, `code-models`, `crate-name`, `crate-root-lint-levels`, `deployment-target`, `file-names`, `host-tuple`, `link-args`, `native-static-libs`, `relocation-models`, `split-debuginfo`, `stack-protector-strategies`, `supported-crate-types`, `sysroot`, `target-cpus`, `target-features`, `target-libdir`, `target-list`, `target-spec-json`, `target-spec-json-schema`, `tls-models`
    = help: for more information, see the rustc book: https://doc.rust-lang.org/rustc/command-line-arguments.html#--print-print-compiler-information
  
diff --git a/tests/run-make/print-request-help-stable-unstable/unstable-invalid-print-request-help.err b/tests/run-make/print-request-help-stable-unstable/unstable-invalid-print-request-help.err
index 50ef340e3dd0..cc6c3c909b36 100644
--- a/tests/run-make/print-request-help-stable-unstable/unstable-invalid-print-request-help.err
+++ b/tests/run-make/print-request-help-stable-unstable/unstable-invalid-print-request-help.err
@@ -1,5 +1,5 @@
 error: unknown print request: `xxx`
   |
-  = help: valid print requests are: `all-target-specs-json`, `calling-conventions`, `cfg`, `check-cfg`, `code-models`, `crate-name`, `crate-root-lint-levels`, `deployment-target`, `file-names`, `host-tuple`, `link-args`, `native-static-libs`, `relocation-models`, `split-debuginfo`, `stack-protector-strategies`, `supported-crate-types`, `sysroot`, `target-cpus`, `target-features`, `target-libdir`, `target-list`, `target-spec-json`, `tls-models`
+  = help: valid print requests are: `all-target-specs-json`, `calling-conventions`, `cfg`, `check-cfg`, `code-models`, `crate-name`, `crate-root-lint-levels`, `deployment-target`, `file-names`, `host-tuple`, `link-args`, `native-static-libs`, `relocation-models`, `split-debuginfo`, `stack-protector-strategies`, `supported-crate-types`, `sysroot`, `target-cpus`, `target-features`, `target-libdir`, `target-list`, `target-spec-json`, `target-spec-json-schema`, `tls-models`
   = help: for more information, see the rustc book: https://doc.rust-lang.org/rustc/command-line-arguments.html#--print-print-compiler-information
 
diff --git a/tests/run-make/raw-dylib-link-ordinal/exporter.def b/tests/run-make/raw-dylib-link-ordinal/exporter.def
index 5d87c580a54d..0544e9f18033 100644
--- a/tests/run-make/raw-dylib-link-ordinal/exporter.def
+++ b/tests/run-make/raw-dylib-link-ordinal/exporter.def
@@ -1,5 +1,5 @@
 LIBRARY exporter
 EXPORTS
     exported_function @13 NONAME
-    exported_variable @5 NONAME
+    exported_variable @5 NONAME DATA
     print_exported_variable @9 NONAME
diff --git a/tests/run-make/raw-dylib-link-ordinal/rmake.rs b/tests/run-make/raw-dylib-link-ordinal/rmake.rs
index 43274b9765bb..b9254b167534 100644
--- a/tests/run-make/raw-dylib-link-ordinal/rmake.rs
+++ b/tests/run-make/raw-dylib-link-ordinal/rmake.rs
@@ -11,7 +11,7 @@
 
 //@ only-windows
 
-use run_make_support::{cc, diff, is_windows_msvc, run, rustc};
+use run_make_support::{cc, diff, extra_c_flags, is_windows_msvc, run, rustc};
 
 // NOTE: build_native_dynamic lib is not used, as the special `def` files
 // must be passed to the CC compiler.
@@ -24,6 +24,7 @@ fn main() {
         cc().input("exporter.obj")
             .arg("exporter.def")
             .args(&["-link", "-dll", "-noimplib", "-out:exporter.dll"])
+            .args(extra_c_flags())
             .run();
     } else {
         cc().arg("-v").arg("-c").out_exe("exporter.obj").input("exporter.c").run();
diff --git a/tests/run-make/rust-lld-custom-target/custom-target.json b/tests/run-make/rust-lld-custom-target/custom-target.json
index e2c64cbdb43c..28c3dc6f387a 100644
--- a/tests/run-make/rust-lld-custom-target/custom-target.json
+++ b/tests/run-make/rust-lld-custom-target/custom-target.json
@@ -53,5 +53,5 @@
   "target-family": [
     "unix"
   ],
-  "target-pointer-width": "64"
+  "target-pointer-width": 64
 }
diff --git a/tests/run-make/rustc-help/help-v.stdout b/tests/run-make/rustc-help/help-v.stdout
index 3fc297fb08e7..cd161c51ee3b 100644
--- a/tests/run-make/rustc-help/help-v.stdout
+++ b/tests/run-make/rustc-help/help-v.stdout
@@ -43,7 +43,7 @@ Options:
         --print [=]
                         Compiler information to print on stdout (or to a file)
                         INFO may be one of
-                        .
+                        .
     -g                  Equivalent to -C debuginfo=2
     -O                  Equivalent to -C opt-level=3
     -o        Write output to FILENAME
diff --git a/tests/run-make/rustc-help/help.stdout b/tests/run-make/rustc-help/help.stdout
index caffe28f4989..74ec083bdee7 100644
--- a/tests/run-make/rustc-help/help.stdout
+++ b/tests/run-make/rustc-help/help.stdout
@@ -43,7 +43,7 @@ Options:
         --print [=]
                         Compiler information to print on stdout (or to a file)
                         INFO may be one of
-                        .
+                        .
     -g                  Equivalent to -C debuginfo=2
     -O                  Equivalent to -C opt-level=3
     -o        Write output to FILENAME
diff --git a/tests/run-make/rustdoc-default-output/output-default.stdout b/tests/run-make/rustdoc-default-output/output-default.stdout
index 506f135ff8e7..49eaf7e2e1e0 100644
--- a/tests/run-make/rustdoc-default-output/output-default.stdout
+++ b/tests/run-make/rustdoc-default-output/output-default.stdout
@@ -150,7 +150,7 @@ Options:
         --generate-redirect-map 
                         Generate JSON file at the top level instead of
                         generating HTML redirection files
-        --emit [unversioned-shared-resources,toolchain-shared-resources,invocation-specific,dep-info]
+        --emit [toolchain-shared-resources,invocation-specific,dep-info]
                         Comma separated list of types of output for rustdoc to
                         emit
         --no-run        Compile doctests without running them
@@ -158,7 +158,7 @@ Options:
                         Remap source names in compiler messages
         --show-type-layout 
                         Include the memory layout of types in the docs
-        --nocapture     Don't capture stdout and stderr of tests
+        --no-capture    Don't capture stdout and stderr of tests
         --generate-link-to-definition 
                         Make the identifiers in the HTML source code pages
                         navigable
@@ -194,6 +194,9 @@ Options:
         --disable-minification 
                         disable the minification of CSS/JS files
                         (perma-unstable, do not use with cached files)
+        --generate-macro-expansion 
+                        Add possibility to expand macros in the HTML source
+                        code pages
         --plugin-path DIR
                         removed, see issue #44136
                          for
diff --git a/tests/run-make/rustdoc-dep-info/rmake.rs b/tests/run-make/rustdoc-dep-info/rmake.rs
index 625f81fd428e..5d6176b18e88 100644
--- a/tests/run-make/rustdoc-dep-info/rmake.rs
+++ b/tests/run-make/rustdoc-dep-info/rmake.rs
@@ -33,4 +33,27 @@ fn main() {
     // Now we check that we can provide a file name to the `dep-info` argument.
     rustdoc().input("lib.rs").arg("-Zunstable-options").emit("dep-info=bla.d").run();
     assert!(path("bla.d").exists());
+
+    // The last emit-type wins. The same behavior as rustc.
+    rustdoc()
+        .input("lib.rs")
+        .arg("-Zunstable-options")
+        .emit("dep-info=precedence1.d")
+        .emit("dep-info=precedence2.d")
+        .emit("dep-info=precedence3.d")
+        .run();
+    assert!(!path("precedence1.d").exists());
+    assert!(!path("precedence2.d").exists());
+    assert!(path("precedence3.d").exists());
+
+    // stdout (-) also wins if being the last.
+    let result = rustdoc()
+        .input("lib.rs")
+        .arg("-Zunstable-options")
+        .emit("dep-info=precedence1.d")
+        .emit("dep-info=-")
+        .run();
+    assert!(!path("precedence1.d").exists());
+    assert!(!path("-").exists()); // `-` shouldn't be treated as a file path
+    assert!(!result.stdout().is_empty()); // Something emitted to stdout
 }
diff --git a/tests/run-make/rustdoc-determinism/rmake.rs b/tests/run-make/rustdoc-determinism/rmake.rs
index 921baef4a979..5705dff6858c 100644
--- a/tests/run-make/rustdoc-determinism/rmake.rs
+++ b/tests/run-make/rustdoc-determinism/rmake.rs
@@ -15,7 +15,7 @@ fn main() {
     rustdoc().input("foo.rs").out_dir(&bar_first).run();
 
     diff()
-        .expected_file(foo_first.join("search-index.js"))
-        .actual_file(bar_first.join("search-index.js"))
+        .expected_file(foo_first.join("search.index/root.js"))
+        .actual_file(bar_first.join("search.index/root.js"))
         .run();
 }
diff --git a/tests/run-make/rustdoc-doctest-output-format/file.rs b/tests/run-make/rustdoc-doctest-output-format/file.rs
new file mode 100644
index 000000000000..51d17849fd71
--- /dev/null
+++ b/tests/run-make/rustdoc-doctest-output-format/file.rs
@@ -0,0 +1,3 @@
+//! ```
+//! let x = 12;
+//! ```
diff --git a/tests/run-make/rustdoc-doctest-output-format/rmake.rs b/tests/run-make/rustdoc-doctest-output-format/rmake.rs
new file mode 100644
index 000000000000..61b5c0c4fd11
--- /dev/null
+++ b/tests/run-make/rustdoc-doctest-output-format/rmake.rs
@@ -0,0 +1,83 @@
+//! Regression test to ensure that the output format is respected for doctests.
+//!
+//! Regression test for .
+
+//@ ignore-cross-compile
+
+use run_make_support::{rustdoc, serde_json};
+
+fn run_test(edition: &str, format: Option<&str>) -> String {
+    let mut r = rustdoc();
+    r.input("file.rs").edition(edition).arg("--test");
+    if let Some(format) = format {
+        r.args(&[
+            "--test-args",
+            "-Zunstable-options",
+            "--test-args",
+            "--format",
+            "--test-args",
+            format,
+        ]);
+    }
+    r.run().stdout_utf8()
+}
+
+fn check_json_output(edition: &str, expected_reports: usize) {
+    let out = run_test(edition, Some("json"));
+    let mut found_report = 0;
+    for (line_nb, line) in out.lines().enumerate() {
+        match serde_json::from_str::(&line) {
+            Ok(value) => {
+                if value.get("type") == Some(&serde_json::json!("report")) {
+                    found_report += 1;
+                }
+            }
+            Err(error) => panic!(
+                "failed for {edition} edition (json format) at line {}: non-JSON value: {error}\n\
+                ====== output ======\n{out}",
+                line_nb + 1,
+            ),
+        }
+    }
+    if found_report != expected_reports {
+        panic!(
+            "failed for {edition} edition (json format): expected {expected_reports} doctest \
+             time `report`, found {found_report}\n====== output ======\n{out}",
+        );
+    }
+}
+
+fn check_non_json_output(edition: &str, expected_reports: usize) {
+    let out = run_test(edition, None);
+    let mut found_report = 0;
+    for (line_nb, line) in out.lines().enumerate() {
+        if line.starts_with('{') && serde_json::from_str::(&line).is_ok() {
+            panic!(
+                "failed for {edition} edition: unexpected json at line {}: `{line}`\n\
+                 ====== output ======\n{out}",
+                line_nb + 1
+            );
+        }
+        if line.starts_with("all doctests ran in")
+            && line.contains("; merged doctests compilation took ")
+        {
+            found_report += 1;
+        }
+    }
+    if found_report != expected_reports {
+        panic!(
+            "failed for {edition} edition: expected {expected_reports} doctest time `report`, \
+             found {found_report}\n====== output ======\n{out}",
+        );
+    }
+}
+
+fn main() {
+    // Only the merged doctests generate the "times report".
+    check_json_output("2021", 0);
+    check_json_output("2024", 1);
+
+    // Only the merged doctests generate the "times report".
+    check_non_json_output("2021", 0);
+    check_non_json_output("2024", 1);
+}
diff --git a/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs b/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs
new file mode 100644
index 000000000000..0b1e1948d5fc
--- /dev/null
+++ b/tests/run-make/rustdoc-merge-no-input-finalize/rmake.rs
@@ -0,0 +1,28 @@
+// Running --merge=finalize without an input crate root should not trigger ICE.
+// Issue: https://github.com/rust-lang/rust/issues/146646
+
+//@ needs-target-std
+
+use run_make_support::{path, rustdoc};
+
+fn main() {
+    let out_dir = path("out");
+    let merged_dir = path("merged");
+    let parts_out_dir = path("parts");
+    rustdoc()
+        .input("sierra.rs")
+        .out_dir(&out_dir)
+        .arg("-Zunstable-options")
+        .arg(format!("--parts-out-dir={}", parts_out_dir.display()))
+        .arg("--merge=none")
+        .run();
+    assert!(parts_out_dir.join("crate-info").exists());
+
+    let output = rustdoc()
+        .arg("-Zunstable-options")
+        .out_dir(&out_dir)
+        .arg(format!("--include-parts-dir={}", parts_out_dir.display()))
+        .arg("--merge=finalize")
+        .run();
+    output.assert_stderr_not_contains("error: the compiler unexpectedly panicked. this is a bug.");
+}
diff --git a/tests/run-make/rustdoc-merge-no-input-finalize/sierra.rs b/tests/run-make/rustdoc-merge-no-input-finalize/sierra.rs
new file mode 100644
index 000000000000..f8fc48341ed6
--- /dev/null
+++ b/tests/run-make/rustdoc-merge-no-input-finalize/sierra.rs
@@ -0,0 +1 @@
+pub struct Sierra;
diff --git a/tests/run-make/rustdoc-search-load-itemtype/bar.rs b/tests/run-make/rustdoc-search-load-itemtype/bar.rs
new file mode 100644
index 000000000000..0416b1b75b59
--- /dev/null
+++ b/tests/run-make/rustdoc-search-load-itemtype/bar.rs
@@ -0,0 +1,27 @@
+#![crate_type = "proc-macro"]
+
+extern crate proc_macro;
+use proc_macro::*;
+
+//@ has bar/macro.a_procmacro.html
+//@ hasraw search.index/name/*.js a_procmacro
+#[proc_macro]
+pub fn a_procmacro(_: TokenStream) -> TokenStream {
+    unimplemented!()
+}
+
+//@ has bar/attr.a_procattribute.html
+//@ hasraw search.index/name/*.js a_procattribute
+#[proc_macro_attribute]
+pub fn a_procattribute(_: TokenStream, _: TokenStream) -> TokenStream {
+    unimplemented!()
+}
+
+//@ has bar/derive.AProcDerive.html
+//@ !has bar/derive.a_procderive.html
+//@ hasraw search.index/name/*.js AProcDerive
+//@ !hasraw search.index/name/*.js a_procderive
+#[proc_macro_derive(AProcDerive)]
+pub fn a_procderive(_: TokenStream) -> TokenStream {
+    unimplemented!()
+}
diff --git a/tests/run-make/rustdoc-search-load-itemtype/baz.rs b/tests/run-make/rustdoc-search-load-itemtype/baz.rs
new file mode 100644
index 000000000000..4d1f5430fc63
--- /dev/null
+++ b/tests/run-make/rustdoc-search-load-itemtype/baz.rs
@@ -0,0 +1,3 @@
+//@ has baz/struct.Baz.html
+//@ hasraw search.index/name/*.js Baz
+pub struct Baz;
diff --git a/tests/run-make/rustdoc-search-load-itemtype/foo.rs b/tests/run-make/rustdoc-search-load-itemtype/foo.rs
new file mode 100644
index 000000000000..93b372d10cbf
--- /dev/null
+++ b/tests/run-make/rustdoc-search-load-itemtype/foo.rs
@@ -0,0 +1,119 @@
+#![feature(extern_types, rustc_attrs, rustdoc_internals, trait_alias)]
+#![allow(internal_features)]
+#![no_std]
+
+//@ has foo/keyword.while.html
+//@ hasraw search.index/name/*.js while
+//@ !hasraw search.index/name/*.js w_keyword
+#[doc(keyword = "while")]
+mod w_keyword {}
+
+//@ has foo/primitive.u32.html
+//@ hasraw search.index/name/*.js u32
+//@ !hasraw search.index/name/*.js u_primitive
+#[rustc_doc_primitive = "u32"]
+mod u_primitive {}
+
+//@ has foo/x_mod/index.html
+//@ hasraw search.index/name/*.js x_mod
+pub mod x_mod {}
+
+//@ hasraw foo/index.html y_crate
+//@ hasraw search.index/name/*.js y_crate
+#[doc(no_inline)]
+pub extern crate core as y_crate;
+
+//@ hasraw foo/index.html z_import
+//@ hasraw search.index/name/*.js z_import
+#[doc(no_inline)]
+pub use core::option as z_import;
+
+//@ has foo/struct.AStruct.html
+//@ hasraw search.index/name/*.js AStruct
+pub struct AStruct {
+    //@ hasraw foo/struct.AStruct.html a_structfield
+    //@ hasraw search.index/name/*.js a_structfield
+    pub a_structfield: i32,
+}
+
+//@ has foo/enum.AEnum.html
+//@ hasraw search.index/name/*.js AEnum
+pub enum AEnum {
+    //@ hasraw foo/enum.AEnum.html AVariant
+    //@ hasraw search.index/name/*.js AVariant
+    AVariant,
+}
+
+//@ has foo/fn.a_fn.html
+//@ hasraw search.index/name/*.js a_fn
+pub fn a_fn() {}
+
+//@ has foo/type.AType.html
+//@ hasraw search.index/name/*.js AType
+pub type AType = AStruct;
+
+//@ has foo/static.a_static.html
+//@ hasraw search.index/name/*.js a_static
+pub static a_static: i32 = 1;
+
+//@ has foo/trait.ATrait.html
+//@ hasraw search.index/name/*.js ATrait
+pub trait ATrait {
+    //@ hasraw foo/trait.ATrait.html a_tymethod
+    //@ hasraw search.index/name/*.js a_tymethod
+    fn a_tymethod();
+    //@ hasraw foo/trait.ATrait.html AAssocType
+    //@ hasraw search.index/name/*.js AAssocType
+    type AAssocType;
+    //@ hasraw foo/trait.ATrait.html AAssocConst
+    //@ hasraw search.index/name/*.js AAssocConst
+    const AAssocConst: bool;
+}
+
+// skip ItemType::Impl, since impls are anonymous
+// and have no search entry
+
+impl AStruct {
+    //@ hasraw foo/struct.AStruct.html a_method
+    //@ hasraw search.index/name/*.js a_method
+    pub fn a_method() {}
+}
+
+//@ has foo/macro.a_macro.html
+//@ hasraw search.index/name/*.js a_macro
+#[macro_export]
+macro_rules! a_macro {
+    () => {};
+}
+
+//@ has foo/constant.A_CONSTANT.html
+//@ hasraw search.index/name/*.js A_CONSTANT
+pub const A_CONSTANT: i32 = 1;
+
+//@ has foo/union.AUnion.html
+//@ hasraw search.index/name/*.js AUnion
+pub union AUnion {
+    //@ hasraw foo/union.AUnion.html a_unionfield
+    //@ hasraw search.index/name/*.js a_unionfield
+    pub a_unionfield: i32,
+}
+
+extern "C" {
+    //@ has foo/foreigntype.AForeignType.html
+    //@ hasraw search.index/name/*.js AForeignType
+    pub type AForeignType;
+}
+
+// procattribute and procderive are defined in
+// bar.rs, because they only work with proc_macro
+// crate type.
+
+//@ has foo/traitalias.ATraitAlias.html
+//@ hasraw search.index/name/*.js ATraitAlias
+pub trait ATraitAlias = ATrait;
+
+//@ has foo/attribute.doc.html
+//@ hasraw search.index/name/*.js doc
+//@ !hasraw search.index/name/*.js aa_mod
+#[doc(attribute = "doc")]
+mod aa_mod {}
diff --git a/tests/run-make/rustdoc-search-load-itemtype/rmake.rs b/tests/run-make/rustdoc-search-load-itemtype/rmake.rs
new file mode 100644
index 000000000000..803eb8a61dad
--- /dev/null
+++ b/tests/run-make/rustdoc-search-load-itemtype/rmake.rs
@@ -0,0 +1,21 @@
+//@ ignore-cross-compile
+//@ needs-crate-type: proc-macro
+
+// Test that rustdoc can deserialize a search index with every itemtype.
+// https://github.com/rust-lang/rust/pull/146117
+
+use std::path::Path;
+
+use run_make_support::{htmldocck, rfs, rustdoc, source_root};
+
+fn main() {
+    let out_dir = Path::new("rustdoc-search-load-itemtype");
+
+    rfs::create_dir_all(&out_dir);
+    rustdoc().out_dir(&out_dir).input("foo.rs").run();
+    rustdoc().out_dir(&out_dir).input("bar.rs").arg("--crate-type=proc-macro").run();
+    rustdoc().out_dir(&out_dir).input("baz.rs").run();
+    htmldocck().arg(out_dir).arg("foo.rs").run();
+    htmldocck().arg(out_dir).arg("bar.rs").run();
+    htmldocck().arg(out_dir).arg("baz.rs").run();
+}
diff --git a/tests/run-make/rustdoc-target-spec-json-path/target.json b/tests/run-make/rustdoc-target-spec-json-path/target.json
index d7e4cac57ae7..6d8fe8528c80 100644
--- a/tests/run-make/rustdoc-target-spec-json-path/target.json
+++ b/tests/run-make/rustdoc-target-spec-json-path/target.json
@@ -33,5 +33,5 @@
     "thread"
   ],
   "target-family": "unix",
-  "target-pointer-width": "64"
+  "target-pointer-width": 64
 }
diff --git a/tests/run-make/sanitizer-cdylib-link/rmake.rs b/tests/run-make/sanitizer-cdylib-link/rmake.rs
index f9d7fd98789e..8ff58594d109 100644
--- a/tests/run-make/sanitizer-cdylib-link/rmake.rs
+++ b/tests/run-make/sanitizer-cdylib-link/rmake.rs
@@ -8,6 +8,8 @@
 //@ needs-sanitizer-support
 //@ needs-sanitizer-address
 
+//@ compile-flags: -C unsafe-allow-abi-mismatch=sanitizer
+
 use run_make_support::{run_fail, rustc};
 
 fn main() {
diff --git a/tests/run-make/sanitizer-dylib-link/rmake.rs b/tests/run-make/sanitizer-dylib-link/rmake.rs
index b43420adc72b..bae3240a2364 100644
--- a/tests/run-make/sanitizer-dylib-link/rmake.rs
+++ b/tests/run-make/sanitizer-dylib-link/rmake.rs
@@ -7,6 +7,8 @@
 //@ needs-sanitizer-support
 //@ needs-sanitizer-address
 
+//@ compile-flags: -C unsafe-allow-abi-mismatch=sanitizer
+
 use run_make_support::{run_fail, rustc};
 
 fn main() {
diff --git a/tests/run-make/sanitizer-staticlib-link/rmake.rs b/tests/run-make/sanitizer-staticlib-link/rmake.rs
index e38d15a8a3c4..dda926261890 100644
--- a/tests/run-make/sanitizer-staticlib-link/rmake.rs
+++ b/tests/run-make/sanitizer-staticlib-link/rmake.rs
@@ -11,6 +11,8 @@
 //@ needs-sanitizer-support
 //@ needs-sanitizer-address
 
+//@ compile-flags: -C unsafe-allow-abi-mismatch=sanitizer
+
 use run_make_support::{cc, extra_c_flags, extra_cxx_flags, run_fail, rustc, static_lib_name};
 
 fn main() {
diff --git a/tests/run-make/separate-link-fail/rmake.rs b/tests/run-make/separate-link-fail/rmake.rs
index b5d5300de68f..ba24f7773953 100644
--- a/tests/run-make/separate-link-fail/rmake.rs
+++ b/tests/run-make/separate-link-fail/rmake.rs
@@ -11,5 +11,5 @@ fn main() {
         .arg("-Zlink-only")
         .input("foo.rs")
         .run_fail()
-        .assert_stderr_contains("The input does not look like a .rlink file");
+        .assert_stderr_contains("the input does not look like a .rlink file");
 }
diff --git a/tests/run-make/split-debuginfo/rmake.rs b/tests/run-make/split-debuginfo/rmake.rs
index e8de5aed1726..e53b71010781 100644
--- a/tests/run-make/split-debuginfo/rmake.rs
+++ b/tests/run-make/split-debuginfo/rmake.rs
@@ -187,6 +187,25 @@ enum UnstableOptions {
     Unspecified,
 }
 
+#[track_caller]
+fn dwo_out_filenames(dwo_out: Option<&str>) -> BTreeSet {
+    let dwo_out = if let Some(d) = dwo_out {
+        d
+    } else {
+        return BTreeSet::new();
+    };
+    let files = shallow_find_files(dwo_out, |path| {
+        // Fiilter out source files
+        !has_extension(path, "rs")
+    });
+    files
+        .iter()
+        .map(|p| {
+            format!("{}/{}", dwo_out, p.file_name().unwrap().to_os_string().into_string().unwrap())
+        })
+        .collect()
+}
+
 #[track_caller]
 fn cwd_filenames() -> BTreeSet {
     let files = shallow_find_files(cwd(), |path| {
@@ -196,6 +215,17 @@ fn cwd_filenames() -> BTreeSet {
     files.iter().map(|p| p.file_name().unwrap().to_os_string().into_string().unwrap()).collect()
 }
 
+#[track_caller]
+fn dwo_out_dwo_filenames(dwo_out: &str) -> BTreeSet {
+    let files = shallow_find_files(dwo_out, |p| has_extension(p, "dwo"));
+    files
+        .iter()
+        .map(|p| {
+            format!("{}/{}", dwo_out, p.file_name().unwrap().to_os_string().into_string().unwrap())
+        })
+        .collect()
+}
+
 #[track_caller]
 fn cwd_dwo_filenames() -> BTreeSet {
     let files = shallow_find_files(cwd(), |path| has_extension(path, "dwo"));
@@ -376,17 +406,19 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
         run_in_tmpdir(|| {
             println!(
-                "checking: unstable_options={:?} + split_kind={:?} + level={:?} + split_dwarf_kind={:?} + lto={:?} + remap_path_prefix={:?} + remap_path_scope={:?}",
+                "checking: unstable_options={:?} + split_kind={:?} + level={:?} + split_dwarf_kind={:?} + lto={:?} + remap_path_prefix={:?} + remap_path_scope={:?} + split_dwarf_out_dir={:?}",
                 unstable_options,
                 split_kind,
                 level,
                 split_dwarf_kind,
                 lto,
                 remap_path_prefix,
-                remap_path_scope
+                remap_path_scope,
+                split_dwarf_output_directory,
             );
 
             match cross_crate_test {
@@ -398,6 +430,7 @@ mod shared_linux_other_tests {
                     lto,
                     remap_path_prefix,
                     remap_path_scope,
+                    split_dwarf_output_directory,
                 ),
                 CrossCrateTest::No => simple_split_debuginfo(
                     unstable_options,
@@ -407,6 +440,7 @@ mod shared_linux_other_tests {
                     lto,
                     remap_path_prefix,
                     remap_path_scope,
+                    split_dwarf_output_directory,
                 ),
             }
         });
@@ -420,7 +454,11 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
+        if let Some(dwo_out) = split_dwarf_output_directory {
+            run_make_support::rfs::create_dir(dwo_out);
+        }
         match (split_kind, level, split_dwarf_kind, lto, remap_path_prefix, remap_path_scope) {
             // packed-crosscrate-split
             // - Debuginfo in `.dwo` files
@@ -531,13 +569,19 @@ mod shared_linux_other_tests {
                     .input("bar.rs")
                     .crate_type("lib")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
 
-                let bar_found_files = cwd_filenames();
+                let mut bar_found_files = cwd_filenames();
+                bar_found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let bar_dwo_files = cwd_dwo_filenames();
+                let bar_dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(bar_dwo_files.len(), 1);
 
                 let mut bar_expected_files = BTreeSet::new();
@@ -553,13 +597,19 @@ mod shared_linux_other_tests {
                     .extern_("bar", "libbar.rlib")
                     .input("main.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
 
-                let overall_found_files = cwd_filenames();
+                let mut overall_found_files = cwd_filenames();
+                overall_found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let overall_dwo_files = cwd_dwo_filenames();
+                let overall_dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(overall_dwo_files.len(), 2);
 
                 let mut overall_expected_files = BTreeSet::new();
@@ -648,7 +698,11 @@ mod shared_linux_other_tests {
         lto: LinkerPluginLto,
         remap_path_prefix: RemapPathPrefix,
         remap_path_scope: RemapPathScope,
+        split_dwarf_output_directory: Option<&str>,
     ) {
+        if let Some(dwo_out) = split_dwarf_output_directory {
+            run_make_support::rfs::create_dir(dwo_out);
+        }
         match (split_kind, level, split_dwarf_kind, lto, remap_path_prefix, remap_path_scope) {
             // off (unspecified):
             // - Debuginfo in `.o` files
@@ -921,14 +975,19 @@ mod shared_linux_other_tests {
                 rustc(unstable_options)
                     .input("foo.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .run();
-                let found_files = cwd_filenames();
+                let mut found_files = cwd_filenames();
+                found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let dwo_files = cwd_dwo_filenames();
+                let dwo_files = if let Some(dwo_dir) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_dir)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(dwo_files.len(), 1);
-
                 let mut expected_files = BTreeSet::new();
                 expected_files.extend(dwo_files);
                 expected_files.insert("foo".to_string());
@@ -1056,14 +1115,20 @@ mod shared_linux_other_tests {
                 rustc(unstable_options)
                     .input("foo.rs")
                     .split_debuginfo(split_kind.cli_value())
+                    .split_dwarf_out_dir(split_dwarf_output_directory)
                     .debuginfo(level.cli_value())
                     .arg(format!("-Zsplit-dwarf-kind={}", split_dwarf_kind.cli_value()))
                     .remap_path_prefix(cwd(), remapped_prefix)
                     .run();
 
-                let found_files = cwd_filenames();
+                let mut found_files = cwd_filenames();
+                found_files.append(&mut dwo_out_filenames(split_dwarf_output_directory));
 
-                let dwo_files = cwd_dwo_filenames();
+                let dwo_files = if let Some(dwo_out) = split_dwarf_output_directory {
+                    dwo_out_dwo_filenames(dwo_out)
+                } else {
+                    cwd_dwo_filenames()
+                };
                 assert_eq!(dwo_files.len(), 1);
 
                 let mut expected_files = BTreeSet::new();
@@ -1358,6 +1423,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // off
@@ -1370,6 +1436,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-split
@@ -1382,6 +1449,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-single
@@ -1394,6 +1462,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-lto-split
@@ -1406,6 +1475,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-lto-single
@@ -1418,6 +1488,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // FIXME: the remapping tests probably need to be reworked, see
@@ -1433,6 +1504,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-remapped-single
@@ -1445,6 +1517,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-remapped-scope
@@ -1457,6 +1530,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("debuginfo"),
+            None,
         );
 
         // packed-remapped-wrong-scope
@@ -1469,6 +1543,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("macro"),
+            None,
         );
 
         // packed-crosscrate-split
@@ -1481,6 +1556,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // packed-crosscrate-single
@@ -1493,6 +1569,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-split
@@ -1505,6 +1582,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::No,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Unspecified,
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-single
@@ -1517,6 +1608,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-lto-split
@@ -1529,6 +1621,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-lto-single
@@ -1541,6 +1634,7 @@ fn main() {
             LinkerPluginLto::Yes,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-remapped-split
@@ -1553,6 +1647,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-remapped-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::No,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-remapped-single
@@ -1565,6 +1673,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Unspecified,
+            None,
         );
 
         // unpacked-remapped-scope
@@ -1577,6 +1686,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("debuginfo"),
+            None,
         );
 
         // unpacked-remapped-wrong-scope
@@ -1589,6 +1699,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Yes { remapped_prefix: "/__MY_REMAPPED_PATH__" },
             RemapPathScope::Yes("macro"),
+            None,
         );
 
         // unpacked-crosscrate-split
@@ -1601,6 +1712,20 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
+        );
+
+        // unpacked-crosscrate-split with split-dwarf-out-dir
+        shared_linux_other_tests::split_debuginfo(
+            CrossCrateTest::Yes,
+            UnstableOptions::Yes,
+            SplitDebuginfo::Unpacked,
+            DebuginfoLevel::Full,
+            SplitDwarfKind::Split,
+            LinkerPluginLto::Unspecified,
+            RemapPathPrefix::Unspecified,
+            RemapPathScope::Unspecified,
+            Some("other-dir"),
         );
 
         // unpacked-crosscrate-single
@@ -1613,6 +1738,7 @@ fn main() {
             LinkerPluginLto::Unspecified,
             RemapPathPrefix::Unspecified,
             RemapPathScope::Unspecified,
+            None,
         );
     }
 }
diff --git a/tests/run-make/target-specs/endianness-mismatch.json b/tests/run-make/target-specs/endianness-mismatch.json
index cc03becc59af..d73ea1cbcfec 100644
--- a/tests/run-make/target-specs/endianness-mismatch.json
+++ b/tests/run-make/target-specs/endianness-mismatch.json
@@ -4,7 +4,7 @@
     "linker-flavor": "gcc",
     "llvm-target": "x86_64-unknown-linux-gnu",
     "target-endian": "big",
-    "target-pointer-width": "64",
+    "target-pointer-width": 64,
     "arch": "x86_64",
     "os": "linux"
 }
diff --git a/tests/run-make/target-specs/mismatching-data-layout.json b/tests/run-make/target-specs/mismatching-data-layout.json
index d12caaad14a0..e948d4d2f99f 100644
--- a/tests/run-make/target-specs/mismatching-data-layout.json
+++ b/tests/run-make/target-specs/mismatching-data-layout.json
@@ -2,5 +2,5 @@
   "arch": "x86_64",
   "data-layout": "e-m:e-i64:16:32:64",
   "llvm-target": "x86_64-unknown-unknown-gnu",
-  "target-pointer-width": "64"
+  "target-pointer-width": 64
 }
diff --git a/tests/run-make/target-specs/my-awesome-platform.json b/tests/run-make/target-specs/my-awesome-platform.json
index d41038b84a86..732a6bacd157 100644
--- a/tests/run-make/target-specs/my-awesome-platform.json
+++ b/tests/run-make/target-specs/my-awesome-platform.json
@@ -3,7 +3,7 @@
     "linker-flavor": "gcc",
     "llvm-target": "i686-unknown-linux-gnu",
     "target-endian": "little",
-    "target-pointer-width": "32",
+    "target-pointer-width": 32,
     "arch": "x86",
     "os": "linux"
 }
diff --git a/tests/run-make/target-specs/my-incomplete-platform.json b/tests/run-make/target-specs/my-incomplete-platform.json
index 8bdc4108f494..68dbfc62d1c2 100644
--- a/tests/run-make/target-specs/my-incomplete-platform.json
+++ b/tests/run-make/target-specs/my-incomplete-platform.json
@@ -2,7 +2,7 @@
     "data-layout": "e-p:32:32-f64:32:64-i64:32:64-f80:32:32-n8:16:32",
     "linker-flavor": "gcc",
     "target-endian": "little",
-    "target-pointer-width": "32",
+    "target-pointer-width": 32,
     "arch": "x86",
     "os": "foo"
 }
diff --git a/tests/run-make/target-specs/my-x86_64-unknown-linux-gnu-platform.json b/tests/run-make/target-specs/my-x86_64-unknown-linux-gnu-platform.json
index 27833f1abdd9..008ea5483831 100644
--- a/tests/run-make/target-specs/my-x86_64-unknown-linux-gnu-platform.json
+++ b/tests/run-make/target-specs/my-x86_64-unknown-linux-gnu-platform.json
@@ -4,7 +4,7 @@
     "linker-flavor": "gcc",
     "llvm-target": "x86_64-unknown-linux-gnu",
     "target-endian": "little",
-    "target-pointer-width": "64",
+    "target-pointer-width": 64,
     "arch": "x86_64",
     "os": "linux"
 }
diff --git a/tests/run-make/target-specs/require-explicit-cpu.json b/tests/run-make/target-specs/require-explicit-cpu.json
index 9744bca168e2..4f23b644d8cf 100644
--- a/tests/run-make/target-specs/require-explicit-cpu.json
+++ b/tests/run-make/target-specs/require-explicit-cpu.json
@@ -3,7 +3,7 @@
     "linker-flavor": "gcc",
     "llvm-target": "i686-unknown-linux-gnu",
     "target-endian": "little",
-    "target-pointer-width": "32",
+    "target-pointer-width": 32,
     "arch": "x86",
     "os": "linux",
     "need-explicit-cpu": true
diff --git a/tests/run-make/thumb-none-cortex-m/rmake.rs b/tests/run-make/thumb-none-cortex-m/rmake.rs
deleted file mode 100644
index 27afef874da6..000000000000
--- a/tests/run-make/thumb-none-cortex-m/rmake.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-//! Test building of the `cortex-m` crate, a foundational crate in the embedded ecosystem
-//! for a collection of thumb targets. This is a smoke test that verifies that both cargo
-//! and rustc work in this case.
-//!
-//! How to run this
-//! $ ./x.py clean
-//! $ ./x.py test --target thumbv6m-none-eabi,thumbv7m-none-eabi tests/run-make
-//!
-//! Supported targets:
-//! - thumbv6m-none-eabi (Bare Cortex-M0, M0+, M1)
-//! - thumbv7em-none-eabi (Bare Cortex-M4, M7)
-//! - thumbv7em-none-eabihf (Bare Cortex-M4F, M7F, FPU, hardfloat)
-//! - thumbv7m-none-eabi (Bare Cortex-M3)
-
-//@ only-thumb
-
-use run_make_support::{cargo, cmd, env, env_var, target};
-
-const CRATE: &str = "cortex-m";
-const CRATE_URL: &str = "https://github.com/rust-embedded/cortex-m";
-const CRATE_SHA1: &str = "a448e9156e2cb1e556e5441fd65426952ef4b927"; // v0.5.0
-
-fn main() {
-    // FIXME: requires an internet connection https://github.com/rust-lang/rust/issues/128733
-    // See below link for git usage:
-    // https://stackoverflow.com/questions/3489173#14091182
-    cmd("git").args(["clone", CRATE_URL, CRATE]).run();
-    env::set_current_dir(CRATE);
-    cmd("git").args(["reset", "--hard", CRATE_SHA1]).run();
-
-    cargo()
-        .args(&[
-            "build",
-            "--manifest-path",
-            "Cargo.toml",
-            "-Zbuild-std=core",
-            "--target",
-            &target(),
-        ])
-        .env("CARGO_TARGET_DIR", "target")
-        // Don't make lints fatal, but they need to at least warn
-        // or they break Cargo's target info parsing.
-        .env("RUSTFLAGS", "-Copt-level=0 -Cdebug-assertions=yes --cap-lints=warn")
-        .run();
-}
diff --git a/tests/run-make/wasm-panic-small/rmake.rs b/tests/run-make/wasm-panic-small/rmake.rs
index e69fbac96356..ea0b6faf037c 100644
--- a/tests/run-make/wasm-panic-small/rmake.rs
+++ b/tests/run-make/wasm-panic-small/rmake.rs
@@ -24,5 +24,5 @@ fn test(cfg: &str) {
 
     let bytes = rfs::read("foo.wasm");
     println!("{}", bytes.len());
-    assert!(bytes.len() < 40_000);
+    assert!(bytes.len() < 40_000, "bytes len was: {}", bytes.len());
 }
diff --git a/tests/run-make/wasm-unexpected-features/foo.rs b/tests/run-make/wasm-unexpected-features/foo.rs
new file mode 100644
index 000000000000..5c7aa2d61904
--- /dev/null
+++ b/tests/run-make/wasm-unexpected-features/foo.rs
@@ -0,0 +1,43 @@
+#![no_core]
+#![crate_type = "cdylib"]
+#![feature(no_core, lang_items, allocator_internals, rustc_attrs)]
+#![needs_allocator]
+#![allow(internal_features)]
+
+#[rustc_std_internal_symbol]
+unsafe fn __rust_alloc(_size: usize, _align: usize) -> *mut u8 {
+    0 as *mut u8
+}
+
+unsafe extern "Rust" {
+    #[rustc_std_internal_symbol]
+    fn __rust_alloc_error_handler(size: usize, align: usize) -> !;
+}
+
+#[used]
+static mut BUF: [u8; 1024] = [0; 1024];
+
+#[unsafe(no_mangle)]
+extern "C" fn init() {
+    unsafe {
+        __rust_alloc_error_handler(0, 0);
+    }
+}
+
+mod minicore {
+    #[lang = "pointee_sized"]
+    pub trait PointeeSized {}
+
+    #[lang = "meta_sized"]
+    pub trait MetaSized: PointeeSized {}
+
+    #[lang = "sized"]
+    pub trait Sized: MetaSized {}
+
+    #[lang = "copy"]
+    pub trait Copy {}
+    impl Copy for u8 {}
+
+    #[lang = "drop_in_place"]
+    fn drop_in_place(_: *mut T) {}
+}
diff --git a/tests/run-make/wasm-unexpected-features/rmake.rs b/tests/run-make/wasm-unexpected-features/rmake.rs
new file mode 100644
index 000000000000..01eff54e823c
--- /dev/null
+++ b/tests/run-make/wasm-unexpected-features/rmake.rs
@@ -0,0 +1,26 @@
+//@ only-wasm32-wasip1
+
+use std::path::Path;
+
+use run_make_support::{rfs, rustc, wasmparser};
+
+fn main() {
+    rustc()
+        .input("foo.rs")
+        .target("wasm32-wasip1")
+        .target_cpu("mvp")
+        .opt_level("z")
+        .lto("fat")
+        .linker_plugin_lto("on")
+        .link_arg("--import-memory")
+        .run();
+    verify_features(Path::new("foo.wasm"));
+}
+
+fn verify_features(path: &Path) {
+    eprintln!("verify {path:?}");
+    let file = rfs::read(&path);
+
+    let mut validator = wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::MVP);
+    validator.validate_all(&file).unwrap();
+}
diff --git a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/rmake.rs b/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/rmake.rs
deleted file mode 100644
index e58762aeb6db..000000000000
--- a/tests/run-make/x86_64-fortanix-unknown-sgx-lvi/rmake.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-// ignore-tidy-linelength
-// Reason: intel.com link
-
-// This security test checks that the disassembled form of certain symbols
-// is "hardened" - that means, the assembly instructions match a pattern that
-// mitigate potential Load Value Injection vulnerabilities.
-// To do so, a test crate is compiled, and certain symbols are found, disassembled
-// and checked one by one.
-// See https://github.com/rust-lang/rust/pull/77008
-
-// On load value injection:
-// https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/technical-documentation/load-value-injection.html
-
-//@ only-x86_64-fortanix-unknown-sgx
-
-use run_make_support::{cmd, cwd, llvm_filecheck, llvm_objdump, regex, set_current_dir, target};
-
-fn main() {
-    let main_dir = cwd();
-    set_current_dir("enclave");
-    // HACK(eddyb) sets `RUSTC_BOOTSTRAP=1` so Cargo can accept nightly features.
-    // These come from the top-level Rust workspace, that this crate is not a
-    // member of, but Cargo tries to load the workspace `Cargo.toml` anyway.
-    cmd("cargo")
-        .env("RUSTC_BOOTSTRAP", "1")
-        .arg("-v")
-        .arg("run")
-        .arg("--target")
-        .arg(target())
-        .run();
-    set_current_dir(&main_dir);
-    // Rust has various ways of adding code to a binary:
-    // - Rust code
-    // - Inline assembly
-    // - Global assembly
-    // - C/C++ code compiled as part of Rust crates
-    // For those different kinds, we do have very small code examples that should be
-    // mitigated in some way. Mostly we check that ret instructions should no longer be present.
-    check("unw_getcontext", "unw_getcontext.checks");
-    check("__libunwind_Registers_x86_64_jumpto", "jumpto.checks");
-
-    check("std::io::stdio::_print::[[:alnum:]]+", "print.with_frame_pointers.checks");
-
-    check("rust_plus_one_global_asm", "rust_plus_one_global_asm.checks");
-
-    check("cc_plus_one_c", "cc_plus_one_c.checks");
-    check("cc_plus_one_c_asm", "cc_plus_one_c_asm.checks");
-    check("cc_plus_one_cxx", "cc_plus_one_cxx.checks");
-    check("cc_plus_one_cxx_asm", "cc_plus_one_cxx_asm.checks");
-    check("cc_plus_one_asm", "cc_plus_one_asm.checks");
-
-    check("cmake_plus_one_c", "cmake_plus_one_c.checks");
-    check("cmake_plus_one_c_asm", "cmake_plus_one_c_asm.checks");
-    check("cmake_plus_one_c_global_asm", "cmake_plus_one_c_global_asm.checks");
-    check("cmake_plus_one_cxx", "cmake_plus_one_cxx.checks");
-    check("cmake_plus_one_cxx_asm", "cmake_plus_one_cxx_asm.checks");
-    check("cmake_plus_one_cxx_global_asm", "cmake_plus_one_cxx_global_asm.checks");
-    check("cmake_plus_one_asm", "cmake_plus_one_asm.checks");
-}
-
-fn check(func_re: &str, mut checks: &str) {
-    let dump = llvm_objdump()
-        .input("enclave/target/x86_64-fortanix-unknown-sgx/debug/enclave")
-        .args(&["--syms", "--demangle"])
-        .run()
-        .stdout_utf8();
-    let re = regex::Regex::new(&format!("[[:blank:]]+{func_re}")).unwrap();
-    let func = re.find_iter(&dump).map(|m| m.as_str().trim()).collect::>().join(",");
-    assert!(!func.is_empty());
-    let dump = llvm_objdump()
-        .input("enclave/target/x86_64-fortanix-unknown-sgx/debug/enclave")
-        .args(&["--demangle", &format!("--disassemble-symbols={func}")])
-        .run()
-        .stdout_utf8();
-    let dump = dump.as_bytes();
-
-    // Unique case, must succeed at one of two possible tests.
-    // This is because frame pointers are optional, and them being enabled requires
-    // an additional `popq` in the pattern checking file.
-    if func_re == "std::io::stdio::_print::[[:alnum:]]+" {
-        let output = llvm_filecheck().stdin_buf(&dump).patterns(checks).run_unchecked();
-        if !output.status().success() {
-            checks = "print.without_frame_pointers.checks";
-            llvm_filecheck().stdin_buf(&dump).patterns(checks).run();
-        }
-    } else {
-        llvm_filecheck().stdin_buf(&dump).patterns(checks).run();
-    }
-    if !["rust_plus_one_global_asm", "cmake_plus_one_c_global_asm", "cmake_plus_one_cxx_global_asm"]
-        .contains(&func_re)
-    {
-        // The assembler cannot avoid explicit `ret` instructions. Sequences
-        // of `shlq $0x0, (%rsp); lfence; retq` are used instead.
-        llvm_filecheck()
-            .args(&["--implicit-check-not", "ret"])
-            .stdin_buf(dump)
-            .patterns(checks)
-            .run();
-    }
-}
diff --git a/tests/rustdoc-gui/code-example-buttons.goml b/tests/rustdoc-gui/code-example-buttons.goml
index b96f6ddcc37a..1429f978a28a 100644
--- a/tests/rustdoc-gui/code-example-buttons.goml
+++ b/tests/rustdoc-gui/code-example-buttons.goml
@@ -5,25 +5,25 @@ include: "utils.goml"
 // First we check we "hover".
 move-cursor-to: ".example-wrap"
 assert-css: (".example-wrap .copy-button", { "visibility": "visible" })
-move-cursor-to: ".search-input"
+move-cursor-to: "#search-button"
 assert-css: (".example-wrap .copy-button", { "visibility": "hidden" })
 
 // Now we check the click.
 assert-count: (".example-wrap:not(:hover) .button-holder.keep-visible", 0)
 click: ".example-wrap"
-move-cursor-to: ".search-input"
+move-cursor-to: "#search-button"
 // It should have a new class and be visible.
 wait-for-count: (".example-wrap:not(:hover) .button-holder.keep-visible", 1)
 wait-for-css: (".example-wrap:not(:hover) .button-holder.keep-visible", { "visibility": "visible" })
 // Clicking again will remove the class.
 click: ".example-wrap"
-move-cursor-to: ".search-input"
+move-cursor-to: "rustdoc-toolbar #search-button"
 assert-count: (".example-wrap:not(:hover) .button-holder.keep-visible", 0)
 assert-css: (".example-wrap .copy-button", { "visibility": "hidden" })
 
 // Clicking on the "copy code" button shouldn't make the buttons stick.
 click: ".example-wrap .copy-button"
-move-cursor-to: ".search-input"
+move-cursor-to: "#search-button"
 assert-count: (".example-wrap:not(:hover) .button-holder.keep-visible", 0)
 assert-css: (".example-wrap .copy-button", { "visibility": "hidden" })
 // Since we clicked on the copy button, the clipboard content should have been updated.
diff --git a/tests/rustdoc-gui/copy-code.goml b/tests/rustdoc-gui/copy-code.goml
index 9cc717bc67a3..a6fb816c4bd6 100644
--- a/tests/rustdoc-gui/copy-code.goml
+++ b/tests/rustdoc-gui/copy-code.goml
@@ -12,7 +12,7 @@ define-function: (
         assert-count: (".example-wrap .copy-button", 1)
         // We now ensure it's only displayed when the example is hovered.
         assert-css: (".example-wrap .copy-button", { "visibility": "visible" })
-        move-cursor-to: ".search-input"
+        move-cursor-to: "rustdoc-toolbar #search-button"
         assert-css: (".example-wrap .copy-button", { "visibility": "hidden" })
         // Checking that the copy button has the same size as the "copy path" button.
         compare-elements-size: (
diff --git a/tests/rustdoc-gui/cursor.goml b/tests/rustdoc-gui/cursor.goml
index 9412987fc323..0d78e192606b 100644
--- a/tests/rustdoc-gui/cursor.goml
+++ b/tests/rustdoc-gui/cursor.goml
@@ -1,4 +1,5 @@
 // This test ensures that several clickable items actually have the pointer cursor.
+include: "utils.goml"
 go-to: "file://" + |DOC_PATH| + "/lib2/struct.Foo.html"
 
 // the `[+]/[-]` button
@@ -8,11 +9,7 @@ assert-css: ("#toggle-all-docs", {"cursor": "pointer"})
 assert-css: ("#copy-path", {"cursor": "pointer"})
 
 // the search tabs
-write-into: (".search-input", "Foo")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "Foo"})
 assert-css: ("#search-tabs > button", {"cursor": "pointer"})
 
 // mobile sidebar toggle button
diff --git a/tests/rustdoc-gui/docblock-code-block-line-number.goml b/tests/rustdoc-gui/docblock-code-block-line-number.goml
index 0df9cc2a6598..a182124aced2 100644
--- a/tests/rustdoc-gui/docblock-code-block-line-number.goml
+++ b/tests/rustdoc-gui/docblock-code-block-line-number.goml
@@ -69,7 +69,7 @@ call-function: ("check-colors", {
 // and make sure it goes away.
 
 // First, open the settings menu.
-click: "#settings-menu"
+click: "rustdoc-toolbar .settings-menu"
 wait-for: "#settings"
 assert-css: ("#settings", {"display": "block"})
 
@@ -121,7 +121,7 @@ call-function: ("check-padding", {
 define-function: ("check-line-numbers-existence", [], block {
     assert-local-storage: {"rustdoc-line-numbers": "true" }
     assert-false: ".example-line-numbers"
-    click: "#settings-menu"
+    click: "rustdoc-toolbar .settings-menu"
     wait-for: "#settings"
 
     // Then, click the toggle button.
@@ -137,7 +137,7 @@ define-function: ("check-line-numbers-existence", [], block {
     // Line numbers should still be there.
     assert-css: ("[data-nosnippet]", { "display": "block"})
     // Closing settings menu.
-    click: "#settings-menu"
+    click: "rustdoc-toolbar .settings-menu"
     wait-for-css: ("#settings", {"display": "none"})
 })
 
@@ -168,7 +168,7 @@ assert: ".example-wrap > pre.rust"
 assert-count: (".example-wrap", 2)
 assert-count: (".example-wrap.digits-1", 2)
 
-click: "#settings-menu"
+click: "rustdoc-toolbar .settings-menu"
 wait-for: "#settings"
 
 // Then, click the toggle button.
diff --git a/tests/rustdoc-gui/escape-key.goml b/tests/rustdoc-gui/escape-key.goml
index ff8557b9b81c..ab5615ebcd80 100644
--- a/tests/rustdoc-gui/escape-key.goml
+++ b/tests/rustdoc-gui/escape-key.goml
@@ -1,13 +1,10 @@
 // This test ensures that the "Escape" shortcut is handled correctly based on the
 // current content displayed.
+include: "utils.goml"
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
 // First, we check that the search results are hidden when the Escape key is pressed.
-write-into: (".search-input", "test")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-wait-for: "#search h1" // The search element is empty before the first search
+call-function: ("perform-search", {"query": "test"})
 // Check that the currently displayed element is search.
-wait-for: "#alternative-display #search"
 assert-attribute: ("#main-content", {"class": "content hidden"})
 assert-document-property: ({"URL": "index.html?search=test"}, ENDS_WITH)
 press-key: "Escape"
@@ -17,8 +14,8 @@ assert-false: "#alternative-display #search"
 assert-attribute: ("#main-content", {"class": "content"})
 assert-document-property: ({"URL": "index.html"}, [ENDS_WITH])
 
-// Check that focusing the search input brings back the search results
-focus: ".search-input"
+// Check that clicking the search button brings back the search results
+click: "#search-button"
 wait-for: "#alternative-display #search"
 assert-attribute: ("#main-content", {"class": "content hidden"})
 assert-document-property: ({"URL": "index.html?search=test"}, ENDS_WITH)
diff --git a/tests/rustdoc-gui/font-serif-change.goml b/tests/rustdoc-gui/font-serif-change.goml
index b14d5ae96f92..1e9f21c35416 100644
--- a/tests/rustdoc-gui/font-serif-change.goml
+++ b/tests/rustdoc-gui/font-serif-change.goml
@@ -8,7 +8,7 @@ assert-css: ("body", {"font-family": |serif_font|})
 assert-css: ("p code", {"font-family": |serif_code_font|})
 
 // We now switch to the sans serif font
-click: "#settings-menu"
+click: "main .settings-menu"
 wait-for: "#sans-serif-fonts"
 click: "#sans-serif-fonts"
 
@@ -23,7 +23,7 @@ assert-css: ("body", {"font-family": |font|})
 assert-css: ("p code", {"font-family": |code_font|})
 
 // We switch back to the serif font
-click: "#settings-menu"
+click: "main .settings-menu"
 wait-for: "#sans-serif-fonts"
 click: "#sans-serif-fonts"
 
diff --git a/tests/rustdoc-gui/globals.goml b/tests/rustdoc-gui/globals.goml
index 7a0e2b9eb746..89f57add8161 100644
--- a/tests/rustdoc-gui/globals.goml
+++ b/tests/rustdoc-gui/globals.goml
@@ -1,6 +1,7 @@
 // Make sure search stores its data in `window`
 // It needs to use a global to avoid racing on search-index.js and search.js
 // https://github.com/rust-lang/rust/pull/118961
+include: "utils.goml"
 
 // URL query
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html?search=sa'%3Bda'%3Bds"
@@ -9,9 +10,7 @@ assert-window-property-false: {"searchIndex": null}
 
 // Form input
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-write-into: (".search-input", "Foo")
-press-key: 'Enter'
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "Foo"})
 assert-window-property-false: {"searchIndex": null}
 
 // source sidebar
diff --git a/tests/rustdoc-gui/help-page.goml b/tests/rustdoc-gui/help-page.goml
index 6d6e353ae362..34b408140276 100644
--- a/tests/rustdoc-gui/help-page.goml
+++ b/tests/rustdoc-gui/help-page.goml
@@ -6,12 +6,12 @@ assert-css: ("#help", {"display": "block"})
 assert-css: ("#help dd", {"font-size": "16px"})
 assert-false: "#help-button > a"
 assert-css: ("#help", {"display": "block"})
-compare-elements-property: (".sub", "#help", ["offsetWidth"])
-compare-elements-position: (".sub", "#help", ["x"])
+compare-elements-property: (".main-heading", "#help", ["offsetWidth"])
+compare-elements-position: (".main-heading", "#help", ["x"])
 set-window-size: (500, 1000) // Try mobile next.
 assert-css: ("#help", {"display": "block"})
-compare-elements-property: (".sub", "#help", ["offsetWidth"])
-compare-elements-position: (".sub", "#help", ["x"])
+compare-elements-property: (".main-heading", "#help", ["offsetWidth"])
+compare-elements-position: (".main-heading", "#help", ["x"])
 
 // Checking the color of the elements of the help menu.
 show-text: true
@@ -54,19 +54,17 @@ go-to: "file://" + |DOC_PATH| + "/test_docs/index.html?search=a"
 wait-for: "#search-tabs" // Waiting for the search.js to load.
 set-window-size: (1000, 1000) // Only supported on desktop.
 assert-false: "#help"
-click: "#help-button > a"
+click: "rustdoc-toolbar .help-menu > a"
 assert-css: ("#help", {"display": "block"})
 assert-css: ("#help dd", {"font-size": "16px"})
-click: "#help-button > a"
-assert-css: ("#help", {"display": "none"})
-compare-elements-property-false: (".sub", "#help", ["offsetWidth"])
-compare-elements-position-false: (".sub", "#help", ["x"])
+click: "rustdoc-toolbar .help-menu > a"
+assert-false: "#help"
 
 // This test ensures that the "the rustdoc book" anchor link within the help popover works.
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html?search=a"
 wait-for: "#search-tabs" // Waiting for the search.js to load.
 set-window-size: (1000, 1000) // Popover only appears when the screen width is >700px.
 assert-false: "#help"
-click: "#help-button > a"
+click: "rustdoc-toolbar .help-menu > a"
 click: "//*[@id='help']//a[text()='the rustdoc book']"
 wait-for-document-property: ({"URL": "https://doc.rust-lang.org/"}, STARTS_WITH)
diff --git a/tests/rustdoc-gui/hide-mobile-topbar.goml b/tests/rustdoc-gui/hide-mobile-topbar.goml
index 46eb8acfe8cd..1e46d2358279 100644
--- a/tests/rustdoc-gui/hide-mobile-topbar.goml
+++ b/tests/rustdoc-gui/hide-mobile-topbar.goml
@@ -1,20 +1,19 @@
 // Checks sidebar resizing stays synced with the setting
-go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
+go-to: "file://" + |DOC_PATH| + "/settings.html"
 set-window-size: (400, 600)
 
 // Verify that the "hide" option is unchecked
-click: "#settings-menu"
 wait-for: "#settings"
 assert-css: ("#settings", {"display": "block"})
 assert-property: ("#hide-sidebar", {"checked": "false"})
-assert-css: (".mobile-topbar", {"display": "flex"})
+assert-css: ("rustdoc-topbar", {"display": "flex"})
 
 // Toggle it
 click: "#hide-sidebar"
 assert-property: ("#hide-sidebar", {"checked": "true"})
-assert-css: (".mobile-topbar", {"display": "none"})
+assert-css: ("rustdoc-topbar", {"display": "none"})
 
 // Toggle it again
 click: "#hide-sidebar"
 assert-property: ("#hide-sidebar", {"checked": "false"})
-assert-css: (".mobile-topbar", {"display": "flex"})
+assert-css: ("rustdoc-topbar", {"display": "flex"})
diff --git a/tests/rustdoc-gui/huge-logo.goml b/tests/rustdoc-gui/huge-logo.goml
index d207ab5bb37c..6ad6948ef2ab 100644
--- a/tests/rustdoc-gui/huge-logo.goml
+++ b/tests/rustdoc-gui/huge-logo.goml
@@ -8,8 +8,3 @@ assert-property: (".sidebar-crate .logo-container", {"offsetWidth": "96", "offse
 // offsetWidth = width of sidebar, offsetHeight = height + top padding
 assert-property: (".sidebar-crate .logo-container img", {"offsetWidth": "48", "offsetHeight": 64})
 assert-css: (".sidebar-crate .logo-container img", {"border-top-width": "16px", "margin-top": "-16px"})
-
-set-window-size: (400, 600)
-// offset = size + margin
-assert-property: (".mobile-topbar .logo-container", {"offsetWidth": "55", "offsetHeight": 45})
-assert-property: (".mobile-topbar .logo-container img", {"offsetWidth": "35", "offsetHeight": 35})
diff --git a/tests/rustdoc-gui/item-info.goml b/tests/rustdoc-gui/item-info.goml
index 647a2fd290de..11388c79e0b8 100644
--- a/tests/rustdoc-gui/item-info.goml
+++ b/tests/rustdoc-gui/item-info.goml
@@ -20,7 +20,7 @@ store-position: (
     {"x": second_line_x, "y": second_line_y},
 )
 assert: |first_line_x| != |second_line_x| && |first_line_x| == 521 && |second_line_x| == 277
-assert: |first_line_y| != |second_line_y| && |first_line_y| == 718 && |second_line_y| == 741
+assert: |first_line_y| != |second_line_y| && |first_line_y| == 676 && |second_line_y| == 699
 
 // Now we ensure that they're not rendered on the same line.
 set-window-size: (1100, 800)
diff --git a/tests/rustdoc-gui/links-color.goml b/tests/rustdoc-gui/links-color.goml
index f11920cdd8c2..a363175c1ddb 100644
--- a/tests/rustdoc-gui/links-color.goml
+++ b/tests/rustdoc-gui/links-color.goml
@@ -9,7 +9,7 @@ show-text: true
 define-function: (
     "check-colors",
     [theme, mod, macro, struct, enum, trait, fn, type, union, keyword,
-     sidebar, sidebar_current, sidebar_current_background],
+     attribute, sidebar, sidebar_current, sidebar_current_background],
     block {
         call-function: ("switch-theme", {"theme": |theme|})
         // Checking results colors.
@@ -22,6 +22,7 @@ define-function: (
         assert-css: (".item-table .type", {"color": |type|}, ALL)
         assert-css: (".item-table .union", {"color": |union|}, ALL)
         assert-css: (".item-table .keyword", {"color": |keyword|}, ALL)
+        assert-css: (".item-table .attribute", {"color": |attribute|}, ALL)
         // Checking sidebar elements.
         assert-css: (
             ".sidebar-elems li:not(.current) a",
@@ -58,6 +59,7 @@ call-function: (
         "type": "#ffa0a5",
         "union": "#ffa0a5",
         "keyword": "#39afd7",
+        "attribute": "#39afd7",
         "sidebar": "#53b1db",
         "sidebar_current": "#ffb44c",
         "sidebar_current_background": "transparent",
@@ -76,6 +78,7 @@ call-function: (
         "type": "#2dbfb8",
         "union": "#2dbfb8",
         "keyword": "#d2991d",
+        "attribute": "#d2991d",
         "sidebar": "#fdbf35",
         "sidebar_current": "#fdbf35",
         "sidebar_current_background": "#444",
@@ -94,6 +97,7 @@ call-function: (
         "type": "#ad378a",
         "union": "#ad378a",
         "keyword": "#3873ad",
+        "attribute": "#3873ad",
         "sidebar": "#356da4",
         "sidebar_current": "#356da4",
         "sidebar_current_background": "#fff",
diff --git a/tests/rustdoc-gui/macro-expansion.goml b/tests/rustdoc-gui/macro-expansion.goml
new file mode 100644
index 000000000000..b87d0e4870af
--- /dev/null
+++ b/tests/rustdoc-gui/macro-expansion.goml
@@ -0,0 +1,126 @@
+// This test ensures that the macro expansion is generated and working as expected.
+go-to: "file://" + |DOC_PATH| + "/src/macro_expansion/lib.rs.html"
+
+define-function: (
+    "check-expansion",
+    [line, original_content],
+    block {
+        assert-text: ("a[id='" + |line| + "'] + .expansion .original", |original_content|)
+        // The "original" content should be expanded.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .original", {"display": "inline"})
+        // The expanded macro should be hidden.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .expanded", {"display": "none"})
+
+        // We "expand" the macro.
+        click: "a[id='" + |line| + "'] + .expansion input[type=checkbox]"
+        // The "original" content is hidden.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .original", {"display": "none"})
+        // The expanded macro is visible.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .expanded", {"display": "inline"})
+
+        // We collapse the macro.
+        click: "a[id='" + |line| + "'] + .expansion input[type=checkbox]"
+        // The "original" content is expanded.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .original", {"display": "inline"})
+        // The expanded macro is hidden.
+        assert-css: ("a[id='" + |line| + "'] + .expansion .expanded", {"display": "none"})
+    }
+)
+
+// First we check the derive macro expansion at line 33.
+call-function: ("check-expansion", {"line": 35, "original_content": "Debug"})
+// Then we check the `bar` macro expansion at line 41.
+call-function: ("check-expansion", {"line": 43, "original_content": "bar!(y)"})
+// Then we check the `println` macro expansion at line 42-44.
+call-function: ("check-expansion", {"line": 44, "original_content": 'println!("
+45    {y}
+46    ")'})
+
+// Then finally we check when there are two macro calls on a same line.
+assert-count: ("#expand-52 ~ .original", 2)
+assert-count: ("#expand-52 ~ .expanded", 2)
+
+store-value: (repeat_o, '/following-sibling::*[@class="original"]')
+store-value: (repeat_e, '/following-sibling::*[@class="expanded"]')
+store-value: (expand_id, "expand-52")
+assert-text: ('//*[@id="' + |expand_id| + '"]' + |repeat_o|, "stringify!(foo)")
+assert-text: ('//*[@id="' + |expand_id| + '"]' + |repeat_o| + |repeat_o|, "stringify!(bar)")
+assert-text: ('//*[@id="' + |expand_id| + '"]' + |repeat_e|, '"foo"')
+assert-text: ('//*[@id="' + |expand_id| + '"]' + |repeat_e| + |repeat_e|, '"bar"')
+
+// The "original" content should be expanded.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o|, {"display": "inline"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o| + |repeat_o|, {"display": "inline"})
+// The expanded macro should be hidden.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e|, {"display": "none"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e| + |repeat_e|, {"display": "none"})
+
+// We "expand" the macro (because the line starts with a string, the label is not at the "top
+// level" of the ``, so we need to use a different selector).
+click: "#" + |expand_id|
+// The "original" content is hidden.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o|, {"display": "none"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o| + |repeat_o|, {"display": "none"})
+// The expanded macro is visible.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e|, {"display": "inline"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e| + |repeat_e|, {"display": "inline"})
+
+// We collapse the macro.
+click: "#" + |expand_id|
+// The "original" content is expanded.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o|, {"display": "inline"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_o| + |repeat_o|, {"display": "inline"})
+// The expanded macro is hidden.
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e|, {"display": "none"})
+assert-css: ('//*[@id="' + |expand_id| + '"]' + |repeat_e| + |repeat_e|, {"display": "none"})
+
+// Checking the line 48 `println` which needs to be handled differently because the line number is
+// inside a "comment" span.
+store-value: (expand_id, "expand-48")
+assert-text: ("#" + |expand_id| + " ~ .original", 'println!("
+49    {y}
+50    ")')
+// The "original" content should be expanded.
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "inline"})
+// The expanded macro should be hidden.
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "none"})
+
+// We "expand" the macro.
+click: "#" + |expand_id|
+// The "original" content is hidden.
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "none"})
+// The expanded macro is visible.
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "inline"})
+
+// We collapse the macro.
+click: "#" + |expand_id|
+// The "original" content is expanded.
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "inline"})
+// The expanded macro is hidden.
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "none"})
+
+// Ensure that the toggles are focusable and can be interacted with keyboard.
+focus: "//a[@id='29']"
+press-key: "Tab"
+store-value: (expand_id, "expand-29")
+assert: "#" + |expand_id| + ":focus"
+assert-css: ("#" + |expand_id| +" ~ .expanded", {"display": "none"})
+assert-css: ("#" + |expand_id| +" ~ .original", {"display": "inline"})
+// We now expand the macro.
+press-key: "Space"
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "inline"})
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "none"})
+// We collapse the macro.
+press-key: "Space"
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "none"})
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "inline"})
+
+// Now we check a macro coming from another file.
+store-value: (expand_id, "expand-55")
+// We "expand" the macro.
+click: "#" + |expand_id|
+// The "original" content is hidden.
+assert-css: ("#" + |expand_id| + " ~ .original", {"display": "none"})
+// The expanded macro is visible.
+assert-css: ("#" + |expand_id| + " ~ .expanded", {"display": "inline"})
+assert-text: ("#" + |expand_id| + " ~ .expanded", "{ y += 2; };")
diff --git a/tests/rustdoc-gui/mobile-crate-name.goml b/tests/rustdoc-gui/mobile-crate-name.goml
index a0c96eec8a5a..524c1d36a8a8 100644
--- a/tests/rustdoc-gui/mobile-crate-name.goml
+++ b/tests/rustdoc-gui/mobile-crate-name.goml
@@ -5,18 +5,18 @@ go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
 // First we change the title to make it big.
 set-window-size: (350, 800)
 // We ensure that the "format" of the title is the same as the one we'll use.
-assert-text: (".mobile-topbar .location a", "test_docs")
+assert-text: ("rustdoc-topbar h2 a", "Crate test_docs")
 // We store the height we know is correct.
-store-property: (".mobile-topbar .location", {"offsetHeight": height})
+store-property: ("rustdoc-topbar h2", {"offsetHeight": height})
 // We change the crate name to something longer.
-set-text: (".mobile-topbar .location a", "cargo_packager_resource_resolver")
+set-text: ("rustdoc-topbar h2 a", "cargo_packager_resource_resolver")
 // And we check that the size remained the same.
-assert-property: (".mobile-topbar .location", {"offsetHeight": |height|})
+assert-property: ("rustdoc-topbar h2", {"offsetHeight": |height|})
 
 // Now we check if it works for the non-crate pages as well.
 go-to: "file://" + |DOC_PATH| + "/test_docs/struct.Foo.html"
 // We store the height we know is correct.
-store-property: (".mobile-topbar .location", {"offsetHeight": height})
-set-text: (".mobile-topbar .location a", "Something_incredibly_long_because")
+store-property: ("rustdoc-topbar h2", {"offsetHeight": height})
+set-text: ("rustdoc-topbar h2 a", "Something_incredibly_long_because")
 // And we check that the size remained the same.
-assert-property: (".mobile-topbar .location", {"offsetHeight": |height|})
+assert-property: ("rustdoc-topbar h2", {"offsetHeight": |height|})
diff --git a/tests/rustdoc-gui/mobile.goml b/tests/rustdoc-gui/mobile.goml
index a9eee53dd1d5..d292281932d7 100644
--- a/tests/rustdoc-gui/mobile.goml
+++ b/tests/rustdoc-gui/mobile.goml
@@ -5,7 +5,7 @@ set-window-size: (400, 600)
 set-font-size: 18
 wait-for: 100 // wait a bit for the resize and the font-size change to be fully taken into account.
 
-assert-property: (".mobile-topbar h2", {"offsetHeight": 33})
+assert-property: ("rustdoc-topbar h2", {"offsetHeight": 33})
 
 // On the settings page, the theme buttons should not line-wrap. Instead, they should
 // all be placed as a group on a line below the setting name "Theme."
diff --git a/tests/rustdoc-gui/module-items-font.goml b/tests/rustdoc-gui/module-items-font.goml
index 0e6dd81c05b8..bed95b378c6a 100644
--- a/tests/rustdoc-gui/module-items-font.goml
+++ b/tests/rustdoc-gui/module-items-font.goml
@@ -65,3 +65,12 @@ assert-css: (
     "#keywords + .item-table dd",
     {"font-family": '"Source Serif 4", NanumBarunGothic, serif'},
 )
+// attributes
+assert-css: (
+    "#attributes + .item-table dt a",
+    {"font-family": '"Fira Sans", Arial, NanumBarunGothic, sans-serif'},
+)
+assert-css: (
+    "#attributes + .item-table dd",
+    {"font-family": '"Source Serif 4", NanumBarunGothic, serif'},
+)
diff --git a/tests/rustdoc-gui/notable-trait.goml b/tests/rustdoc-gui/notable-trait.goml
index 7fc70e0675df..6bd4661ac8f4 100644
--- a/tests/rustdoc-gui/notable-trait.goml
+++ b/tests/rustdoc-gui/notable-trait.goml
@@ -8,10 +8,10 @@ define-function: (
     [x, i_x],
     block {
         // Checking they have the same y position.
-        compare-elements-position: (
+        compare-elements-position-near: (
             "//*[@id='method.create_an_iterator_from_read']//a[normalize-space()='NotableStructWithLongName']",
             "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']",
-            ["y"],
+            {"y": 1},
         )
         // Checking they don't have the same x position.
         compare-elements-position-false: (
@@ -82,15 +82,6 @@ call-function: ("check-notable-tooltip-position", {
     "i_x": 528,
 })
 
-// Checking on mobile now.
-set-window-size: (650, 600)
-wait-for-size: ("body", {"width": 650})
-call-function: ("check-notable-tooltip-position-complete", {
-    "x": 26,
-    "i_x": 305,
-    "popover_x": 0,
-})
-
 // Now check the colors.
 define-function: (
     "check-colors",
@@ -176,6 +167,15 @@ call-function: (
     },
 )
 
+// Checking on mobile now.
+set-window-size: (650, 600)
+wait-for-size: ("body", {"width": 650})
+call-function: ("check-notable-tooltip-position-complete", {
+    "x": 26,
+    "i_x": 305,
+    "popover_x": 0,
+})
+
 reload:
 
 // Check that pressing escape works
@@ -189,7 +189,7 @@ assert: "#method\.create_an_iterator_from_read .tooltip:focus"
 // Check that clicking outside works.
 click: "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']"
 assert-count: ("//*[@class='tooltip popover']", 1)
-click: ".search-input"
+click: ".main-heading h1"
 assert-count: ("//*[@class='tooltip popover']", 0)
 assert-false: "#method\.create_an_iterator_from_read .tooltip:focus"
 
@@ -219,14 +219,14 @@ define-function: (
         store-window-property: {"scrollY": scroll}
         click: "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']"
         wait-for: "//*[@class='tooltip popover']"
-        click: "#settings-menu a"
+        click: ".main-heading h1"
     }
 )
 
 // Now we check that the focus isn't given back to the wrong item when opening
 // another popover.
 call-function: ("setup-popup", {})
-click: ".search-input"
+click: ".main-heading h1"
 // We ensure we didn't come back to the previous focused item.
 assert-window-property-false: {"scrollY": |scroll|}
 
@@ -251,7 +251,7 @@ reload:
 assert-count: ("//*[@class='tooltip popover']", 0)
 click: "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']"
 assert-count: ("//*[@class='tooltip popover']", 1)
-click: "#settings-menu a"
+click: "rustdoc-toolbar .settings-menu a"
 wait-for: "#settings"
 assert-count: ("//*[@class='tooltip popover']", 0)
 assert-false: "#method\.create_an_iterator_from_read .tooltip:focus"
diff --git a/tests/rustdoc-gui/pocket-menu.goml b/tests/rustdoc-gui/pocket-menu.goml
index 073172dd8a79..a0815bfa9a09 100644
--- a/tests/rustdoc-gui/pocket-menu.goml
+++ b/tests/rustdoc-gui/pocket-menu.goml
@@ -3,33 +3,33 @@ include: "utils.goml"
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html?search=test"
 wait-for: "#crate-search"
 // First we check that the help menu doesn't exist yet.
-assert-false: "#help-button .popover"
+assert-false: "rustdoc-toolbar .help-menu .popover"
 // Then we display the help menu.
-click: "#help-button"
-assert: "#help-button .popover"
-assert-css: ("#help-button .popover", {"display": "block"})
+click: "rustdoc-toolbar .help-menu"
+assert: "rustdoc-toolbar .help-menu .popover"
+assert-css: ("rustdoc-toolbar .help-menu .popover", {"display": "block"})
 
 // Now we click somewhere else on the page to ensure it is handling the blur event
 // correctly.
 click: ".sidebar"
-assert-css: ("#help-button .popover", {"display": "none"})
+assert-false: "rustdoc-toolbar .help-menu .popover"
 
 // Now we will check that we cannot have two "pocket menus" displayed at the same time.
-click: "#help-button"
-assert-css: ("#help-button .popover", {"display": "block"})
-click: "#settings-menu"
-assert-css: ("#help-button .popover", {"display": "none"})
-assert-css: ("#settings-menu .popover", {"display": "block"})
+click: "rustdoc-toolbar .help-menu"
+assert-css: ("rustdoc-toolbar .help-menu .popover", {"display": "block"})
+click: "rustdoc-toolbar .settings-menu"
+assert-false: "rustdoc-toolbar .help-menu .popover"
+assert-css: ("rustdoc-toolbar .settings-menu .popover", {"display": "block"})
 
 // Now the other way.
-click: "#help-button"
-assert-css: ("#help-button .popover", {"display": "block"})
-assert-css: ("#settings-menu .popover", {"display": "none"})
+click: "rustdoc-toolbar .help-menu"
+assert-css: ("rustdoc-toolbar .help-menu .popover", {"display": "block"})
+assert-css: ("rustdoc-toolbar .settings-menu .popover", {"display": "none"})
 
 // Now verify that clicking the help menu again closes it.
-click: "#help-button"
-assert-css: ("#help-button .popover", {"display": "none"})
-assert-css: ("#settings-menu .popover", {"display": "none"})
+click: "rustdoc-toolbar .help-menu"
+assert-false: "rustdoc-toolbar .help-menu .popover"
+assert-css: (".settings-menu .popover", {"display": "none"})
 
 define-function: (
     "check-popover-colors",
@@ -37,13 +37,21 @@ define-function: (
     block {
         call-function: ("switch-theme", {"theme": |theme|})
 
-        click: "#help-button"
+        click: "rustdoc-toolbar .help-menu"
         assert-css: (
-            "#help-button .popover",
+            "rustdoc-toolbar .help-menu .popover",
             {"display": "block", "border-color": |border_color|},
         )
-        compare-elements-css: ("#help-button .popover", "#help-button .top", ["border-color"])
-        compare-elements-css: ("#help-button .popover", "#help-button .bottom", ["border-color"])
+        compare-elements-css: (
+            "rustdoc-toolbar .help-menu .popover",
+            "rustdoc-toolbar .help-menu .top",
+            ["border-color"],
+        )
+        compare-elements-css: (
+            "rustdoc-toolbar .help-menu .popover",
+            "rustdoc-toolbar .help-menu .bottom",
+            ["border-color"],
+        )
     }
 )
 
@@ -63,8 +71,21 @@ call-function: ("check-popover-colors", {
 
 // Opening the mobile sidebar should close the settings popover.
 set-window-size: (650, 600)
-click: "#settings-menu a"
-assert-css: ("#settings-menu .popover", {"display": "block"})
+click: "rustdoc-topbar .settings-menu a"
+assert-css: ("rustdoc-topbar .settings-menu .popover", {"display": "block"})
 click: ".sidebar-menu-toggle"
 assert: "//*[@class='sidebar shown']"
-assert-css: ("#settings-menu .popover", {"display": "none"})
+assert-css: ("rustdoc-topbar .settings-menu .popover", {"display": "none"})
+// Opening the settings popover should close the sidebar.
+click: ".settings-menu a"
+assert-css: ("rustdoc-topbar .settings-menu .popover", {"display": "block"})
+assert-false: "//*[@class='sidebar shown']"
+
+// Opening the settings popover at start (which async loads stuff) should also close.
+reload:
+click: ".sidebar-menu-toggle"
+assert: "//*[@class='sidebar shown']"
+assert-false: "rustdoc-topbar .settings-menu .popover"
+click: "rustdoc-topbar .settings-menu a"
+assert-false: "//*[@class='sidebar shown']"
+wait-for: "rustdoc-topbar .settings-menu .popover"
diff --git a/tests/rustdoc-gui/scrape-examples-color.goml b/tests/rustdoc-gui/scrape-examples-color.goml
index b0faca190a57..c84fe1f34111 100644
--- a/tests/rustdoc-gui/scrape-examples-color.goml
+++ b/tests/rustdoc-gui/scrape-examples-color.goml
@@ -27,7 +27,7 @@ define-function: (
             "color": |help_hover_color|,
         })
         // Moving the cursor to another item to not break next runs.
-        move-cursor-to: ".search-input"
+        move-cursor-to: "#search-button"
     }
 )
 
diff --git a/tests/rustdoc-gui/scrape-examples-ice-links.goml b/tests/rustdoc-gui/scrape-examples-ice-links.goml
new file mode 100644
index 000000000000..0a1596219350
--- /dev/null
+++ b/tests/rustdoc-gui/scrape-examples-ice-links.goml
@@ -0,0 +1,8 @@
+// Check that the line number column has the correct layout.
+go-to: "file://" + |DOC_PATH| + "/scrape_ice/struct.ObscurelyNamedType1.html"
+wait-for: ".scraped-example-title"
+assert-attribute: (".scraped-example-title a", {"href": "../src/bar/bar.rs.html#2"})
+click: ".scraped-example-title a"
+wait-for-property: ("h1", {"innerText": "bar/\nbar.rs"})
+// Ensure that the `--html-after-content` option was correctly handled.
+assert: "#outer-html"
diff --git a/tests/rustdoc-gui/scrape-examples-layout.goml b/tests/rustdoc-gui/scrape-examples-layout.goml
index 85a3b2a62873..681d0c24c6d8 100644
--- a/tests/rustdoc-gui/scrape-examples-layout.goml
+++ b/tests/rustdoc-gui/scrape-examples-layout.goml
@@ -64,8 +64,8 @@ assert-size: (".more-scraped-examples .scraped-example .example-wrap", {
 store-value: (offset_y, 4)
 
 // First with desktop
-assert-position: (".scraped-example", {"y": 256})
-assert-position: (".scraped-example .prev", {"y": 256 + |offset_y|})
+assert-position: (".scraped-example", {"y": 214})
+assert-position: (".scraped-example .prev", {"y": 214 + |offset_y|})
 
 // Gradient background should be at the top of the code block.
 assert-css: (".scraped-example .example-wrap::before", {"top": "0px"})
@@ -74,8 +74,8 @@ assert-css: (".scraped-example .example-wrap::after", {"bottom": "0px"})
 // Then with mobile
 set-window-size: (600, 600)
 store-size: (".scraped-example .scraped-example-title", {"height": title_height})
-assert-position: (".scraped-example", {"y": 291})
-assert-position: (".scraped-example .prev", {"y": 291 + |offset_y| + |title_height|})
+assert-position: (".scraped-example", {"y": 249})
+assert-position: (".scraped-example .prev", {"y": 249 + |offset_y| + |title_height|})
 
 define-function: (
     "check_title_and_code_position",
diff --git a/tests/rustdoc-gui/scrape-examples-toggle.goml b/tests/rustdoc-gui/scrape-examples-toggle.goml
index 441895a7c0ee..ec5710fbcdcd 100644
--- a/tests/rustdoc-gui/scrape-examples-toggle.goml
+++ b/tests/rustdoc-gui/scrape-examples-toggle.goml
@@ -25,7 +25,7 @@ define-function: (
         // We put the toggle in the original state.
         click: ".more-examples-toggle"
         // Moving cursor away from the toggle line to prevent disrupting next test.
-        move-cursor-to: ".search-input"
+        move-cursor-to: "rustdoc-toolbar #search-button"
     },
 )
 
diff --git a/tests/rustdoc-gui/search-about-this-result.goml b/tests/rustdoc-gui/search-about-this-result.goml
index 1d45c06dc43d..ec1df737c815 100644
--- a/tests/rustdoc-gui/search-about-this-result.goml
+++ b/tests/rustdoc-gui/search-about-this-result.goml
@@ -7,6 +7,7 @@ focus: ".search-input"
 press-key: "Enter"
 
 wait-for: "#search-tabs"
+wait-for-false: "#search-tabs .count.loading"
 assert-count: ("#search-tabs button", 1)
 assert-count: (".search-results > a", 1)
 
@@ -32,6 +33,7 @@ focus: ".search-input"
 press-key: "Enter"
 
 wait-for: "#search-tabs"
+wait-for-false: "#search-tabs .count.loading"
 assert-text: ("//div[@class='type-signature']", "F -> WhereWhitespace")
 assert-count: ("#search-tabs button", 1)
 assert-count: (".search-results > a", 1)
diff --git a/tests/rustdoc-gui/search-corrections.goml b/tests/rustdoc-gui/search-corrections.goml
index f80675730c41..a14a80f357c1 100644
--- a/tests/rustdoc-gui/search-corrections.goml
+++ b/tests/rustdoc-gui/search-corrections.goml
@@ -1,101 +1,60 @@
 // ignore-tidy-linelength
+include: "utils.goml"
 
 // Checks that the search tab result tell the user about corrections
 // First, try a search-by-name
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-// Intentionally wrong spelling of "NotableStructWithLongName"
-write-into: (".search-input", "NotableStructWithLongNamr")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "NotableStructWithLongNamr"})
 
 // Corrections aren't shown on the "In Names" tab.
 assert: "#search-tabs button.selected:first-child"
-assert-css: (".search-corrections", {
-    "display": "none"
-})
+assert-false: ".search-results:nth-child(1) .search-corrections"
 
 // Corrections do get shown on the "In Parameters" tab.
 click: "#search-tabs button:nth-child(2)"
 assert: "#search-tabs button.selected:nth-child(2)"
-assert-css: (".search-corrections", {
-    "display": "block"
-})
 assert-text: (
-    ".search-corrections",
-    "Type \"NotableStructWithLongNamr\" not found. Showing results for closest type name \"notablestructwithlongname\" instead."
+    ".search-results:nth-child(2) .search-corrections",
+    "Type \"NotableStructWithLongNamr\" not found. Showing results for closest type name \"NotableStructWithLongName\" instead."
 )
 
 // Corrections do get shown on the "In Return Type" tab.
 click: "#search-tabs button:nth-child(3)"
 assert: "#search-tabs button.selected:nth-child(3)"
-assert-css: (".search-corrections", {
-    "display": "block"
-})
 assert-text: (
-    ".search-corrections",
-    "Type \"NotableStructWithLongNamr\" not found. Showing results for closest type name \"notablestructwithlongname\" instead."
+    ".search-results:nth-child(3) .search-corrections",
+    "Type \"NotableStructWithLongNamr\" not found. Showing results for closest type name \"NotableStructWithLongName\" instead."
 )
 
 // Now, explicit return values
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-// Intentionally wrong spelling of "NotableStructWithLongName"
-write-into: (".search-input", "-> NotableStructWithLongNamr")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "-> NotableStructWithLongNamr"})
 
-assert-css: (".search-corrections", {
-    "display": "block"
-})
 assert-text: (
-    ".search-corrections",
-    "Type \"NotableStructWithLongNamr\" not found. Showing results for closest type name \"notablestructwithlongname\" instead."
+    ".search-results.active .search-corrections",
+    "Type \"NotableStructWithLongNamr\" not found and used as generic parameter. Consider searching for \"NotableStructWithLongName\" instead."
 )
 
 // Now, generic correction
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-// Intentionally wrong spelling of "NotableStructWithLongName"
-write-into: (".search-input", "NotableStructWithLongNamr, NotableStructWithLongNamr")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "NotableStructWithLongNamr, NotableStructWithLongNamr"})
 
-assert-css: (".search-corrections", {
-    "display": "block"
-})
 assert-text: (
-    ".search-corrections",
-    "Type \"NotableStructWithLongNamr\" not found and used as generic parameter. Consider searching for \"notablestructwithlongname\" instead."
+    ".search-failed.active .search-corrections",
+    "Type \"NotableStructWithLongNamr\" not found and used as generic parameter. Consider searching for \"NotableStructWithLongName\" instead."
 )
 
 // Now, generic correction plus error
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-// Intentionally wrong spelling of "NotableStructWithLongName"
-write-into: (".search-input", "Foo,y")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "Foo,y"})
 
-assert-css: (".search-corrections", {
-    "display": "block"
-})
 assert-text: (
-    ".search-corrections",
-    "Type \"NotableStructWithLongNamr\" not found and used as generic parameter. Consider searching for \"notablestructwithlongname\" instead."
+    ".search-failed.active .search-corrections",
+    "Type \"NotableStructWithLongNamr\" not found and used as generic parameter. Consider searching for \"NotableStructWithLongName\" instead."
 )
 
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
-// Intentionally wrong spelling of "NotableStructWithLongName"
-write-into: (".search-input", "generic:NotableStructWithLongNamr,y")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "generic:NotableStructWithLongNamr,y"})
 
 assert-css: (".error", {
     "display": "block"
diff --git a/tests/rustdoc-gui/search-error.goml b/tests/rustdoc-gui/search-error.goml
index 4dc60669c7a5..4d7c2263fd12 100644
--- a/tests/rustdoc-gui/search-error.goml
+++ b/tests/rustdoc-gui/search-error.goml
@@ -8,6 +8,7 @@ define-function: (
     [theme, error_background],
     block {
         call-function: ("switch-theme", {"theme": |theme|})
+        wait-for-false: "#search-tabs .count.loading"
         wait-for: "#search .error code"
         assert-css: ("#search .error code", {"background-color": |error_background|})
     }
diff --git a/tests/rustdoc-gui/search-filter.goml b/tests/rustdoc-gui/search-filter.goml
index c5038e0892b0..d92d522c119d 100644
--- a/tests/rustdoc-gui/search-filter.goml
+++ b/tests/rustdoc-gui/search-filter.goml
@@ -2,11 +2,7 @@
 include: "utils.goml"
 go-to: "file://" + |DOC_PATH| + "/test_docs/index.html"
 show-text: true
-write-into: (".search-input", "test")
-// To be SURE that the search will be run.
-press-key: 'Enter'
-// Waiting for the search results to appear...
-wait-for: "#search-tabs"
+call-function: ("perform-search", {"query": "test"})
 assert-text: ("#results .externcrate", "test_docs")
 
 wait-for: "#crate-search"
@@ -21,6 +17,7 @@ press-key: "ArrowDown"
 press-key: "Enter"
 // Waiting for the search results to appear...
 wait-for: "#search-tabs"
+wait-for-false: "#search-tabs .count.loading"
 assert-document-property: ({"URL": "&filter-crate="}, CONTAINS)
 // We check that there is no more "test_docs" appearing.
 assert-false: "#results .externcrate"
@@ -31,7 +28,8 @@ assert-property: ("#crate-search", {"value": "lib2"})
 // crate filtering.
 press-key: "Escape"
 wait-for-css: ("#main-content", {"display": "block"})
-focus: ".search-input"
+click: "#search-button"
+wait-for: ".search-input"
 wait-for-css: ("#main-content", {"display": "none"})
 // We check that there is no more "test_docs" appearing.
 assert-false: "#results .externcrate"
@@ -47,6 +45,7 @@ press-key: "ArrowUp"
 press-key: "Enter"
 // Waiting for the search results to appear...
 wait-for: "#search-tabs"
+wait-for-false: "#search-tabs .count.loading"
 assert-property: ("#crate-search", {"value": "all crates"})
 
 // Checking that the URL parameter is taken into account for crate filtering.
@@ -56,8 +55,7 @@ assert-property: ("#crate-search", {"value": "lib2"})
 assert-false: "#results .externcrate"
 
 // Checking that the text for the "title" is correct (the "all crates" comes from the "