user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rollup merge of #146340 - fmease:frontmatter-containment, r=fee1-dead,Urgau

Browse source 
Strip frontmatter in fewer places

* Stop stripping frontmatter in `proc_macro::Literal::from_str` (RUST-146132)
* Stop stripping frontmatter in expr-ctxt (but not item-ctxt!) `include`s (RUST-145945)
* Stop stripping shebang (!) in `proc_macro::Literal::from_str`
  * Not a breaking change because it did compare spans already to ensure there wasn't extra whitespace or comments (`Literal::from_str("#!\n0")` already yields `Err(_)` thankfully!)
* Stop stripping frontmatter+shebang inside some rustdoc code where it doesn't make any observable difference (see self review comments)
* (Stop stripping frontmatter+shebang inside internal test code)

Fixes https://github.com/rust-lang/rust/issues/145945.
Fixes https://github.com/rust-lang/rust/issues/146132.

r? fee1-dead
This commit is contained in:
Matthias Krüger 2025-09-10 20:29:09 +02:00 • committed by GitHub
commit 86d39a0673
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 228 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

100
compiler/rustc_builtin_macros/src/source_util.rs
View file

@ -1,3 +1,5 @@
//! The implementation of built-in macros which relate to the file system.
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::sync::Arc;
@ -11,9 +13,11 @@ use rustc_expand::base::{
};
use rustc_expand::module::DirOwnership;
use rustc_lint_defs::BuiltinLintDiag;
use rustc_parse::parser::{ForceCollect, Parser};
use rustc_parse::lexer::StripTokens;
use rustc_parse::parser::ForceCollect;
use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal, utf8_error};
use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
use rustc_session::parse::ParseSess;
use rustc_span::source_map::SourceMap;
use rustc_span::{ByteSymbol, Pos, Span, Symbol};
use smallvec::SmallVec;
@ -23,11 +27,7 @@ use crate::util::{
check_zero_tts, get_single_str_from_tts, get_single_str_spanned_from_tts, parse_expr,
};
// These macros all relate to the file system; they either return
// the column/row/filename of the expression, or they include
// a given file into the current one.
/// line!(): expands to the current line number
/// Expand `line!()` to the current line number.
pub(crate) fn expand_line(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -42,7 +42,7 @@ pub(crate) fn expand_line(
ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.line as u32)))
}
/* column!(): expands to the current column number */
/// Expand `column!()` to the current column number.
pub(crate) fn expand_column(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -57,9 +57,7 @@ pub(crate) fn expand_column(
ExpandResult::Ready(MacEager::expr(cx.expr_u32(topmost, loc.col.to_usize() as u32 + 1)))
}
/// file!(): expands to the current filename */
/// The source_file (`loc.file`) contains a bunch more information we could spit
/// out if we wanted.
/// Expand `file!()` to the current filename.
pub(crate) fn expand_file(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -81,6 +79,7 @@ pub(crate) fn expand_file(
)))
}
/// Expand `stringify!($input)`.
pub(crate) fn expand_stringify(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -91,6 +90,7 @@ pub(crate) fn expand_stringify(
ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&s))))
}
/// Expand `module_path!()` to (a textual representation of) the current module path.
pub(crate) fn expand_mod(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -104,9 +104,9 @@ pub(crate) fn expand_mod(
ExpandResult::Ready(MacEager::expr(cx.expr_str(sp, Symbol::intern(&string))))
}
/// include! : parse the given file as an expr
/// This is generally a bad idea because it's going to behave
/// unhygienically.
/// Expand `include!($input)`.
///
/// This works in item and expression position. Notably, it doesn't work in pattern position.
pub(crate) fn expand_include<'cx>(
cx: &'cx mut ExtCtxt<'_>,
sp: Span,
@ -116,39 +116,48 @@ pub(crate) fn expand_include<'cx>(
let ExpandResult::Ready(mac) = get_single_str_from_tts(cx, sp, tts, "include!") else {
return ExpandResult::Retry(());
};
let file = match mac {
Ok(file) => file,
let path = match mac {
Ok(path) => path,
Err(guar) => return ExpandResult::Ready(DummyResult::any(sp, guar)),
};
// The file will be added to the code map by the parser
let file = match resolve_path(&cx.sess, file.as_str(), sp) {
Ok(f) => f,
let path = match resolve_path(&cx.sess, path.as_str(), sp) {
Ok(path) => path,
Err(err) => {
let guar = err.emit();
return ExpandResult::Ready(DummyResult::any(sp, guar));
}
};
let p = unwrap_or_emit_fatal(new_parser_from_file(cx.psess(), &file, Some(sp)));
// If in the included file we have e.g., `mod bar;`,
// then the path of `bar.rs` should be relative to the directory of `file`.
// then the path of `bar.rs` should be relative to the directory of `path`.
// See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion.
// `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained.
let dir_path = file.parent().unwrap_or(&file).to_owned();
let dir_path = path.parent().unwrap_or(&path).to_owned();
cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path));
cx.current_expansion.dir_ownership = DirOwnership::Owned { relative: None };
struct ExpandInclude<'a> {
p: Parser<'a>,
psess: &'a ParseSess,
path: PathBuf,
node_id: ast::NodeId,
span: Span,
}
impl<'a> MacResult for ExpandInclude<'a> {
fn make_expr(mut self: Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>> {
let expr = parse_expr(&mut self.p).ok()?;
if self.p.token != token::Eof {
self.p.psess.buffer_lint(
fn make_expr(self: Box<ExpandInclude<'a>>) -> Option<Box<ast::Expr>> {
let mut p = unwrap_or_emit_fatal(new_parser_from_file(
self.psess,
&self.path,
// Don't strip frontmatter for backward compatibility, `---` may be the start of a
// manifold negation. FIXME: Ideally, we wouldn't strip shebangs here either.
StripTokens::Shebang,
Some(self.span),
));
let expr = parse_expr(&mut p).ok()?;
if p.token != token::Eof {
p.psess.buffer_lint(
INCOMPLETE_INCLUDE,
self.p.token.span,
p.token.span,
self.node_id,
BuiltinLintDiag::IncompleteInclude,
);
@ -156,24 +165,27 @@ pub(crate) fn expand_include<'cx>(
Some(expr)
}
fn make_items(mut self: Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>; 1]>> {
fn make_items(self: Box<ExpandInclude<'a>>) -> Option<SmallVec<[Box<ast::Item>; 1]>> {
let mut p = unwrap_or_emit_fatal(new_parser_from_file(
self.psess,
&self.path,
StripTokens::ShebangAndFrontmatter,
Some(self.span),
));
let mut ret = SmallVec::new();
loop {
match self.p.parse_item(ForceCollect::No) {
match p.parse_item(ForceCollect::No) {
Err(err) => {
err.emit();
break;
}
Ok(Some(item)) => ret.push(item),
Ok(None) => {
if self.p.token != token::Eof {
self.p
.dcx()
.create_err(errors::ExpectedItem {
span: self.p.token.span,
token: &pprust::token_to_string(&self.p.token),
})
.emit();
if p.token != token::Eof {
p.dcx().emit_err(errors::ExpectedItem {
span: p.token.span,
token: &pprust::token_to_string(&p.token),
});
}
break;
@ -184,10 +196,17 @@ pub(crate) fn expand_include<'cx>(
}
}
ExpandResult::Ready(Box::new(ExpandInclude { p, node_id: cx.current_expansion.lint_node_id }))
ExpandResult::Ready(Box::new(ExpandInclude {
psess: cx.psess(),
path,
node_id: cx.current_expansion.lint_node_id,
span: sp,
}))
}
/// `include_str!`: read the given file, insert it as a literal string expr
/// Expand `include_str!($input)` to the content of the UTF-8-encoded file given by path `$input` as a string literal.
///
/// This works in expression, pattern and statement position.
pub(crate) fn expand_include_str(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -206,6 +225,7 @@ pub(crate) fn expand_include_str(
Ok((bytes, bsp)) => match std::str::from_utf8(&bytes) {
Ok(src) => {
let interned_src = Symbol::intern(src);
// MacEager converts the expr into a pat if need be.
MacEager::expr(cx.expr_str(cx.with_def_site_ctxt(bsp), interned_src))
}
Err(utf8err) => {
@ -218,6 +238,9 @@ pub(crate) fn expand_include_str(
})
}
/// Expand `include_bytes!($input)` to the content of the file given by path `$input`.
///
/// This works in expression, pattern and statement position.
pub(crate) fn expand_include_bytes(
cx: &mut ExtCtxt<'_>,
sp: Span,
@ -237,6 +260,7 @@ pub(crate) fn expand_include_bytes(
// Don't care about getting the span for the raw bytes,
// because the console can't really show them anyway.
let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(ByteSymbol::intern(&bytes)));
// MacEager converts the expr into a pat if need be.
MacEager::expr(expr)
}
Err(dummy) => dummy,

12
compiler/rustc_driver_impl/src/lib.rs
View file

@ -51,6 +51,7 @@ use rustc_lint::unerased_lint_store;
use rustc_metadata::creader::MetadataLoader;
use rustc_metadata::locator;
use rustc_middle::ty::TyCtxt;
use rustc_parse::lexer::StripTokens;
use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
use rustc_session::config::{
CG_OPTIONS, CrateType, ErrorOutputType, Input, OptionDesc, OutFileName, OutputType, Sysroot,
@ -1288,10 +1289,15 @@ fn warn_on_confusing_output_filename_flag(
fn parse_crate_attrs<'a>(sess: &'a Session) -> PResult<'a, ast::AttrVec> {
let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
Input::File(file) => new_parser_from_file(&sess.psess, file, None),
Input::Str { name, input } => {
new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
Input::File(file) => {
new_parser_from_file(&sess.psess, file, StripTokens::ShebangAndFrontmatter, None)
}
Input::Str { name, input } => new_parser_from_source_str(
&sess.psess,
name.clone(),
input.clone(),
StripTokens::ShebangAndFrontmatter,
),
});
parser.parse_inner_attributes()
}

9
compiler/rustc_expand/src/module.rs
View file

@ -4,6 +4,7 @@ use std::path::{self, Path, PathBuf};
use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans};
use rustc_attr_parsing::validate_attr;
use rustc_errors::{Diag, ErrorGuaranteed};
use rustc_parse::lexer::StripTokens;
use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal};
use rustc_session::Session;
use rustc_session::parse::ParseSess;
@ -67,8 +68,12 @@ pub(crate) fn parse_external_mod(
}
// Actually parse the external file as a module.
let mut parser =
unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span)));
let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
&sess.psess,
&mp.file_path,
StripTokens::ShebangAndFrontmatter,
Some(span),
));
let (inner_attrs, items, inner_span) =
parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?;
attrs.extend(inner_attrs);

11
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -8,7 +8,7 @@ use rustc_ast::util::literal::escape_byte_str_symbol;
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
use rustc_parse::lexer::nfc_normalize;
use rustc_parse::lexer::{StripTokens, nfc_normalize};
use rustc_parse::parser::Parser;
use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
use rustc_proc_macro::bridge::{
@ -485,8 +485,13 @@ impl server::FreeFunctions for Rustc<'_, '_> {
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
let name = FileName::proc_macro_source_code(s);
let mut parser =
unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned()));
let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
self.psess(),
name,
s.to_owned(),
StripTokens::Nothing,
));
let first_span = parser.token.span.data();
let minus_present = parser.eat(exp!(Minus));

22
compiler/rustc_interface/src/interface.rs
View file

@ -13,7 +13,8 @@ use rustc_lint::LintStore;
use rustc_middle::ty;
use rustc_middle::ty::CurrentGcx;
use rustc_middle::util::Providers;
use rustc_parse::new_parser_from_simple_source_str;
use rustc_parse::lexer::StripTokens;
use rustc_parse::new_parser_from_source_str;
use rustc_parse::parser::attr::AllowLeadingUnsafe;
use rustc_query_impl::QueryCtxt;
use rustc_query_system::query::print_query_stack;
@ -68,7 +69,8 @@ pub(crate) fn parse_cfg(dcx: DiagCtxtHandle<'_>, cfgs: Vec<String>) -> Cfg {
};
}
match new_parser_from_simple_source_str(&psess, filename, s.to_string()) {
match new_parser_from_source_str(&psess, filename, s.to_string(), StripTokens::Nothing)
{
Ok(mut parser) => match parser.parse_meta_item(AllowLeadingUnsafe::No) {
Ok(meta_item) if parser.token == token::Eof => {
if meta_item.path.segments.len() != 1 {
@ -166,13 +168,15 @@ pub(crate) fn parse_check_cfg(dcx: DiagCtxtHandle<'_>, specs: Vec<String>) -> Ch
error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
};
let mut parser = match new_parser_from_simple_source_str(&psess, filename, s.to_string()) {
Ok(parser) => parser,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
expected_error();
}
};
let mut parser =
match new_parser_from_source_str(&psess, filename, s.to_string(), StripTokens::Nothing)
{
Ok(parser) => parser,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
expected_error();
}
};
let meta_item = match parser.parse_meta_item(AllowLeadingUnsafe::No) {
Ok(meta_item) if parser.token == token::Eof => meta_item,

17
compiler/rustc_interface/src/passes.rs
View file

@ -28,6 +28,7 @@ use rustc_middle::arena::Arena;
use rustc_middle::dep_graph::DepsType;
use rustc_middle::ty::{self, CurrentGcx, GlobalCtxt, RegisteredTools, TyCtxt};
use rustc_middle::util::Providers;
use rustc_parse::lexer::StripTokens;
use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
use rustc_passes::{abi_test, input_stats, layout_test};
use rustc_resolve::{Resolver, ResolverOutputs};
@ -52,10 +53,18 @@ pub fn parse<'a>(sess: &'a Session) -> ast::Crate {
let mut krate = sess
.time("parse_crate", || {
let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
Input::File(file) => new_parser_from_file(&sess.psess, file, None),
Input::Str { input, name } => {
new_parser_from_source_str(&sess.psess, name.clone(), input.clone())
}
Input::File(file) => new_parser_from_file(
&sess.psess,
file,
StripTokens::ShebangAndFrontmatter,
None,
),
Input::Str { input, name } => new_parser_from_source_str(
&sess.psess,
name.clone(),
input.clone(),
StripTokens::ShebangAndFrontmatter,
),
});
parser.parse_crate_mod()
})

2
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -45,7 +45,7 @@ pub(crate) struct UnmatchedDelim {
}
/// Which tokens should be stripped before lexing the tokens.
pub(crate) enum StripTokens {
pub enum StripTokens {
/// Strip both shebang and frontmatter.
ShebangAndFrontmatter,
/// Strip the shebang but not frontmatter.

40
compiler/rustc_parse/src/lib.rs
View file

@ -54,29 +54,18 @@ pub fn unwrap_or_emit_fatal<T>(expr: Result<T, Vec<Diag<'_>>>) -> T {
}
}
/// Creates a new parser from a source string. On failure, the errors must be consumed via
/// `unwrap_or_emit_fatal`, `emit`, `cancel`, etc., otherwise a panic will occur when they are
/// dropped.
/// Creates a new parser from a source string.
///
/// On failure, the errors must be consumed via `unwrap_or_emit_fatal`, `emit`, `cancel`,
/// etc., otherwise a panic will occur when they are dropped.
pub fn new_parser_from_source_str(
psess: &ParseSess,
name: FileName,
source: String,
strip_tokens: StripTokens,
) -> Result<Parser<'_>, Vec<Diag<'_>>> {
let source_file = psess.source_map().new_source_file(name, source);
new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter)
}
/// Creates a new parser from a simple (no shebang, no frontmatter) source string.
///
/// On failure, the errors must be consumed via `unwrap_or_emit_fatal`, `emit`, `cancel`,
/// etc., otherwise a panic will occur when they are dropped.
pub fn new_parser_from_simple_source_str(
psess: &ParseSess,
name: FileName,
source: String,
) -> Result<Parser<'_>, Vec<Diag<'_>>> {
let source_file = psess.source_map().new_source_file(name, source);
new_parser_from_source_file(psess, source_file, StripTokens::Nothing)
new_parser_from_source_file(psess, source_file, strip_tokens)
}
/// Creates a new parser from a filename. On failure, the errors must be consumed via
@ -87,6 +76,7 @@ pub fn new_parser_from_simple_source_str(
pub fn new_parser_from_file<'a>(
psess: &'a ParseSess,
path: &Path,
strip_tokens: StripTokens,
sp: Option<Span>,
) -> Result<Parser<'a>, Vec<Diag<'a>>> {
let sm = psess.source_map();
@ -110,7 +100,7 @@ pub fn new_parser_from_file<'a>(
}
err.emit();
});
new_parser_from_source_file(psess, source_file, StripTokens::ShebangAndFrontmatter)
new_parser_from_source_file(psess, source_file, strip_tokens)
}
pub fn utf8_error<E: EmissionGuarantee>(
@ -172,6 +162,9 @@ fn new_parser_from_source_file(
Ok(parser)
}
/// Given a source string, produces a sequence of token trees.
///
/// NOTE: This only strips shebangs, not frontmatter!
pub fn source_str_to_stream(
psess: &ParseSess,
name: FileName,
@ -179,13 +172,16 @@ pub fn source_str_to_stream(
override_span: Option<Span>,
) -> Result<TokenStream, Vec<Diag<'_>>> {
let source_file = psess.source_map().new_source_file(name, source);
// used mainly for `proc_macro` and the likes, not for our parsing purposes, so don't parse
// frontmatters as frontmatters, but for compatibility reason still strip the shebang
// FIXME(frontmatter): Consider stripping frontmatter in a future edition. We can't strip them
// in the current edition since that would be breaking.
// See also <https://github.com/rust-lang/rust/issues/145520>.
// Alternatively, stop stripping shebangs here, too, if T-lang and crater approve.
source_file_to_stream(psess, source_file, override_span, StripTokens::Shebang)
}
/// Given a source file, produces a sequence of token trees. Returns any buffered errors from
/// parsing the token stream.
/// Given a source file, produces a sequence of token trees.
///
/// Returns any buffered errors from parsing the token stream.
fn source_file_to_stream<'psess>(
psess: &'psess ParseSess,
source_file: Arc<SourceFile>,

7
compiler/rustc_parse/src/parser/tests.rs
View file

@ -22,6 +22,7 @@ use rustc_span::{
};
use termcolor::WriteColor;
use crate::lexer::StripTokens;
use crate::parser::{ForceCollect, Parser};
use crate::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
@ -35,6 +36,7 @@ fn string_to_parser(psess: &ParseSess, source_str: String) -> Parser<'_> {
psess,
PathBuf::from("bogofile").into(),
source_str,
StripTokens::Nothing,
))
}
@ -2240,7 +2242,7 @@ fn parse_item_from_source_str(
source: String,
psess: &ParseSess,
) -> PResult<'_, Option<Box<ast::Item>>> {
unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source))
unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source, StripTokens::Nothing))
.parse_item(ForceCollect::No)
}
@ -2520,7 +2522,8 @@ fn ttdelim_span() {
source: String,
psess: &ParseSess,
) -> PResult<'_, Box<ast::Expr>> {
unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source)).parse_expr()
unwrap_or_emit_fatal(new_parser_from_source_str(psess, name, source, StripTokens::Nothing))
.parse_expr()
}
create_default_session_globals_then(|| {

21
src/librustdoc/clean/render_macro_matchers.rs
View file

@ -3,6 +3,7 @@ use rustc_ast::tokenstream::{TokenStream, TokenTree};
use rustc_ast_pretty::pprust::PrintState;
use rustc_ast_pretty::pprust::state::State as Printer;
use rustc_middle::ty::TyCtxt;
use rustc_parse::lexer::StripTokens;
use rustc_session::parse::ParseSess;
use rustc_span::symbol::{Ident, Symbol, kw};
use rustc_span::{FileName, Span};
@ -64,14 +65,18 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String
// Create a Parser.
let psess = ParseSess::new(rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec());
let file_name = FileName::macro_expansion_source_code(&snippet);
let mut parser =
match rustc_parse::new_parser_from_source_str(&psess, file_name, snippet.clone()) {
Ok(parser) => parser,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
return None;
}
};
let mut parser = match rustc_parse::new_parser_from_source_str(
&psess,
file_name,
snippet.clone(),
StripTokens::Nothing,
) {
Ok(parser) => parser,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
return None;
}
};
// Reparse a single token tree.
if parser.token == token::Eof {

19
src/librustdoc/doctest/make.rs
View file

@ -10,6 +10,7 @@ use rustc_ast::tokenstream::TokenTree;
use rustc_ast::{self as ast, AttrStyle, HasAttrs, StmtKind};
use rustc_errors::emitter::stderr_destination;
use rustc_errors::{ColorConfig, DiagCtxtHandle};
use rustc_parse::lexer::StripTokens;
use rustc_parse::new_parser_from_source_str;
use rustc_session::parse::ParseSess;
use rustc_span::edition::{DEFAULT_EDITION, Edition};
@ -468,14 +469,16 @@ fn parse_source(
let dcx = DiagCtxt::new(Box::new(emitter)).disable_warnings();
let psess = ParseSess::with_dcx(dcx, sm);
let mut parser = match new_parser_from_source_str(&psess, filename, wrapped_source) {
Ok(p) => p,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
reset_error_count(&psess);
return Err(());
}
};
// Don't strip any tokens; it wouldn't matter anyway because the source is wrapped in a function.
let mut parser =
match new_parser_from_source_str(&psess, filename, wrapped_source, StripTokens::Nothing) {
Ok(p) => p,
Err(errs) => {
errs.into_iter().for_each(|err| err.cancel());
reset_error_count(&psess);
return Err(());
}
};
fn push_to_s(s: &mut String, source: &str, span: rustc_span::Span, prev_span_hi: &mut usize) {
let extra_len = DOCTEST_CODE_WRAPPER.len();

16
src/tools/clippy/clippy_lints/src/doc/needless_doctest_main.rs
View file

@ -8,6 +8,7 @@ use rustc_ast::{CoroutineKind, Fn, FnRetTy, Item, ItemKind};
use rustc_errors::emitter::HumanEmitter;
use rustc_errors::{Diag, DiagCtxt};
use rustc_lint::LateContext;
use rustc_parse::lexer::StripTokens;
use rustc_parse::new_parser_from_source_str;
use rustc_parse::parser::ForceCollect;
use rustc_session::parse::ParseSess;
@ -49,13 +50,14 @@ pub fn check(
let sm = Arc::new(SourceMap::new(FilePathMapping::empty()));
let psess = ParseSess::with_dcx(dcx, sm);
let mut parser = match new_parser_from_source_str(&psess, filename, code) {
Ok(p) => p,
Err(errs) => {
errs.into_iter().for_each(Diag::cancel);
return (false, test_attr_spans);
},
};
let mut parser =
match new_parser_from_source_str(&psess, filename, code, StripTokens::ShebangAndFrontmatter) {
Ok(p) => p,
Err(errs) => {
errs.into_iter().for_each(Diag::cancel);
return (false, test_attr_spans);
},
};
let mut relevant_main_found = false;
let mut eligible = true;

14
src/tools/rustfmt/src/parse/parser.rs
View file

@ -3,6 +3,7 @@ use std::path::{Path, PathBuf};
use rustc_ast::{ast, attr};
use rustc_errors::Diag;
use rustc_parse::lexer::StripTokens;
use rustc_parse::parser::Parser as RawParser;
use rustc_parse::{exp, new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
use rustc_span::{Span, sym};
@ -64,11 +65,14 @@ impl<'a> ParserBuilder<'a> {
input: Input,
) -> Result<RawParser<'a>, Vec<Diag<'a>>> {
match input {
Input::File(ref file) => new_parser_from_file(psess, file, None),
Input::File(ref file) => {
new_parser_from_file(psess, file, StripTokens::ShebangAndFrontmatter, None)
}
Input::Text(text) => new_parser_from_source_str(
psess,
rustc_span::FileName::Custom("stdin".to_owned()),
text,
StripTokens::ShebangAndFrontmatter,
),
}
}
@ -104,8 +108,12 @@ impl<'a> Parser<'a> {
span: Span,
) -> Result<(ast::AttrVec, ThinVec<Box<ast::Item>>, Span), ParserError> {
let result = catch_unwind(AssertUnwindSafe(|| {
let mut parser =
unwrap_or_emit_fatal(new_parser_from_file(psess.inner(), path, Some(span)));
let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
psess.inner(),
path,
StripTokens::ShebangAndFrontmatter,
Some(span),
));
match parser.parse_mod(exp!(Eof)) {
Ok((a, i, spans)) => Some((a, i, spans.inner_span)),
Err(e) => {

3
tests/ui-fulldeps/auxiliary/parser.rs
View file

@ -10,7 +10,7 @@ extern crate rustc_span;
use rustc_ast::ast::{AttrKind, Attribute, DUMMY_NODE_ID, Expr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::node_id::NodeId;
use rustc_ast::token::{self, Token};
use rustc_ast::token;
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, LazyAttrTokenStream};
use rustc_errors::Diag;
use rustc_parse::parser::Recovery;
@ -23,6 +23,7 @@ pub fn parse_expr(psess: &ParseSess, source_code: &str) -> Option<Box<Expr>> {
psess,
FileName::anon_source_code(source_code),
source_code.to_owned(),
rustc_parse::lexer::StripTokens::Nothing,
));
let mut parser = parser.recovery(Recovery::Forbidden);

9
tests/ui-fulldeps/mod_dir_path_canonicalized.rs
View file

@ -16,7 +16,7 @@ extern crate rustc_span;
#[allow(unused_extern_crates)]
extern crate rustc_driver;
use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal};
use rustc_parse::{lexer::StripTokens, new_parser_from_file, unwrap_or_emit_fatal};
use rustc_session::parse::ParseSess;
use std::path::Path;
@ -34,6 +34,11 @@ fn parse() {
let path = Path::new(file!());
let path = path.canonicalize().unwrap();
let mut parser = unwrap_or_emit_fatal(new_parser_from_file(&psess, &path, None));
let mut parser = unwrap_or_emit_fatal(new_parser_from_file(
&psess,
&path,
StripTokens::ShebangAndFrontmatter,
None,
));
let _ = parser.parse_crate_mod();
}

4
tests/ui/frontmatter/auxiliary/expr.rs Normal file
View file

@ -0,0 +1,4 @@
---
-
---
1

14
tests/ui/frontmatter/auxiliary/makro.rs
View file

@ -1,8 +1,20 @@
extern crate proc_macro;
use proc_macro::TokenStream;
use proc_macro::{Literal, TokenStream};
#[proc_macro]
pub fn check(_: TokenStream) -> TokenStream {
// In the following test cases, the `---` may look like the start of frontmatter but it is not!
// That's because it would be backward incompatible to interpret them as such in the latest
// stable edition. That's not only the case due to the feature gate error but also due to the
// fact that we "eagerly" emit errors on malformed frontmatter.
// issue: <https://github.com/rust-lang/rust/issues/145520>
_ = "---".parse::<TokenStream>();
// Just a sequence of regular Rust punctuation tokens.
assert_eq!(6, "---\n---".parse::<TokenStream>().unwrap().into_iter().count());
// issue: <https://github.com/rust-lang/rust/issues/146132>
assert!("---".parse::<Literal>().is_err());
Default::default()
}

9
tests/ui/frontmatter/include-in-expr-ctxt.rs Normal file
View file

@ -0,0 +1,9 @@
// Check that an expr-ctxt `include` doesn't try to parse frontmatter and instead
// treats it as a regular Rust token sequence.
//@ check-pass
#![expect(double_negations)]
fn main() {
// issue: <https://github.com/rust-lang/rust/issues/145945>
const _: () = assert!(-1 == include!("auxiliary/expr.rs"));
}

10
tests/ui/frontmatter/include-in-item-ctxt.rs Normal file
View file

@ -0,0 +1,10 @@
// Ensure that in item ctxts we can `include` files that contain frontmatter.
//@ check-pass
#![feature(frontmatter)]
include!("auxiliary/lib.rs");
fn main() {
foo(1);
}

12
tests/ui/frontmatter/included-frontmatter.rs
View file

@ -1,12 +0,0 @@
#![feature(frontmatter)]
//@ check-pass
include!("auxiliary/lib.rs");
// auxiliary/lib.rs contains a frontmatter. Ensure that we can use them in an
// `include!` macro.
fn main() {
foo(1);
}

7
tests/ui/frontmatter/proc-macro-observer.rs
View file

@ -2,10 +2,9 @@
//@ proc-macro: makro.rs
//@ edition: 2021
// Check that a proc-macro doesn't try to parse frontmatter and instead treats
// it as a regular Rust token sequence. See `auxiliary/makro.rs` for details.
makro::check!();
// checks that a proc-macro doesn't know or parse frontmatters at all and instead treats
// it as normal Rust code.
// see auxiliary/makro.rs for how it is tested.
fn main() {}