diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index 18ebed7c70e1..c71e6ffe34d9 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -23,15 +23,17 @@ // We want to be able to build this crate with a stable compiler, so no // `#![feature]` attributes should be added. -pub mod cursor; +mod cursor; pub mod unescape; #[cfg(test)] mod tests; +pub use crate::cursor::Cursor; + use self::LiteralKind::*; use self::TokenKind::*; -use crate::cursor::{Cursor, EOF_CHAR}; +use crate::cursor::EOF_CHAR; use std::convert::TryFrom; /// Parsed token. diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 151e80e2b3e9..bcd078a89677 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -4,8 +4,8 @@ use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::unicode::contains_text_flow_control_chars; use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult}; -use rustc_lexer::cursor::Cursor; use rustc_lexer::unescape::{self, Mode}; +use rustc_lexer::Cursor; use rustc_lexer::{Base, DocStyle, RawStrError}; use rustc_session::lint::builtin::{ RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT, diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index 6f6ab16cb597..364753154db0 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -53,7 +53,7 @@ impl<'a> TokenTreesReader<'a> { token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)), token::CloseDelim(delim) => return Err(self.close_delim_err(delim)), token::Eof => return Ok(buf.into_token_stream()), - _ => buf.push(self.parse_token_tree_other()), + _ => buf.push(self.parse_token_tree_non_delim_non_eof()), } } } @@ -66,11 +66,10 @@ impl<'a> TokenTreesReader<'a> { token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)), token::CloseDelim(..) => return buf.into_token_stream(), token::Eof => { - let mut err = self.eof_err(); - err.emit(); + self.eof_err().emit(); return buf.into_token_stream(); } - _ => buf.push(self.parse_token_tree_other()), + _ => buf.push(self.parse_token_tree_non_delim_non_eof()), } } } @@ -245,9 +244,10 @@ impl<'a> TokenTreesReader<'a> { } #[inline] - fn parse_token_tree_other(&mut self) -> TokenTree { - // `spacing` for the returned token is determined by the next token: - // its kind and its `preceded_by_whitespace` status. + fn parse_token_tree_non_delim_non_eof(&mut self) -> TokenTree { + // `this_spacing` for the returned token refers to whether the token is + // immediately followed by another op token. It is determined by the + // next token: its kind and its `preceded_by_whitespace` status. let (next_tok, is_next_tok_preceded_by_whitespace) = self.string_reader.next_token(); let this_spacing = if is_next_tok_preceded_by_whitespace || !next_tok.is_op() { Spacing::Alone diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index ea65a6334c9e..78b98431b190 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -13,7 +13,7 @@ use std::collections::VecDeque; use std::fmt::{Display, Write}; use rustc_data_structures::fx::FxHashMap; -use rustc_lexer::cursor::Cursor; +use rustc_lexer::Cursor; use rustc_lexer::{LiteralKind, TokenKind}; use rustc_span::edition::Edition; use rustc_span::symbol::Symbol;