itsscb/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-25 11:17:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lexer): Allow including frontmatter with 'tokenize'

Browse Source
This commit is contained in:
Ed Page 2025-07-09 16:40:22 -05:00
parent 3a5e13a8b3
commit f4d9018a48
3 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
crates/parser/src/lexed_str.rs
View File

@ -11,8 +11,8 @@
use std::ops;
use rustc_literal_escaper::{
EscapeError, Mode, unescape_byte, unescape_byte_str, unescape_c_str, unescape_char,
unescape_str,
unescape_byte, unescape_byte_str, unescape_c_str, unescape_char, unescape_str, EscapeError,
Mode,
};
use crate::{
@ -44,7 +44,9 @@ impl<'a> LexedStr<'a> {
// Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer
// but we want to split it to two in edition <2024.
while let Some(token) = rustc_lexer::tokenize(&text[conv.offset..]).next() {
while let Some(token) =
rustc_lexer::tokenize(&text[conv.offset..], rustc_lexer::FrontmatterAllowed::No).next()
{
let token_text = &text[conv.offset..][..token.len as usize];
conv.extend_token(&token.kind, token_text);
@ -58,7 +60,7 @@ impl<'a> LexedStr<'a> {
return None;
}
let token = rustc_lexer::tokenize(text).next()?;
let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next()?;
if token.len as usize != text.len() {
return None;
}

2
crates/proc-macro-srv/src/server_impl.rs
View File

@ -121,7 +121,7 @@ pub(super) fn literal_from_str<Span: Copy>(
use proc_macro::bridge::LitKind;
use rustc_lexer::{LiteralKind, Token, TokenKind};
let mut tokens = rustc_lexer::tokenize(s);
let mut tokens = rustc_lexer::tokenize(s, rustc_lexer::FrontmatterAllowed::No);
let minus_or_lit = tokens.next().unwrap_or(Token { kind: TokenKind::Eof, len: 0 });
let lit = if minus_or_lit.kind == TokenKind::Minus {

2
crates/tt/src/lib.rs
View File

@ -579,7 +579,7 @@ where
{
use rustc_lexer::LiteralKind;
let token = rustc_lexer::tokenize(text).next_tuple();
let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
let Some((rustc_lexer::Token {
kind: rustc_lexer::TokenKind::Literal { kind, suffix_start },
..