itsscb/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-10-01 11:31:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge #1668

Browse Source 
1668: Reduce code duplication in lexer r=matklad a=eupn

During an investigation of a fix for #1218, I stumbled upon some code duplication in the lexer.

Co-authored-by: Evgenii P <eupn@protonmail.com>
This commit is contained in:
bors[bot] 2019-08-09 08:11:26 +00:00
commit 2fbec23d99
1 changed files with 15 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
crates/ra_syntax/src/parsing/lexer.rs
View File

@ -12,6 +12,19 @@ pub struct Token {
pub len: TextUnit, pub len: TextUnit,
} }
fn match_literal_kind(kind: ra_rustc_lexer::LiteralKind) -> SyntaxKind {
match kind {
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
}
}
/// Break a string up into its component tokens /// Break a string up into its component tokens
pub fn tokenize(text: &str) -> Vec<Token> { pub fn tokenize(text: &str) -> Vec<Token> {
if text.is_empty() { if text.is_empty() {
@ -53,16 +66,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
} }
} }
ra_rustc_lexer::TokenKind::RawIdent => IDENT, ra_rustc_lexer::TokenKind::RawIdent => IDENT,
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
},
ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME, ra_rustc_lexer::TokenKind::Lifetime { .. } => LIFETIME,
ra_rustc_lexer::TokenKind::Semi => SEMI, ra_rustc_lexer::TokenKind::Semi => SEMI,
ra_rustc_lexer::TokenKind::Comma => COMMA, ra_rustc_lexer::TokenKind::Comma => COMMA,
@ -131,16 +135,7 @@ pub fn classify_literal(text: &str) -> Option<Token> {
return None; return None;
} }
let kind = match t.kind { let kind = match t.kind {
ra_rustc_lexer::TokenKind::Literal { kind, .. } => match kind { ra_rustc_lexer::TokenKind::Literal { kind, .. } => match_literal_kind(kind),
ra_rustc_lexer::LiteralKind::Int { .. } => INT_NUMBER,
ra_rustc_lexer::LiteralKind::Float { .. } => FLOAT_NUMBER,
ra_rustc_lexer::LiteralKind::Char { .. } => CHAR,
ra_rustc_lexer::LiteralKind::Byte { .. } => BYTE,
ra_rustc_lexer::LiteralKind::Str { .. } => STRING,
ra_rustc_lexer::LiteralKind::ByteStr { .. } => BYTE_STRING,
ra_rustc_lexer::LiteralKind::RawStr { .. } => RAW_STRING,
ra_rustc_lexer::LiteralKind::RawByteStr { .. } => RAW_BYTE_STRING,
},
_ => return None, _ => return None,
}; };
Some(Token { kind, len: TextUnit::from_usize(t.len) }) Some(Token { kind, len: TextUnit::from_usize(t.len) })