From e94ba4494124078a4a6426a74938274030f11c88 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 28 Jul 2020 23:46:26 +0200 Subject: [PATCH 01/84] 1.0.0 --- .gitignore | 2 + Cargo.toml | 11 + README.md | 5 + rust.ungram | 618 +++++++++++++++++++++++++++++++++++++++++++++++ src/error.rs | 45 ++++ src/lexer.rs | 129 ++++++++++ src/lib.rs | 89 +++++++ src/parser.rs | 217 +++++++++++++++++ ungrammar.ungram | 16 ++ 9 files changed, 1132 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 rust.ungram create mode 100644 src/error.rs create mode 100644 src/lexer.rs create mode 100644 src/lib.rs create mode 100644 src/parser.rs create mode 100644 ungrammar.ungram diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..9c71cc94a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..53e92506e9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ungrammar" +description = "A DSL for describing concrete syntax trees" +version = "1.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/matklad/ungrammar" +authors = ["Aleksey Kladov "] +edition = "2018" + +[dependencies] +# nope diff --git a/README.md b/README.md new file mode 100644 index 0000000000..b4f8f375ec --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# ungrammar + +A DLS for specifying concrete syntax tree. + +See [./rust.ungram](./rust.ungram) for an example. diff --git a/rust.ungram b/rust.ungram new file mode 100644 index 0000000000..ef5da72622 --- /dev/null +++ b/rust.ungram @@ -0,0 +1,618 @@ +// Rust Un-Grammar. +// +// This grammar specifies the structure of Rust's concrete sytnax tree. +// It does not specify parsing rules (ambiguities, precedence, etc are out of scope). +// Tokens are processed -- contextual keywords are recogniesed, compound operators glued. +// +// Legend: +// +// // -- comment +// Name = -- non-termial defition +// 'ident' -- token (terminal) +// A B -- sequence +// A | B -- alternation +// A* -- zero or more repetition +// A? -- zero or one repetition +// (A) -- same as A +// label:A -- suggested name for field of AST node + +//*************************// +// Names, Paths and Macros // +//*************************// + +Name = + 'ident' + +NameRef = + 'ident' | 'int_number' + +Path = + (qualifier:Path '::')? segment:PathSegment + +PathSegment = + 'crate' | 'self' | 'super' +| '::'? NameRef +| NameRef GenericArgList? +| NameRef ParamList RetType? +| '<' PathType ('as' PathType)? '>' + +GenericArgList = + '::'? '<' (GenericArg (',' GenericArg)* ','?)? '>' + +GenericArg = + TypeArg +| AssocTypeArg +| LifetimeArg +| ConstArg + +TypeArg = + Type + +AssocTypeArg = + NameRef (':' TypeBoundList | '=' Type) + +LifetimeArg = + 'lifetime' + +ConstArg = + Expr + +MacroCall = + Attr* Path '!' Name? TokenTree ';'? + +TokenTree = + '(' ')' +| '{' '}' +| '[' ']' + +MacroItems = + Item* + +MacroStmts = + statements:Stmt* + Expr? + +//*************************// +// Items // +//*************************// + +SourceFile = + 'shebang'? + Attr* + Item* + +Item = + Const +| Enum +| ExternBlock +| ExternCrate +| Fn +| Impl +| MacroCall +| Module +| Static +| Struct +| Trait +| TypeAlias +| Union +| Use + +Module = + Attr* Visibility? + 'mod' Name + (ItemList | ';') + +ItemList = + '{' Attr* Item* '}' + +ExternCrate = + Attr* Visibility? + 'extern' 'crate' (NameRef | 'self') Rename? ';' + +Rename = + 'as' (Name | '_') + +Use = + Attr* Visibility? + 'use' UseTree ';' + +UseTree = + (Path? '::')? ('*' | UseTreeList) +| Path Rename? + +UseTreeList = + '{' (UseTree (',' UseTree)* ','?)? '}' + +Fn = + Attr* Visibility? + 'default'? ('async' | 'const')? 'unsafe'? Abi? + 'fn' Name GenericParamList? ParamList RetType? WhereClause? + (body:BlockExpr | ';') + +Abi = + 'extern' 'string'? + +ParamList = + '('( + SelfParam + | (SelfParam ',')? (Param (',' Param)* ','?)? + )')' + +SelfParam = + Attr* ( + ('&' 'lifetime'?)? 'mut'? 'self' + | 'mut'? 'self' ':' Type + ) + +Param = + Attr* ( + Pat (':' Type) + | Type + | '...' + ) + +RetType = + '->' Type + +TypeAlias = + Attr* Visibility? + 'default'? + 'type' Name GenericParamList? (':' TypeBoundList?)? WhereClause? + '=' Type ';' + +Struct = + Attr* Visibility? + 'struct' Name GenericParamList? ( + WhereClause? (RecordFieldList | ';') + | TupleFieldList WhereClause? ';' + ) + +RecordFieldList = + '{' fields:(RecordField (',' RecordField)* ','?)? '}' + +RecordField = + Attr* Visibility? + Name ':' Type + +TupleFieldList = + '(' fields:(TupleField (',' TupleField)* ','?)? ')' + +TupleField = + Attr* Visibility? + Type + +FieldList = + RecordFieldList +| TupleFieldList + +Enum = + Attr* Visibility? + 'enum' Name GenericParamList? WhereClause? + VariantList + +VariantList = + '{' (Variant (',' Variant)* ','?)? '}' + +Variant = + Attr* Visibility? + Name FieldList ('=' Expr)? + +Union = + Attr* Visibility? + 'union' Name GenericParamList? WhereClause? + RecordFieldList + +AdtDef = + Enum +| Struct +| Union + +Const = + Attr* Visibility? + 'default'? + 'const' (Name | '_') ':' Type + '=' body:Expr ';' + +Static = + Attr* Visibility? + 'static'? 'mut'? Name ':' Type + '=' body:Expr ';' + +Trait = + Attr* Visibility? + 'unsafe'? 'auto'? + 'trait' Name GenericParamList (':' TypeBoundList?)? WhereClause + AssocItemList + +AssocItemList = + '{' Attr* AssocItem* '}' + +AssocItem = + Const +| Fn +| MacroCall +| TypeAlias + +Impl = + Attr* Visibility? + 'default'? 'unsafe'? + 'impl' 'const'? GenericParamList? ('!'? trait:Type 'for')? self_ty:Type WhereClause? + AssocItemList + +ExternBlock = + Attr* Abi ExternItemList + +ExternItemList = + '{' Attr* ExternItem* '}' + +ExternItem = + Fn | Static | MacroCall + +GenericParamList = + '<' (GenericParam (',' GenericParam)* ','?)? '>' + +GenericParam = + ConstParam +| LifetimeParam +| TypeParam + +TypeParam = + Attr* Name (':' TypeBoundList?)? + ('=' default_type:Type)? + +ConstParam = + Attr* 'const' Name ':' Type + ('=' default_val:Expr)? + +LifetimeParam = + Attr* 'lifetime' (':' TypeBoundList?)? + +WhereClause = + 'where' predicates:(WherePred (',' WherePred)* ','?) + +WherePred = + ('for' GenericParamList)? ('lifetime' | Type) ':' TypeBoundList + +Visibility = + 'pub' ('(' + 'super' + | 'self' + | 'crate' + | 'in' Path + ')')? + +Attr = + '#' '!'? '[' Path ('=' Literal | TokenTree)? ']' + +//****************************// +// Statements and Expressions // +//****************************// + +Stmt = + ExprStmt +| Item +| LetStmt + +LetStmt = + Attr* 'let' Pat (':' Type)? + '=' initializer:Expr ';' + +ExprStmt = + Attr* Expr ';'? + +Expr = + ArrayExpr +| AwaitExpr +| BinExpr +| BlockExpr +| BoxExpr +| BreakExpr +| CallExpr +| CastExpr +| ClosureExpr +| ContinueExpr +| EffectExpr +| FieldExpr +| ForExpr +| IfExpr +| IndexExpr +| Literal +| LoopExpr +| MacroCall +| MatchExpr +| MethodCallExpr +| ParenExpr +| PathExpr +| PrefixExpr +| RangeExpr +| RecordExpr +| RefExpr +| ReturnExpr +| TryExpr +| TupleExpr +| WhileExpr + +Literal = + Attr* value:( + 'int_number' | 'float_number' + | 'string' | 'raw_string' + | 'byte_string' | 'raw_byte_string' + | 'true' | 'false' + | 'char' | 'byte' + ) + +PathExpr = + Attr* Path + +BlockExpr = + '{' + Attr* + statements:Stmt* + Expr? + '}' + +RefExpr = + Attr* '&' ('raw' |'mut' | 'const') Expr + +TryExpr = + Attr* Expr '?' + +EffectExpr = + Attr* Label? ('try' | 'unsafe' | 'async') BlockExpr + +PrefixExpr = + Attr* op:('-' | '!' | '*') Expr + +BinExpr = + Attr* + lhs:Expr + op:( + '||' | '&&' + | '==' | '!=' | '<=' | '>=' | '<' | '>' + | '+' | '*' | '-' | '/' | '%' | '<<' | '>>' | '^' | '|' | '&' + | '=' | '+=' | '/=' | '*=' | '%=' | '>>=' | '<<=' | '-=' | '|=' | '&=' | '^=' + ) + rhs:Expr + +CastExpr = + Attr* Expr 'as' Type + +ParenExpr = + Attr* '(' Attr* Expr ')' + +ArrayExpr = + Attr* '[' Attr* ( + (Expr (',' Expr)* ','?)? + | Expr ';' Expr + ) ']' + +IndexExpr = + Attr* base:Expr '[' index:Expr ']' + +TupleExpr = + Attr* '(' Attr* fields:(Expr (',' Expr)* ','?)? ')' + +RecordExpr = + Path RecordExprFieldList + +RecordExprFieldList = + '{' + Attr* + fields:(RecordExprField (',' RecordExprField)* ','?) + ('..' spread:Expr)? + '}' + +RecordExprField = + Attr* NameRef (':' Expr)? + +CallExpr = + Attr* Expr ArgList + +ArgList = + '(' args:(Expr (',' Expr)* ','?)? ')' + +MethodCallExpr = + Attr* Expr '.' NameRef GenericArgList? ArgList + +FieldExpr = + Attr* Expr '.' NameRef + +ClosureExpr = + Attr* 'static'? 'async'? 'move'? ParamList RetType? + body:Expr + +IfExpr = + Attr* 'if' Condition then_branch:BlockExpr + ('else' else_branch:(IfExpr | BlockExpr))? + +Condition = + 'let' Pat '=' Expr +| Expr + +LoopExpr = + Attr* Label? 'loop' + loop_body:BlockExpr + +ForExpr = + Attr* Label? 'for' Pat 'in' iterable:Expr + loop_body:BlockExpr + +WhileExpr = + Attr* Label? 'while' Condition + loop_body:BlockExpr + +Label = + 'lifetime' + +BreakExpr = + Attr* 'break' 'lifetime'? Expr? + +ContinueExpr = + Attr* 'continue' 'lifetime'? + +RangeExpr = + Attr* start:Expr? op:('..' | '..=') end:Expr? + +MatchExpr = + Attr* 'match' Expr MatchArmList + +MatchArmList = + '{' + Attr* + arms:MatchArm* + '}' + +MatchArm = + Attr* Pat guard:MatchGuard? '=>' Expr ','? + +MatchGuard = + 'if' Expr + +ReturnExpr = + Attr* 'return' Expr? + +AwaitExpr = + Attr* Expr '.' 'await' + +BoxExpr = + Attr* 'box' Expr + +//*************************// +// Types // +//*************************// + +Type = + ArrayType +| DynTraitType +| FnPointerType +| ForType +| ImplTraitType +| InferType +| NeverType +| ParenType +| PathType +| PointerType +| ReferenceType +| SliceType +| TupleType + +ParenType = + '(' Type ')' + +NeverType = + '!' + +PathType = + Path + +TupleType = + '(' fields:(Type (',' Type)* ','?)? ')' + +PointerType = + '*' ('const' | 'mut') Type + +ReferenceType = + '&' 'lifetime'? 'mut'? Type + +ArrayType = + '[' Type ';' Expr ']' + +SliceType = + '[' Type ']' + +InferType = + '_' + +FnPointerType = + 'const'? 'async'? 'unsafe'? Abi? 'fn' ParamList RetType? + +ForType = + 'for' GenericParamList Type + +ImplTraitType = + 'impl' TypeBoundList + +DynTraitType = + 'dyn' TypeBoundList + +TypeBoundList = + bounds:(TypeBound ('+' TypeBound)* '+'?) + +TypeBound = + 'lifetime' +| '?'? Type + +//************************// +// Patterns // +//************************// + +Pat = + IdentPat +| BoxPat +| RestPat +| LiteralPat +| MacroPat +| OrPat +| ParenPat +| PathPat +| WildcardPat +| RangePat +| RecordPat +| RefPat +| SlicePat +| TuplePat +| TupleStructPat + +LiteralPat = + Literal + +IdentPat = + Attr* 'ref'? 'mut'? Name ('@' Pat)? + +WildcardPat = + '_' + +RangePat = + start:Pat op:('..' | '..=') end:Pat + +RefPat = + '&' 'mut'? Pat + +RecordPat = + Path RecordPatFieldList + +RecordPatFieldList = + '{' + fields:(RecordPatField (',' RecordPatField)* ','?) + '..'? + '}' + +RecordPatField = + Attr* (NameRef ':')? Pat + +TupleStructPat = + Path '(' fields:(Pat (',' Pat)* ','?)? ')' + +TuplePat = + '(' fields:(Pat (',' Pat)* ','?)? ')' + +ParenPat = + '(' Pat ')' + +SlicePat = + '[' (Pat (',' Pat)* ','?)? ']' + +PathPat = + Path + +OrPat = + (Pat ('|' Pat)* '|'?) + +BoxPat = + 'box' Pat + +RestPat = + '..' + +MacroPat = + MacroCall diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000000..8d1738dbf3 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,45 @@ +//! Boilerplate error definitions. +use std::fmt; + +use crate::lexer::Location; + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub struct Error { + pub(crate) message: String, + pub(crate) location: Option, +} + +impl Error { + pub(crate) fn with_location(self, location: Location) -> Error { + Error { + location: Some(location), + ..self + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if let Some(loc) = self.location { + write!(f, "{}:{}: ", loc.line, loc.column)? + } + write!(f, "{}", self.message) + } +} + +macro_rules! _format_err { + ($($tt:tt)*) => { + $crate::error::Error { + message: format!($($tt)*), + location: None, + } + }; +} +pub(crate) use _format_err as format_err; + +macro_rules! _bail { + ($($tt:tt)*) => { return Err($crate::error::format_err!($($tt)*)) }; +} +pub(crate) use _bail as bail; diff --git a/src/lexer.rs b/src/lexer.rs new file mode 100644 index 0000000000..f4c979b5bd --- /dev/null +++ b/src/lexer.rs @@ -0,0 +1,129 @@ +//! Simple hand-written ungrammar lexer +use crate::error::{bail, Result}; + +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum TokenKind { + Node(String), + Token(String), + Eq, + Star, + Pipe, + QMark, + Colon, + LParen, + RParen, +} + +#[derive(Debug)] +pub(crate) struct Token { + pub(crate) kind: TokenKind, + pub(crate) loc: Location, +} + +#[derive(Copy, Clone, Default, Debug)] +pub(crate) struct Location { + pub(crate) line: usize, + pub(crate) column: usize, +} + +impl Location { + fn advance(&mut self, text: &str) { + match text.rfind('\n') { + Some(idx) => { + self.line += text.chars().filter(|&it| it == '\n').count(); + self.column = text[idx + 1..].chars().count(); + } + None => self.column += text.chars().count(), + } + } +} + +pub(crate) fn tokenize(mut input: &str) -> Result> { + let mut res = Vec::new(); + let mut loc = Location::default(); + while !input.is_empty() { + let old_input = input; + skip_ws(&mut input); + skip_comment(&mut input); + if old_input.len() == input.len() { + match advance(&mut input) { + Ok(kind) => { + res.push(Token { kind, loc }); + } + Err(err) => return Err(err.with_location(loc)), + } + } + let consumed = old_input.len() - input.len(); + loc.advance(&old_input[..consumed]); + } + + Ok(res) +} + +fn skip_ws(input: &mut &str) { + *input = input.trim_start_matches(is_whitespace) +} +fn skip_comment(input: &mut &str) { + if input.starts_with("//") { + let idx = input.find('\n').map_or(input.len(), |it| it + 1); + *input = &input[idx..] + } +} + +fn advance(input: &mut &str) -> Result { + let mut chars = input.chars(); + let c = chars.next().unwrap(); + let res = match c { + '=' => TokenKind::Eq, + '*' => TokenKind::Star, + '?' => TokenKind::QMark, + '(' => TokenKind::LParen, + ')' => TokenKind::RParen, + '|' => TokenKind::Pipe, + ':' => TokenKind::Colon, + '\'' => { + let mut buf = String::new(); + loop { + match chars.next() { + None => bail!("unclosed token literal"), + Some('\\') => match chars.next() { + Some(c) if is_escapable(c) => buf.push(c), + _ => bail!("invalid escape in token literal"), + }, + Some('\'') => break, + Some(c) => buf.push(c), + } + } + TokenKind::Token(buf) + } + c if is_ident_char(c) => { + let mut buf = String::new(); + buf.push(c); + loop { + match chars.clone().next() { + Some(c) if is_ident_char(c) => { + chars.next(); + buf.push(c); + } + _ => break, + } + } + TokenKind::Node(buf) + } + '\r' => bail!("unexpected `\\r`, only Unix-style line endings allowed"), + c => bail!("unexpected character: `{}`", c), + }; + + *input = chars.as_str(); + Ok(res) +} + +fn is_escapable(c: char) -> bool { + matches!(c, '\\' | '\'') +} +fn is_whitespace(c: char) -> bool { + matches!(c, ' ' | '\t' | '\n') +} +fn is_ident_char(c: char) -> bool { + matches!(c, 'a'..='z' | 'A'..='Z' | '_') +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000000..ff56cae9ee --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,89 @@ +//! Ungrammar -- a DSL for specifying concrete syntax tree grammar. +//! +//! Producing a parser is an explicit non-goal -- it's ok for this grammar to be +//! ambiguous, non LL, non LR, etc. +mod error; +mod lexer; +mod parser; + +use std::{ops, str::FromStr}; + +pub use error::{Error, Result}; + +pub fn rust_grammar() -> Grammar { + let src = include_str!("../rust.ungram"); + src.parse().unwrap() +} + +#[derive(Eq, PartialEq, Debug, Copy, Clone)] +pub struct Node(usize); +#[derive(Eq, PartialEq, Debug, Copy, Clone)] +pub struct Token(usize); + +#[derive(Default, Debug)] +pub struct Grammar { + nodes: Vec, + tokens: Vec, +} + +impl FromStr for Grammar { + type Err = Error; + fn from_str(s: &str) -> Result { + let tokens = lexer::tokenize(s)?; + parser::parse(tokens) + } +} + +impl Grammar { + pub fn iter(&self) -> impl Iterator + '_ { + (0..self.nodes.len()).map(Node) + } +} + +impl ops::Index for Grammar { + type Output = NodeData; + fn index(&self, Node(index): Node) -> &NodeData { + &self.nodes[index] + } +} + +impl ops::Index for Grammar { + type Output = TokenData; + fn index(&self, Token(index): Token) -> &TokenData { + &self.tokens[index] + } +} + +#[derive(Debug)] +pub struct NodeData { + pub name: String, + pub rule: Rule, +} + +#[derive(Debug)] +pub struct TokenData { + pub name: String, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum Rule { + Labeled { label: String, rule: Box }, + Node(Node), + Token(Token), + Seq(Vec), + Alt(Vec), + Opt(Box), + Rep(Box), +} + +#[test] +fn smoke() { + let grammar = include_str!("../ungrammar.ungram"); + let grammar = grammar.parse::().unwrap(); + drop(grammar) +} + +#[test] +fn test_rust_grammar() { + let _ = rust_grammar(); +} diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000000..bd067f22a5 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,217 @@ +//! Simple hand-written ungrammar parser. +use std::collections::HashMap; + +use crate::{ + error::{bail, format_err, Result}, + lexer::{self, TokenKind}, + Grammar, Node, NodeData, Rule, Token, TokenData, +}; + +macro_rules! bail { + ($loc:expr, $($tt:tt)*) => {{ + let err = $crate::error::format_err!($($tt)*) + .with_location($loc); + return Err(err); + }}; +} + +pub(crate) fn parse(tokens: Vec) -> Result { + let mut p = Parser::new(tokens); + while !p.is_eof() { + node(&mut p)?; + } + p.finish() +} + +#[derive(Default)] +struct Parser { + grammar: Grammar, + tokens: Vec, + node_table: HashMap, + token_table: HashMap, +} + +const DUMMY_RULE: Rule = Rule::Node(Node(!0)); + +impl Parser { + fn new(mut tokens: Vec) -> Parser { + tokens.reverse(); + Parser { + tokens, + ..Parser::default() + } + } + + fn peek(&self) -> Option<&lexer::Token> { + self.peek_n(0) + } + fn peek_n(&self, n: usize) -> Option<&lexer::Token> { + self.tokens.iter().nth_back(n) + } + fn bump(&mut self) -> Result { + self.tokens + .pop() + .ok_or_else(|| format_err!("unexpected EOF")) + } + fn expect(&mut self, kind: TokenKind, what: &str) -> Result<()> { + let token = self.bump()?; + if token.kind != kind { + bail!(token.loc, "unexpected token, expected `{}`", what); + } + Ok(()) + } + fn is_eof(&self) -> bool { + self.tokens.is_empty() + } + fn finish(self) -> Result { + for node_data in &self.grammar.nodes { + if matches!(node_data.rule, DUMMY_RULE) { + crate::error::bail!("Undefined node: {}", node_data.name) + } + } + Ok(self.grammar) + } + fn intern_node(&mut self, name: String) -> Node { + let len = self.node_table.len(); + let grammar = &mut self.grammar; + *self.node_table.entry(name.clone()).or_insert_with(|| { + grammar.nodes.push(NodeData { + name, + rule: DUMMY_RULE, + }); + Node(len) + }) + } + fn intern_token(&mut self, name: String) -> Token { + let len = self.token_table.len(); + let grammar = &mut self.grammar; + *self.token_table.entry(name.clone()).or_insert_with(|| { + grammar.tokens.push(TokenData { name }); + Token(len) + }) + } +} + +fn node(p: &mut Parser) -> Result<()> { + let token = p.bump()?; + let node = match token.kind { + TokenKind::Node(it) => p.intern_node(it), + _ => bail!(token.loc, "expected ident"), + }; + p.expect(TokenKind::Eq, "=")?; + if !matches!(p.grammar[node].rule, DUMMY_RULE) { + bail!(token.loc, "duplicate rule: `{}`", p.grammar[node].name) + } + + let rule = rule(p)?; + p.grammar.nodes[node.0].rule = rule; + Ok(()) +} + +fn rule(p: &mut Parser) -> Result { + let lhs = seq_rule(p)?; + let mut alt = vec![lhs]; + while let Some(token) = p.peek() { + if token.kind != TokenKind::Pipe { + break; + } + p.bump()?; + let rule = seq_rule(p)?; + alt.push(rule) + } + let res = if alt.len() == 1 { + alt.pop().unwrap() + } else { + Rule::Alt(alt) + }; + Ok(res) +} + +fn seq_rule(p: &mut Parser) -> Result { + let lhs = atom_rule(p)?; + + let mut seq = vec![lhs]; + while let Some(rule) = opt_atom_rule(p)? { + seq.push(rule) + } + let res = if seq.len() == 1 { + seq.pop().unwrap() + } else { + Rule::Seq(seq) + }; + Ok(res) +} + +fn atom_rule(p: &mut Parser) -> Result { + match opt_atom_rule(p)? { + Some(it) => Ok(it), + None => { + let token = p.bump()?; + bail!(token.loc, "unexpected token") + } + } +} + +fn opt_atom_rule(p: &mut Parser) -> Result> { + let token = match p.peek() { + Some(it) => it, + None => return Ok(None), + }; + let mut res = match &token.kind { + TokenKind::Node(name) => { + if let Some(lookahead) = p.peek_n(1) { + match lookahead.kind { + TokenKind::Eq => return Ok(None), + TokenKind::Colon => { + let label = name.clone(); + p.bump()?; + p.bump()?; + let rule = atom_rule(p)?; + let res = Rule::Labeled { + label, + rule: Box::new(rule), + }; + return Ok(Some(res)); + } + _ => (), + } + } + match p.peek_n(1) { + Some(token) if token.kind == TokenKind::Eq => return Ok(None), + _ => (), + } + let name = name.clone(); + p.bump()?; + let node = p.intern_node(name); + Rule::Node(node) + } + TokenKind::Token(name) => { + let name = name.clone(); + p.bump()?; + let token = p.intern_token(name); + Rule::Token(token) + } + TokenKind::LParen => { + p.bump()?; + let rule = rule(p)?; + p.expect(TokenKind::RParen, ")")?; + rule + } + _ => return Ok(None), + }; + + if let Some(token) = p.peek() { + match &token.kind { + TokenKind::QMark => { + p.bump()?; + res = Rule::Opt(Box::new(res)); + } + TokenKind::Star => { + p.bump()?; + res = Rule::Rep(Box::new(res)); + } + _ => (), + } + } + Ok(Some(res)) +} diff --git a/ungrammar.ungram b/ungrammar.ungram new file mode 100644 index 0000000000..6cb4e10fb1 --- /dev/null +++ b/ungrammar.ungram @@ -0,0 +1,16 @@ +/// ungrammar for ungrammar +Grammar = + Node * + +Node = + name:'ident' '=' Rule + +Rule = + 'ident' +| 'token_ident' +| Rule * +| Rule ( '|' Rule) * +| Rule '?' +| Rule '*' +| '(' Rule ')' +| label:'ident' ':' From a5cf3c3422ccd8895581e33741abcae90f04ac31 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 01:50:19 +0200 Subject: [PATCH 02/84] ci --- .github/workflows/ci.yaml | 38 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ bors.toml | 2 ++ 3 files changed, 42 insertions(+) create mode 100644 .github/workflows/ci.yaml create mode 100644 bors.toml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..8b170eb62e --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,38 @@ +name: CI +on: + pull_request: + push: + branches: + - master + - staging + - trying + +env: + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + CI: 1 + RUST_BACKTRACE: short + RUSTFLAGS: -D warnings + RUSTUP_MAX_RETRIES: 10 + +jobs: + rust: + name: Rust + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + override: true + + - name: Compile + run: cargo test --no-run + + - name: Test + run: cargo test diff --git a/Cargo.toml b/Cargo.toml index 53e92506e9..32a03ef44b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,7 @@ repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] edition = "2018" +exclude = ["/bors.toml", "/.github"] + [dependencies] # nope diff --git a/bors.toml b/bors.toml new file mode 100644 index 0000000000..b92b99ac30 --- /dev/null +++ b/bors.toml @@ -0,0 +1,2 @@ +status = [ "Rust" ] +delete_merged_branches = true From 7e073b08bc29621e4fc51bfbdc3742e54cc102e4 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 02:02:21 +0200 Subject: [PATCH 03/84] Allow empty statement --- rust.ungram | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index ef5da72622..c0855f5fbd 100644 --- a/rust.ungram +++ b/rust.ungram @@ -289,7 +289,8 @@ Attr = //****************************// Stmt = - ExprStmt + ';' +| ExprStmt | Item | LetStmt From d7a03cae259be1920b4f19d1faee3de515088d6d Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 04:44:24 +0200 Subject: [PATCH 04/84] Add ungrammar2json tool --- .github/workflows/ci.yaml | 4 +- Cargo.toml | 5 ++- src/error.rs | 20 +++++----- ungrammar2json/Cargo.toml | 12 ++++++ ungrammar2json/src/main.rs | 77 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 ungrammar2json/Cargo.toml create mode 100644 ungrammar2json/src/main.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8b170eb62e..f03f3a9fd6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -32,7 +32,7 @@ jobs: override: true - name: Compile - run: cargo test --no-run + run: cargo test --workspace --no-run - name: Test - run: cargo test + run: cargo test --workspace diff --git a/Cargo.toml b/Cargo.toml index 32a03ef44b..cd62884e03 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.0.0" +version = "1.1.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] @@ -9,5 +9,8 @@ edition = "2018" exclude = ["/bors.toml", "/.github"] +[workspace] +members = ["ungrammar2json"] + [dependencies] # nope diff --git a/src/error.rs b/src/error.rs index 8d1738dbf3..a7a62d0cc0 100644 --- a/src/error.rs +++ b/src/error.rs @@ -11,15 +11,6 @@ pub struct Error { pub(crate) location: Option, } -impl Error { - pub(crate) fn with_location(self, location: Location) -> Error { - Error { - location: Some(location), - ..self - } - } -} - impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(loc) = self.location { @@ -29,6 +20,17 @@ impl fmt::Display for Error { } } +impl std::error::Error for Error {} + +impl Error { + pub(crate) fn with_location(self, location: Location) -> Error { + Error { + location: Some(location), + ..self + } + } +} + macro_rules! _format_err { ($($tt:tt)*) => { $crate::error::Error { diff --git a/ungrammar2json/Cargo.toml b/ungrammar2json/Cargo.toml new file mode 100644 index 0000000000..19ca3d8324 --- /dev/null +++ b/ungrammar2json/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ungrammar2json" +description = "Convert ungrammar files to JSON" +version = "1.0.0" +license = "MIT OR Apache-2.0" +repository = "https://github.com/matklad/ungrammar" +authors = ["Aleksey Kladov "] +edition = "2018" + +[dependencies] +write-json = "0.1.1" +ungrammar = { path = "../", version = "1.1.0" } diff --git a/ungrammar2json/src/main.rs b/ungrammar2json/src/main.rs new file mode 100644 index 0000000000..f588ed5eb6 --- /dev/null +++ b/ungrammar2json/src/main.rs @@ -0,0 +1,77 @@ +use std::{ + env, + io::{self, Read}, + process, +}; + +use ungrammar::{Grammar, Rule}; + +fn main() { + if let Err(err) = try_main() { + eprintln!("{}", err); + process::exit(101); + } +} + +fn try_main() -> io::Result<()> { + if env::args().count() != 1 { + eprintln!("Usage: ungrammar2json < grammar.ungram > grammar.json"); + return Ok(()); + } + let grammar = read_stdin()?; + let grammar = grammar + .parse::() + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + + let mut buf = String::new(); + grammar_to_json(&grammar, write_json::object(&mut buf)); + println!("{}", buf); + Ok(()) +} + +fn read_stdin() -> io::Result { + let mut buf = String::new(); + io::stdin().lock().read_to_string(&mut buf)?; + Ok(buf) +} + +fn grammar_to_json(grammar: &Grammar, mut obj: write_json::Object<'_>) { + for node in grammar.iter() { + let node = &grammar[node]; + rule_to_json(grammar, &node.rule, obj.object(&node.name)); + } +} + +fn rule_to_json(grammar: &Grammar, rule: &Rule, mut obj: write_json::Object) { + match rule { + Rule::Labeled { label, rule } => { + obj.string("label", label); + rule_to_json(grammar, rule, obj.object("rule")) + } + Rule::Node(node) => { + obj.string("node", &grammar[*node].name); + } + Rule::Token(token) => { + obj.string("token", &grammar[*token].name); + } + Rule::Seq(rules) | Rule::Alt(rules) => { + let tag = match rule { + Rule::Seq(_) => "seq", + Rule::Alt(_) => "alt", + _ => unreachable!(), + }; + let mut array = obj.array(tag); + for rule in rules { + rule_to_json(grammar, rule, array.object()); + } + } + Rule::Opt(arg) | Rule::Rep(arg) => { + let tag = match rule { + Rule::Opt(_) => "opt", + Rule::Rep(_) => "rep", + _ => unreachable!(), + }; + rule_to_json(grammar, arg, obj.object(tag)); + } + } +} From 7671fa26df0086cb6c212f256f6f40754dc490e5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 13:05:26 +0200 Subject: [PATCH 05/84] Fmt --- rust.ungram | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index c0855f5fbd..b4c4f93ec7 100644 --- a/rust.ungram +++ b/rust.ungram @@ -246,7 +246,9 @@ ExternItemList = '{' Attr* ExternItem* '}' ExternItem = - Fn | Static | MacroCall + Fn +| MacroCall +| Static GenericParamList = '<' (GenericParam (',' GenericParam)* ','?)? '>' From eacfce808cec9d0ab5921f7cc870092f3fe119ca Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 13:06:08 +0200 Subject: [PATCH 06/84] Allow empty fields --- rust.ungram | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust.ungram b/rust.ungram index b4c4f93ec7..5b6dccd769 100644 --- a/rust.ungram +++ b/rust.ungram @@ -401,7 +401,7 @@ RecordExpr = RecordExprFieldList = '{' Attr* - fields:(RecordExprField (',' RecordExprField)* ','?) + fields:(RecordExprField (',' RecordExprField)* ','?)? ('..' spread:Expr)? '}' @@ -586,7 +586,7 @@ RecordPat = RecordPatFieldList = '{' - fields:(RecordPatField (',' RecordPatField)* ','?) + fields:(RecordPatField (',' RecordPatField)* ','?)? '..'? '}' From 2ad534aa4e88df5ab87cb104177e1e6709e640bc Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 13:13:00 +0200 Subject: [PATCH 07/84] Shorten Pointer -> Ptr --- rust.ungram | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust.ungram b/rust.ungram index 5b6dccd769..f39a120122 100644 --- a/rust.ungram +++ b/rust.ungram @@ -487,14 +487,14 @@ BoxExpr = Type = ArrayType | DynTraitType -| FnPointerType +| FnPtrType | ForType | ImplTraitType | InferType | NeverType | ParenType | PathType -| PointerType +| PtrType | ReferenceType | SliceType | TupleType @@ -511,7 +511,7 @@ PathType = TupleType = '(' fields:(Type (',' Type)* ','?)? ')' -PointerType = +PtrType = '*' ('const' | 'mut') Type ReferenceType = @@ -526,7 +526,7 @@ SliceType = InferType = '_' -FnPointerType = +FnPtrType = 'const'? 'async'? 'unsafe'? Abi? 'fn' ParamList RetType? ForType = From d16b5cb88417b2c2aa445ab41883db420cb97e24 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 13:23:32 +0200 Subject: [PATCH 08/84] Shorten ReferenceType -> RefType --- rust.ungram | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust.ungram b/rust.ungram index f39a120122..a7dcce4789 100644 --- a/rust.ungram +++ b/rust.ungram @@ -495,7 +495,7 @@ Type = | ParenType | PathType | PtrType -| ReferenceType +| RefType | SliceType | TupleType @@ -514,7 +514,7 @@ TupleType = PtrType = '*' ('const' | 'mut') Type -ReferenceType = +RefType = '&' 'lifetime'? 'mut'? Type ArrayType = From da1aa8a99738c1a4fe5e321e39aae857c1eb54c8 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 1 Aug 2020 13:46:46 +0200 Subject: [PATCH 09/84] Bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index cd62884e03..af7e29238f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.1.0" +version = "1.1.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From 71350085009cad6ed195d6e8e398b94b40d02153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Mon, 3 Aug 2020 22:32:36 +0100 Subject: [PATCH 10/84] Typo fix --- rust.ungram | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust.ungram b/rust.ungram index a7dcce4789..533539fed5 100644 --- a/rust.ungram +++ b/rust.ungram @@ -1,8 +1,8 @@ // Rust Un-Grammar. // -// This grammar specifies the structure of Rust's concrete sytnax tree. +// This grammar specifies the structure of Rust's concrete syntax tree. // It does not specify parsing rules (ambiguities, precedence, etc are out of scope). -// Tokens are processed -- contextual keywords are recogniesed, compound operators glued. +// Tokens are processed -- contextual keywords are recognised, compound operators glued. // // Legend: // From e5430cd24328acffbd069bc7ce386021308f1366 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Duarte?= Date: Mon, 3 Aug 2020 22:40:54 +0100 Subject: [PATCH 11/84] Another typo fix --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 533539fed5..94890002d5 100644 --- a/rust.ungram +++ b/rust.ungram @@ -7,7 +7,7 @@ // Legend: // // // -- comment -// Name = -- non-termial defition +// Name = -- non-terminal definition // 'ident' -- token (terminal) // A B -- sequence // A | B -- alternation From 047f7c2de448024f31ccf5e94e8b532c978a3d86 Mon Sep 17 00:00:00 2001 From: Jeff Smits Date: Fri, 7 Aug 2020 14:47:52 +0200 Subject: [PATCH 12/84] Make type in Param optional I assumer that's why there were already parentheses around the `':' Type`? --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 94890002d5..49379322be 100644 --- a/rust.ungram +++ b/rust.ungram @@ -146,7 +146,7 @@ SelfParam = Param = Attr* ( - Pat (':' Type) + Pat (':' Type)? | Type | '...' ) From fe7ac06f08b3ed5aae7d4f03d11ba97f7d86c6d9 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 21 Aug 2020 19:04:57 +0200 Subject: [PATCH 13/84] Allow pipes in parameters closes #8 --- rust.ungram | 1 + 1 file changed, 1 insertion(+) diff --git a/rust.ungram b/rust.ungram index 49379322be..7343c39251 100644 --- a/rust.ungram +++ b/rust.ungram @@ -137,6 +137,7 @@ ParamList = SelfParam | (SelfParam ',')? (Param (',' Param)* ','?)? )')' +| '|' (Param (',' Param)* ','?)? '|' SelfParam = Attr* ( From 01ab701c16c9a85106b05cfc7b6e08cf8971c949 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 21 Aug 2020 19:07:34 +0200 Subject: [PATCH 14/84] Name MethodCall's receiver --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 7343c39251..1f3a70529b 100644 --- a/rust.ungram +++ b/rust.ungram @@ -416,7 +416,7 @@ ArgList = '(' args:(Expr (',' Expr)* ','?)? ')' MethodCallExpr = - Attr* Expr '.' NameRef GenericArgList? ArgList + Attr* receiver:Expr '.' NameRef GenericArgList? ArgList FieldExpr = Attr* Expr '.' NameRef From f9a230c9e8ae415565c6eddb0ef55c239a81915e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Fri, 21 Aug 2020 19:08:56 +0200 Subject: [PATCH 15/84] v1.1.2 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index af7e29238f..da42bb449c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.1.1" +version = "1.1.2" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From ebf2403a66969c4d24001fd4120b63fda347f426 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 24 Aug 2020 21:19:59 +0200 Subject: [PATCH 16/84] Allow type aliases in extern blocks --- rust.ungram | 1 + 1 file changed, 1 insertion(+) diff --git a/rust.ungram b/rust.ungram index 1f3a70529b..9e9f82a43e 100644 --- a/rust.ungram +++ b/rust.ungram @@ -250,6 +250,7 @@ ExternItem = Fn | MacroCall | Static +| TypeAlias GenericParamList = '<' (GenericParam (',' GenericParam)* ','?)? '>' From 4640b2ef74cf07cd3d9500f08e8dcf2ef4765d52 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 24 Aug 2020 21:50:33 +0200 Subject: [PATCH 17/84] Bump to 1.1.3 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index da42bb449c..f17b3cda53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.1.2" +version = "1.1.3" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From b6d0d743f2ac28e9dba78fd04cb11aec9a5fa02a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 24 Aug 2020 23:29:43 +0200 Subject: [PATCH 18/84] Continious release --- .github/ci.rs | 114 ++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yaml | 8 +-- 2 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 .github/ci.rs diff --git a/.github/ci.rs b/.github/ci.rs new file mode 100644 index 0000000000..ff37c746bd --- /dev/null +++ b/.github/ci.rs @@ -0,0 +1,114 @@ +use std::{ + env, fs, + process::{self, Command, ExitStatus, Stdio}, + time::Instant, +}; + +type Error = Box; +type Result = std::result::Result; + +fn main() { + if let Err(err) = try_main() { + eprintln!("{}", err); + process::exit(1); + } +} + +fn try_main() -> Result<()> { + let cwd = env::current_dir()?; + let cargo_toml = cwd.join("Cargo.toml"); + assert!( + cargo_toml.exists(), + "Cargo.toml not found, cwd: {}", + cwd.display() + ); + + { + let _s = Section::new("BUILD"); + shell("cargo test --workspace --no-run")?; + } + + { + let _s = Section::new("TEST"); + shell("cargo test")?; + } + + let current_branch = shell_output("git branch --show-current")?; + if ¤t_branch == "master" { + let _s = Section::new("PUBLISH"); + let manifest = fs::read_to_string(&cargo_toml)?; + let version = get_field(&manifest, "version")?; + let tag = format!("v{}", version); + let tags = shell_output("git tag --list")?; + + if !tags.contains(&tag) { + let token = env::var("CRATES_IO_TOKEN").unwrap(); + shell(&format!("git tag v{}", version))?; + shell(&format!("cargo publish --token {}", token))?; + shell("git push --tags")?; + } + } + Ok(()) +} + +fn get_field<'a>(text: &'a str, name: &str) -> Result<&'a str> { + for line in text.lines() { + let words = line.split_ascii_whitespace().collect::>(); + match words.as_slice() { + [n, "=", v, ..] if n.trim() == name => { + assert!(v.starts_with('"') && v.ends_with('"')); + return Ok(&v[1..v.len() - 1]); + } + _ => (), + } + } + Err(format!("can't find `{}` in\n----\n{}\n----\n", name, text))? +} + +fn shell(cmd: &str) -> Result<()> { + let status = command(cmd).status()?; + check_status(status) +} + +fn shell_output(cmd: &str) -> Result { + let output = command(cmd).stderr(Stdio::inherit()).output()?; + check_status(output.status)?; + let res = String::from_utf8(output.stdout)?; + Ok(res.trim().to_string()) +} + +fn command(cmd: &str) -> Command { + eprintln!("> {}", cmd); + let words = cmd.split_ascii_whitespace().collect::>(); + let (cmd, args) = words.split_first().unwrap(); + let mut res = Command::new(cmd); + res.args(args); + res +} + +fn check_status(status: ExitStatus) -> Result<()> { + if !status.success() { + Err(format!("$status: {}", status))?; + } + Ok(()) +} + +struct Section { + name: &'static str, + start: Instant, +} + +impl Section { + fn new(name: &'static str) -> Section { + println!("::group::{}", name); + let start = Instant::now(); + Section { name, start } + } +} + +impl Drop for Section { + fn drop(&mut self) { + eprintln!("{}: {:.2?}", self.name, self.start.elapsed()); + println!("::endgroup::"); + } +} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f03f3a9fd6..88f133867e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,8 +31,6 @@ jobs: profile: minimal override: true - - name: Compile - run: cargo test --workspace --no-run - - - name: Test - run: cargo test --workspace + - run: rustc ./.github/ci.rs && ./ci + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} From 84597d486ce0abe64fe2ec03fcb2f2ef36b7dc2b Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 24 Aug 2020 23:31:46 +0200 Subject: [PATCH 19/84] Allow both const & async modifiers closes #9 --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f17b3cda53..3f40dcc648 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.1.3" +version = "1.1.4" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 9e9f82a43e..6ecdb7bfb0 100644 --- a/rust.ungram +++ b/rust.ungram @@ -125,7 +125,7 @@ UseTreeList = Fn = Attr* Visibility? - 'default'? ('async' | 'const')? 'unsafe'? Abi? + 'default'? 'const'? 'async'? 'unsafe'? Abi? 'fn' Name GenericParamList? ParamList RetType? WhereClause? (body:BlockExpr | ';') From aa77837c05cfb84fa7f28c45ba9dab4a25f7d896 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 24 Aug 2020 23:33:18 +0200 Subject: [PATCH 20/84] Fix .gitignore --- .github/ci.rs | 2 +- .gitignore | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/ci.rs b/.github/ci.rs index ff37c746bd..87eb307d63 100644 --- a/.github/ci.rs +++ b/.github/ci.rs @@ -30,7 +30,7 @@ fn try_main() -> Result<()> { { let _s = Section::new("TEST"); - shell("cargo test")?; + shell("cargo test --workspace")?; } let current_branch = shell_output("git branch --show-current")?; diff --git a/.gitignore b/.gitignore index 9c71cc94a0..e3bd43f693 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +/ci /Cargo.lock -/target \ No newline at end of file +/target From 784f345e5e799e828650da1b1acbb947f1e49a52 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 24 Oct 2020 02:05:24 +0200 Subject: [PATCH 21/84] Minor --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 6ecdb7bfb0..d8950d2deb 100644 --- a/rust.ungram +++ b/rust.ungram @@ -164,7 +164,7 @@ TypeAlias = Struct = Attr* Visibility? 'struct' Name GenericParamList? ( - WhereClause? (RecordFieldList | ';') + WhereClause? (RecordFieldList | ';') | TupleFieldList WhereClause? ';' ) From eb7e474d641fdb5210318479696cb7b362190cfd Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 24 Oct 2020 11:52:21 +0200 Subject: [PATCH 22/84] Link post --- README.md | 2 +- src/lib.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b4f8f375ec..ea47622f22 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # ungrammar -A DLS for specifying concrete syntax tree. +A DLS for specifying concrete syntax tree. See this [introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html). See [./rust.ungram](./rust.ungram) for an example. diff --git a/src/lib.rs b/src/lib.rs index ff56cae9ee..2d51dcc5d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,10 @@ //! //! Producing a parser is an explicit non-goal -- it's ok for this grammar to be //! ambiguous, non LL, non LR, etc. +//! +//! See this +//! [introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html) +//! for details. mod error; mod lexer; mod parser; From 9f9a6f0bc96e53961973f848cd218d3474ee0d52 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Fri, 27 Nov 2020 18:50:39 +0100 Subject: [PATCH 23/84] Move towards upstream `macro_rules!` model --- rust.ungram | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index d8950d2deb..1d8769fe84 100644 --- a/rust.ungram +++ b/rust.ungram @@ -58,7 +58,7 @@ ConstArg = Expr MacroCall = - Attr* Path '!' Name? TokenTree ';'? + Attr* Path '!' TokenTree ';'? TokenTree = '(' ')' @@ -89,6 +89,7 @@ Item = | Fn | Impl | MacroCall +| MacroRules | Module | Static | Struct @@ -97,6 +98,14 @@ Item = | Union | Use +MacroRules = + Attr* Visibility? + 'macro_rules' '!' Name + '{' MacroArm (';' MacroArm)* ';'? '}' + +MacroArm = + TokenTree '=>' TokenTree + Module = Attr* Visibility? 'mod' Name From fab616288a6ccf2d16a4e6915efc633d8b9e5583 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 14 Dec 2020 14:54:29 +0100 Subject: [PATCH 24/84] Bump to 1.2.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3f40dcc648..e4f058883b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.1.4" +version = "1.2.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From d41a4110554501314bdfcabe5b41971abda99397 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 14 Dec 2020 15:13:59 +0100 Subject: [PATCH 25/84] Fixup MacroRules to work with xtask codegen --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e4f058883b..6c66b66e5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.2.0" +version = "1.2.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 1d8769fe84..46257f375f 100644 --- a/rust.ungram +++ b/rust.ungram @@ -101,7 +101,7 @@ Item = MacroRules = Attr* Visibility? 'macro_rules' '!' Name - '{' MacroArm (';' MacroArm)* ';'? '}' + '{' (MacroArm (';' MacroArm)* ';'?)? '}' MacroArm = TokenTree '=>' TokenTree From 143cc528b11290bf2463de1fb5a97a15f9b2ed44 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Tue, 15 Dec 2020 15:15:28 +0100 Subject: [PATCH 26/84] Roll back `MacroArm` change It's unclear if this is worthwhile, and this requires a lot of changes in r-a --- Cargo.toml | 2 +- rust.ungram | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6c66b66e5b..c49486f08d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.2.1" +version = "1.2.2" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 46257f375f..ca3cf9292a 100644 --- a/rust.ungram +++ b/rust.ungram @@ -101,10 +101,7 @@ Item = MacroRules = Attr* Visibility? 'macro_rules' '!' Name - '{' (MacroArm (';' MacroArm)* ';'?)? '}' - -MacroArm = - TokenTree '=>' TokenTree + TokenTree Module = Attr* Visibility? From 728247759e38e791f4a398c61cdaa433d22ebadc Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Tue, 15 Dec 2020 18:41:02 +0100 Subject: [PATCH 27/84] Add `MacroDef` for "Macros 2.0" --- rust.ungram | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust.ungram b/rust.ungram index ca3cf9292a..bfa2ade2df 100644 --- a/rust.ungram +++ b/rust.ungram @@ -90,6 +90,7 @@ Item = | Impl | MacroCall | MacroRules +| MacroDef | Module | Static | Struct @@ -103,6 +104,11 @@ MacroRules = 'macro_rules' '!' Name TokenTree +MacroDef = + Attr* Visibility? + 'macro' Name args:TokenTree? + body:TokenTree + Module = Attr* Visibility? 'mod' Name From a188a93e693d875658cc0eaa1843406d819ed3a9 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 14 Dec 2020 18:06:40 +0100 Subject: [PATCH 28/84] Fix labels missing the colon token in rust ungrammar --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index ca3cf9292a..e0142b64cd 100644 --- a/rust.ungram +++ b/rust.ungram @@ -453,7 +453,7 @@ WhileExpr = loop_body:BlockExpr Label = - 'lifetime' + 'lifetime' ':' BreakExpr = Attr* 'break' 'lifetime'? Expr? From 9e81b8bb79dd5c98be9bc7320efcd29cf07785c1 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 14 Dec 2020 18:14:49 +0100 Subject: [PATCH 29/84] Node-ify lifetime --- Cargo.toml | 2 +- rust.ungram | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c49486f08d..533abba2fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.2.2" +version = "1.3.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index e0142b64cd..fdff06a0f2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -26,6 +26,9 @@ Name = NameRef = 'ident' | 'int_number' +Lifetime = + 'lifetime_ident' + Path = (qualifier:Path '::')? segment:PathSegment @@ -52,7 +55,7 @@ AssocTypeArg = NameRef (':' TypeBoundList | '=' Type) LifetimeArg = - 'lifetime' + Lifetime ConstArg = Expr @@ -147,7 +150,7 @@ ParamList = SelfParam = Attr* ( - ('&' 'lifetime'?)? 'mut'? 'self' + ('&' Lifetime?)? 'mut'? 'self' | 'mut'? 'self' ':' Type ) @@ -275,13 +278,13 @@ ConstParam = ('=' default_val:Expr)? LifetimeParam = - Attr* 'lifetime' (':' TypeBoundList?)? + Attr* Lifetime (':' TypeBoundList?)? WhereClause = 'where' predicates:(WherePred (',' WherePred)* ','?) WherePred = - ('for' GenericParamList)? ('lifetime' | Type) ':' TypeBoundList + ('for' GenericParamList)? (Lifetime | Type) ':' TypeBoundList Visibility = 'pub' ('(' @@ -453,13 +456,13 @@ WhileExpr = loop_body:BlockExpr Label = - 'lifetime' ':' + Lifetime ':' BreakExpr = - Attr* 'break' 'lifetime'? Expr? + Attr* 'break' Lifetime? Expr? ContinueExpr = - Attr* 'continue' 'lifetime'? + Attr* 'continue' Lifetime? RangeExpr = Attr* start:Expr? op:('..' | '..=') end:Expr? @@ -523,7 +526,7 @@ PtrType = '*' ('const' | 'mut') Type RefType = - '&' 'lifetime'? 'mut'? Type + '&' Lifetime? 'mut'? Type ArrayType = '[' Type ';' Expr ']' @@ -550,7 +553,7 @@ TypeBoundList = bounds:(TypeBound ('+' TypeBound)* '+'?) TypeBound = - 'lifetime' + Lifetime | '?'? Type //************************// From ccc54ede2345dfc3e272c568163275181f5c4708 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 16 Dec 2020 12:47:15 +0100 Subject: [PATCH 30/84] Bump to 1.3.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c49486f08d..533abba2fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.2.2" +version = "1.3.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From c87f01ec3c2f8af11a0f51445b92889ee5787fb5 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 16 Dec 2020 13:33:00 +0100 Subject: [PATCH 31/84] Bump to 1.4.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 533abba2fe..1ba385676d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.3.0" +version = "1.4.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From a2900d258a4ade4b32f924b84bcbe5f0de8ca901 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 15 Dec 2020 20:58:03 +0100 Subject: [PATCH 32/84] Add inline const expression and pattern --- Cargo.toml | 2 +- rust.ungram | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1ba385676d..10aff55025 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.4.0" +version = "1.5.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index fdb381f98d..a954d9efd2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -372,13 +372,13 @@ BlockExpr = '}' RefExpr = - Attr* '&' ('raw' |'mut' | 'const') Expr + Attr* '&' ('raw' | 'mut' | 'const') Expr TryExpr = Attr* Expr '?' EffectExpr = - Attr* Label? ('try' | 'unsafe' | 'async') BlockExpr + Attr* Label? ('try' | 'unsafe' | 'async' | 'const') BlockExpr PrefixExpr = Attr* op:('-' | '!' | '*') Expr @@ -582,6 +582,7 @@ Pat = | SlicePat | TuplePat | TupleStructPat +| ConstBlockPat LiteralPat = Literal @@ -636,3 +637,6 @@ RestPat = MacroPat = MacroCall + +ConstBlockPat = + 'const' BlockExpr From fe4a8e6f7b6497d700f3f77ca8308ac580101122 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 5 Jan 2021 15:44:45 +0300 Subject: [PATCH 33/84] Rename tail_expr --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 10aff55025..d2d67e9249 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.5.0" +version = "1.6.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index a954d9efd2..d6471d2279 100644 --- a/rust.ungram +++ b/rust.ungram @@ -368,7 +368,7 @@ BlockExpr = '{' Attr* statements:Stmt* - Expr? + tail_expr:Expr? '}' RefExpr = From bf2dc9934912e68a9f8471b4eb2316e83b0db640 Mon Sep 17 00:00:00 2001 From: Daiki Ihara Date: Mon, 11 Jan 2021 20:57:14 +0900 Subject: [PATCH 34/84] Add YieldExpr --- rust.ungram | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rust.ungram b/rust.ungram index d6471d2279..13d5968e35 100644 --- a/rust.ungram +++ b/rust.ungram @@ -351,6 +351,7 @@ Expr = | TryExpr | TupleExpr | WhileExpr +| YieldExpr Literal = Attr* value:( @@ -491,6 +492,9 @@ MatchGuard = ReturnExpr = Attr* 'return' Expr? +YieldExpr = + Attr* 'yield' Expr? + AwaitExpr = Attr* Expr '.' 'await' From c332d9d8d7558f58c53dc850679be596b0054e5f Mon Sep 17 00:00:00 2001 From: Daiki Ihara Date: Tue, 12 Jan 2021 23:39:35 +0900 Subject: [PATCH 35/84] Bump to 1.7.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d2d67e9249..ffc01e3aca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.6.0" +version = "1.7.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From 8ba7eadb46e02575c822be345e1883bf81798e03 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 15 Jan 2021 18:55:01 +0100 Subject: [PATCH 36/84] Replace self/super/crate in PathSegment with NameRef --- Cargo.toml | 2 +- rust.ungram | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ffc01e3aca..2a86a58d83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.7.0" +version = "1.8.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 13d5968e35..cb55060a88 100644 --- a/rust.ungram +++ b/rust.ungram @@ -24,7 +24,7 @@ Name = 'ident' NameRef = - 'ident' | 'int_number' + 'ident' | 'int_number' | 'self' | 'super' | 'crate' Lifetime = 'lifetime_ident' @@ -33,8 +33,7 @@ Path = (qualifier:Path '::')? segment:PathSegment PathSegment = - 'crate' | 'self' | 'super' -| '::'? NameRef + '::'? NameRef | NameRef GenericArgList? | NameRef ParamList RetType? | '<' PathType ('as' PathType)? '>' From 54df3c1e4930463d1518e9ee84cd19c0cfd1beb0 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Fri, 15 Jan 2021 20:19:56 +0100 Subject: [PATCH 37/84] Replace other self/super/crate usages with NameRef --- Cargo.toml | 2 +- rust.ungram | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2a86a58d83..dcb1aea5ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.8.0" +version = "1.9.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index cb55060a88..34377f5e92 100644 --- a/rust.ungram +++ b/rust.ungram @@ -21,7 +21,7 @@ //*************************// Name = - 'ident' + 'ident' | 'self' NameRef = 'ident' | 'int_number' | 'self' | 'super' | 'crate' @@ -121,7 +121,7 @@ ItemList = ExternCrate = Attr* Visibility? - 'extern' 'crate' (NameRef | 'self') Rename? ';' + 'extern' 'crate' NameRef Rename? ';' Rename = 'as' (Name | '_') @@ -155,8 +155,8 @@ ParamList = SelfParam = Attr* ( - ('&' Lifetime?)? 'mut'? 'self' - | 'mut'? 'self' ':' Type + ('&' Lifetime?)? 'mut'? Name + | 'mut'? Name ':' Type ) Param = @@ -292,12 +292,7 @@ WherePred = ('for' GenericParamList)? (Lifetime | Type) ':' TypeBoundList Visibility = - 'pub' ('(' - 'super' - | 'self' - | 'crate' - | 'in' Path - ')')? + 'pub' ('(' 'in'? Path ')')? Attr = '#' '!'? '[' Path ('=' Literal | TokenTree)? ']' From af30e7c517b5b47a4813ee26d227ac3a8d2dbf02 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 18 Jan 2021 15:15:35 +0100 Subject: [PATCH 38/84] `Type` can also be `MacroCall` --- Cargo.toml | 2 +- rust.ungram | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index dcb1aea5ea..80b6af77ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.9.0" +version = "1.9.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 34377f5e92..8dfed606eb 100644 --- a/rust.ungram +++ b/rust.ungram @@ -506,6 +506,7 @@ Type = | ForType | ImplTraitType | InferType +| MacroCall | NeverType | ParenType | PathType From 1959fdfe4ac43642bb24c6b55538ffbdb88e1392 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 18 Jan 2021 16:36:22 +0100 Subject: [PATCH 39/84] Wrap type macro in new `MacroType` node --- Cargo.toml | 2 +- rust.ungram | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 80b6af77ab..252ce0eb07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.9.1" +version = "1.9.2" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 8dfed606eb..0e91b7a849 100644 --- a/rust.ungram +++ b/rust.ungram @@ -506,7 +506,7 @@ Type = | ForType | ImplTraitType | InferType -| MacroCall +| MacroType | NeverType | ParenType | PathType @@ -521,6 +521,9 @@ ParenType = NeverType = '!' +MacroType = + MacroCall + PathType = Path From 075778d56da3be5c67507ad1d29c06d5b5bfc805 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 18 Jan 2021 19:39:19 +0100 Subject: [PATCH 40/84] Swap RecordExprField optional part --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 0e91b7a849..c3b31cb671 100644 --- a/rust.ungram +++ b/rust.ungram @@ -418,7 +418,7 @@ RecordExprFieldList = '}' RecordExprField = - Attr* NameRef (':' Expr)? + Attr* (NameRef ':')? Expr CallExpr = Attr* Expr ArgList From effe644c66163c78b49404cc2c3e0bc990bac475 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Mon, 18 Jan 2021 20:03:04 +0100 Subject: [PATCH 41/84] Correct `const` keyword position in `Impl` rule --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 252ce0eb07..bf4a6fa0ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.9.2" +version = "1.9.3" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index c3b31cb671..fbbc3161c0 100644 --- a/rust.ungram +++ b/rust.ungram @@ -251,7 +251,7 @@ AssocItem = Impl = Attr* Visibility? 'default'? 'unsafe'? - 'impl' 'const'? GenericParamList? ('!'? trait:Type 'for')? self_ty:Type WhereClause? + 'impl' GenericParamList? ('const'? '!'? trait:Type 'for')? self_ty:Type WhereClause? AssocItemList ExternBlock = From 66763c849cce0a3ff42e1a7afa53ba327ebe9573 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 18:46:51 -0800 Subject: [PATCH 42/84] Derive more for Node and Token --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 2d51dcc5d2..d4a5e6a44c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,9 +19,9 @@ pub fn rust_grammar() -> Grammar { src.parse().unwrap() } -#[derive(Eq, PartialEq, Debug, Copy, Clone)] +#[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] pub struct Node(usize); -#[derive(Eq, PartialEq, Debug, Copy, Clone)] +#[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] pub struct Token(usize); #[derive(Default, Debug)] From d873e88012aa8d4d35ae0484296b4aa2c7b6a136 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 18:47:52 -0800 Subject: [PATCH 43/84] Add a tokens iterator --- src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index d4a5e6a44c..228952389f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,10 @@ impl Grammar { pub fn iter(&self) -> impl Iterator + '_ { (0..self.nodes.len()).map(Node) } + + pub fn tokens(&self) -> impl Iterator + '_ { + (0..self.tokens.len()).map(Token) + } } impl ops::Index for Grammar { From 3beaff7e387b0eb194ea31522919ad2fb8993324 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 18:59:29 -0800 Subject: [PATCH 44/84] Write docs --- src/error.rs | 2 ++ src/lib.rs | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index a7a62d0cc0..6cc86f52f9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -3,8 +3,10 @@ use std::fmt; use crate::lexer::Location; +/// A type alias for std's Result with the Error as our error type. pub type Result = std::result::Result; +/// An error encountered when parsing a Grammar. #[derive(Debug)] pub struct Error { pub(crate) message: String, diff --git a/src/lib.rs b/src/lib.rs index 228952389f..7d7c14f535 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,11 @@ //! See this //! [introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html) //! for details. + +#![deny(missing_debug_implementations)] +#![deny(missing_docs)] +#![deny(rust_2018_idioms)] + mod error; mod lexer; mod parser; @@ -14,16 +19,21 @@ use std::{ops, str::FromStr}; pub use error::{Error, Result}; +/// Returns a Rust grammar. pub fn rust_grammar() -> Grammar { let src = include_str!("../rust.ungram"); src.parse().unwrap() } +/// A node, like `A = 'b' | 'c'`. #[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] pub struct Node(usize); + +/// A token, denoted with single quotes, like `'+'` or `'struct'`. #[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] pub struct Token(usize); +/// An Ungrammar grammar. #[derive(Default, Debug)] pub struct Grammar { nodes: Vec, @@ -39,10 +49,12 @@ impl FromStr for Grammar { } impl Grammar { + /// Returns an iterator over all nodes in the grammar. pub fn iter(&self) -> impl Iterator + '_ { (0..self.nodes.len()).map(Node) } + /// Returns an iterator over all tokens in the grammar. pub fn tokens(&self) -> impl Iterator + '_ { (0..self.tokens.len()).map(Token) } @@ -62,25 +74,47 @@ impl ops::Index for Grammar { } } +/// Data about a node. #[derive(Debug)] pub struct NodeData { + /// The name of the node. + /// + /// In the rule `A = 'b' | 'c'`, this is `"A"`. pub name: String, + /// The rule for this node. + /// + /// In the rule `A = 'b' | 'c'`, this represents `'b' | 'c'`. pub rule: Rule, } +/// Data about a token. #[derive(Debug)] pub struct TokenData { + /// The name of the token. pub name: String, } +/// A production rule. #[derive(Debug, Eq, PartialEq)] pub enum Rule { - Labeled { label: String, rule: Box }, + /// A labeled rule, like `a:B` (`"a"` is the label, `B` is the rule). + Labeled { + /// The label. + label: String, + /// The rule. + rule: Box, + }, + /// A node, like `A`. Node(Node), + /// A token, like `'struct'`. Token(Token), + /// A sequence of rules, like `'while' '(' Expr ')' Stmt`. Seq(Vec), + /// An alternative between many rules, like `'+' | '-' | '*' | '/'`. Alt(Vec), + /// An optional rule, like `A?`. Opt(Box), + /// An repeated rule, like `A*`. Rep(Box), } From 63617418291fa757f0a519eadb5f26ac7151a70f Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 18:59:59 -0800 Subject: [PATCH 45/84] Reorder --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7d7c14f535..eb1feaa340 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,11 +26,11 @@ pub fn rust_grammar() -> Grammar { } /// A node, like `A = 'b' | 'c'`. -#[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Node(usize); /// A token, denoted with single quotes, like `'+'` or `'struct'`. -#[derive(Eq, PartialEq, Debug, Copy, Clone, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Token(usize); /// An Ungrammar grammar. From 02be05795f7c2fff200c9df1b3fe4b3b5a4c5099 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 19:07:50 -0800 Subject: [PATCH 46/84] Fix typo --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ea47622f22..a0990d747b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # ungrammar -A DLS for specifying concrete syntax tree. See this [introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html). +A DSL for specifying concrete syntax tree. See this +[introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html). See [./rust.ungram](./rust.ungram) for an example. From ab160cda347d2d068ea2b38f4b8d91c1478b18fd Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 19:20:08 -0800 Subject: [PATCH 47/84] Pluralize --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0990d747b..b5a3f48ab3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ungrammar -A DSL for specifying concrete syntax tree. See this +A DSL for specifying concrete syntax trees. See this [introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html). See [./rust.ungram](./rust.ungram) for an example. From e4ab990f5579933e945f537213fd12b16c3bbcd4 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 19:25:10 -0800 Subject: [PATCH 48/84] Add more doc --- src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index eb1feaa340..858b680c81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,10 +26,16 @@ pub fn rust_grammar() -> Grammar { } /// A node, like `A = 'b' | 'c'`. +/// +/// Indexing into a [`Grammar`] with a [`Node`] returns a reference to a +/// [`NodeData`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Node(usize); /// A token, denoted with single quotes, like `'+'` or `'struct'`. +/// +/// Indexing into a [`Grammar`] with a [`Token`] returns a reference to a +/// [`TokenData`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Token(usize); From b82d2fc7a33d05ff6bfdf5a0b44d2a4c70abf741 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Thu, 21 Jan 2021 19:26:35 -0800 Subject: [PATCH 49/84] Fix typo --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 858b680c81..7aa0ce9c88 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,7 +120,7 @@ pub enum Rule { Alt(Vec), /// An optional rule, like `A?`. Opt(Box), - /// An repeated rule, like `A*`. + /// A repeated rule, like `A*`. Rep(Box), } From e4e81fe54d93774d71091232a06bfa8e81c23234 Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Fri, 22 Jan 2021 01:25:03 -0800 Subject: [PATCH 50/84] Bump minor version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bf4a6fa0ab..539abe6ee3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.9.3" +version = "1.10.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From d0d1fc0cf134996c72f406212ee129db68cbfdaa Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 7 Feb 2021 14:10:38 +0300 Subject: [PATCH 51/84] adt --- Cargo.toml | 2 +- rust.ungram | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 539abe6ee3..108ef72d7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.10.0" +version = "1.11.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index fbbc3161c0..3d2e189edf 100644 --- a/rust.ungram +++ b/rust.ungram @@ -217,7 +217,10 @@ Union = 'union' Name GenericParamList? WhereClause? RecordFieldList -AdtDef = +// A Data Type. +// +// Not used directly in the grammar, but handy to have anyway. +Adt = Enum | Struct | Union From 723a57a8ddb6ff62f1db3dcfc46d182eb30ec96b Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 18 Feb 2021 14:47:43 +0100 Subject: [PATCH 52/84] Split out macro calls in item position --- Cargo.toml | 2 +- rust.ungram | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 108ef72d7c..b983cb414a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.11.0" +version = "1.12.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 3d2e189edf..9a332398b2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -90,7 +90,7 @@ Item = | ExternCrate | Fn | Impl -| MacroCall +| MacroItem | MacroRules | MacroDef | Module @@ -101,6 +101,9 @@ Item = | Union | Use +MacroItem = + MacroCall + MacroRules = Attr* Visibility? 'macro_rules' '!' Name From 190981decd4fb47dfee736377c7f8b4b55ad1949 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 18 Feb 2021 19:18:10 +0100 Subject: [PATCH 53/84] Make ExternItem and AssocItem use MacroItem --- Cargo.toml | 2 +- rust.ungram | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b983cb414a..ec3b5dd7a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.12.0" +version = "1.12.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 9a332398b2..4dbf43c7bd 100644 --- a/rust.ungram +++ b/rust.ungram @@ -251,7 +251,7 @@ AssocItemList = AssocItem = Const | Fn -| MacroCall +| MacroItem | TypeAlias Impl = @@ -268,7 +268,7 @@ ExternItemList = ExternItem = Fn -| MacroCall +| MacroItem | Static | TypeAlias From 49294fec86b64be03d38e9ddbd3defa792c83d9e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 8 Mar 2021 12:59:34 +0800 Subject: [PATCH 54/84] Macro Statements should be expr --- Cargo.toml | 2 +- rust.ungram | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index ec3b5dd7a5..002a38651b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.12.1" +version = "1.12.2" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 4dbf43c7bd..ab65fee7ef 100644 --- a/rust.ungram +++ b/rust.ungram @@ -339,6 +339,7 @@ Expr = | Literal | LoopExpr | MacroCall +| MacroStmts | MatchExpr | MethodCallExpr | ParenExpr From 928c6459424e4625ffd07b7294f6bb6353221163 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 15 Mar 2021 14:44:27 +0100 Subject: [PATCH 55/84] Revert "Make ExternItem and AssocItem use MacroItem" This reverts commit 190981decd4fb47dfee736377c7f8b4b55ad1949. --- Cargo.toml | 2 +- rust.ungram | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ec3b5dd7a5..b983cb414a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.12.1" +version = "1.12.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 4dbf43c7bd..9a332398b2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -251,7 +251,7 @@ AssocItemList = AssocItem = Const | Fn -| MacroItem +| MacroCall | TypeAlias Impl = @@ -268,7 +268,7 @@ ExternItemList = ExternItem = Fn -| MacroItem +| MacroCall | Static | TypeAlias From 113a7ac93a1dfb29cc90ec7b19a2d717118ecbfe Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 15 Mar 2021 14:44:32 +0100 Subject: [PATCH 56/84] Revert "Split out macro calls in item position" This reverts commit 723a57a8ddb6ff62f1db3dcfc46d182eb30ec96b. --- Cargo.toml | 2 +- rust.ungram | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b983cb414a..108ef72d7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.12.0" +version = "1.11.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 9a332398b2..3d2e189edf 100644 --- a/rust.ungram +++ b/rust.ungram @@ -90,7 +90,7 @@ Item = | ExternCrate | Fn | Impl -| MacroItem +| MacroCall | MacroRules | MacroDef | Module @@ -101,9 +101,6 @@ Item = | Union | Use -MacroItem = - MacroCall - MacroRules = Attr* Visibility? 'macro_rules' '!' Name From fab562aabe3a4305cedfb814073864c1455c5e9d Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Mon, 15 Mar 2021 14:46:34 +0100 Subject: [PATCH 57/84] Restore current version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 108ef72d7c..ec3b5dd7a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.11.0" +version = "1.12.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From 9abe6453ccccde2833da97dc5038c4efd895828c Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 18 Mar 2021 22:12:32 +0100 Subject: [PATCH 58/84] extended_key_value_attributes --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 002a38651b..e5c1d501a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.12.2" +version = "1.13.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index e7542b182a..3ebe1cbd32 100644 --- a/rust.ungram +++ b/rust.ungram @@ -298,7 +298,7 @@ Visibility = 'pub' ('(' 'in'? Path ')')? Attr = - '#' '!'? '[' Path ('=' Literal | TokenTree)? ']' + '#' '!'? '[' Path ('=' Expr | TokenTree)? ']' //****************************// // Statements and Expressions // From 0f5b22ff18043b9fa8ec5417ce94ada0418f136f Mon Sep 17 00:00:00 2001 From: Christopher Durham Date: Sat, 1 May 2021 22:56:42 -0500 Subject: [PATCH 59/84] Report 1-based indices in Error's Display impl This matches every (mainstream) text editor's use of the line:column format. --- src/error.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/error.rs b/src/error.rs index 6cc86f52f9..355e0b7ebc 100644 --- a/src/error.rs +++ b/src/error.rs @@ -16,7 +16,8 @@ pub struct Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { if let Some(loc) = self.location { - write!(f, "{}:{}: ", loc.line, loc.column)? + // Report 1-based indices, to match text editors + write!(f, "{}:{}: ", loc.line + 1, loc.column + 1)? } write!(f, "{}", self.message) } From 3c3095f2e971a2cfc9848bf1c6f5777965e8cc56 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Fri, 11 Jun 2021 18:12:35 +0200 Subject: [PATCH 60/84] Add a `Meta` node representing attribute contents --- rust.ungram | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 3ebe1cbd32..d08ae24905 100644 --- a/rust.ungram +++ b/rust.ungram @@ -298,7 +298,10 @@ Visibility = 'pub' ('(' 'in'? Path ')')? Attr = - '#' '!'? '[' Path ('=' Expr | TokenTree)? ']' + '#' '!'? '[' Meta ']' + +Meta = + Path ('=' Expr | TokenTree)? //****************************// // Statements and Expressions // From b344c9a41be74bc3ba6e641bad76690c951e9692 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Fri, 11 Jun 2021 18:27:11 +0200 Subject: [PATCH 61/84] Bump to 1.14.0 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e5c1d501a7..56dbc825fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.13.0" +version = "1.14.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] From bcb24cd3b9cc8d75a6c9634267d9790fdbf0bc6e Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 19 Jun 2021 17:30:37 +0300 Subject: [PATCH 62/84] fix indentation --- rust.ungram | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust.ungram b/rust.ungram index d08ae24905..ce8e9367ec 100644 --- a/rust.ungram +++ b/rust.ungram @@ -390,9 +390,9 @@ BinExpr = lhs:Expr op:( '||' | '&&' - | '==' | '!=' | '<=' | '>=' | '<' | '>' - | '+' | '*' | '-' | '/' | '%' | '<<' | '>>' | '^' | '|' | '&' - | '=' | '+=' | '/=' | '*=' | '%=' | '>>=' | '<<=' | '-=' | '|=' | '&=' | '^=' + | '==' | '!=' | '<=' | '>=' | '<' | '>' + | '+' | '*' | '-' | '/' | '%' | '<<' | '>>' | '^' | '|' | '&' + | '=' | '+=' | '/=' | '*=' | '%=' | '>>=' | '<<=' | '-=' | '|=' | '&=' | '^=' ) rhs:Expr From 29ab704b75e71ebede40ccf51837a5466055b9dd Mon Sep 17 00:00:00 2001 From: ammkrn Date: Sat, 19 Jun 2021 11:29:31 -0500 Subject: [PATCH 63/84] More specific error for leading pipes --- Cargo.toml | 2 +- src/parser.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56dbc825fa..18c9dffecf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.0" +version = "1.14.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/src/parser.rs b/src/parser.rs index bd067f22a5..a4ce9c1202 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -109,6 +109,14 @@ fn node(p: &mut Parser) -> Result<()> { } fn rule(p: &mut Parser) -> Result { + if let Some(lexer::Token { kind: TokenKind::Pipe, loc }) = p.peek() { + bail!( + *loc, + "The first element in a sequence of productions or alternatives \ + must not have a leading pipe (`|`)" + ); + } + let lhs = seq_rule(p)?; let mut alt = vec![lhs]; while let Some(token) = p.peek() { From 683747fdede75816d7035db250a0187094f03d78 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 17 Jul 2021 03:42:37 +0200 Subject: [PATCH 64/84] Add GenericParamList to AssocTypeArg --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 18c9dffecf..99b020548b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.1" +version = "1.14.2" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index ce8e9367ec..2ac52ddcb5 100644 --- a/rust.ungram +++ b/rust.ungram @@ -51,7 +51,7 @@ TypeArg = Type AssocTypeArg = - NameRef (':' TypeBoundList | '=' Type) + NameRef GenericParamList? (':' TypeBoundList | '=' Type) LifetimeArg = Lifetime From 19176cd726b5959438419de802f3a2c4d0c19aca Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Fri, 13 Aug 2021 00:07:57 +0200 Subject: [PATCH 65/84] Add syntax elements for `if let` match guards --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 99b020548b..067b9b302f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.2" +version = "1.14.3" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" authors = ["Aleksey Kladov "] diff --git a/rust.ungram b/rust.ungram index 2ac52ddcb5..6bb123a3e2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -488,7 +488,7 @@ MatchArm = Attr* Pat guard:MatchGuard? '=>' Expr ','? MatchGuard = - 'if' Expr + 'if' ('let' Pat '=')? Expr ReturnExpr = Attr* 'return' Expr? From 50019bae42f8bb4357ea52b1c783b92af8717a06 Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 3 Aug 2021 21:35:16 -0700 Subject: [PATCH 66/84] Add both variants of half open range patterns to the grammar This is prompted by https://github.com/rust-analyzer/rust-analyzer/issues/9779, but it is not actually a prerequisite of making that one happen as this commit doesn't change the generated code on the r-a side. --- rust.ungram | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 6bb123a3e2..52bdf323af 100644 --- a/rust.ungram +++ b/rust.ungram @@ -603,7 +603,12 @@ WildcardPat = '_' RangePat = - start:Pat op:('..' | '..=') end:Pat + // 1.. + start:Pat op:('..' | '..=') + // 1..2 + | start:Pat op:('..' | '..=') end:Pat + // ..2 + | op:('..' | '..=') end:Pat RefPat = '&' 'mut'? Pat From 1fec3b1017c7a7115b2a10f280e8c0abe557c942 Mon Sep 17 00:00:00 2001 From: Jade Date: Wed, 4 Aug 2021 20:31:22 -0700 Subject: [PATCH 67/84] Bump version and remove soft-deprecated authors field --- Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 067b9b302f..79f3c39e93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,9 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.3" +version = "1.14.4" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" -authors = ["Aleksey Kladov "] edition = "2018" exclude = ["/bors.toml", "/.github"] From 81876f7b56f05001e8ec090db1fb4980fa0f8a1c Mon Sep 17 00:00:00 2001 From: Ariel Davis Date: Mon, 13 Sep 2021 20:37:39 -0700 Subject: [PATCH 68/84] Add ungrammar extensions --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b5a3f48ab3..a5e130fedf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,21 @@ # ungrammar -A DSL for specifying concrete syntax trees. See this -[introductory post](https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html). +A DSL for specifying concrete syntax trees. + +See the [blog post][post] for an introduction. See [./rust.ungram](./rust.ungram) for an example. + +## Editor support + +- Vim + - [vim-ungrammar][] + - [ungrammar.vim][] +- VSCode + - [ungrammar-tools][] + +[post]: + https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html +[vim-ungrammar]: https://github.com/Iron-E/vim-ungrammar +[ungrammar.vim]: https://github.com/drtychai/ungrammar.vim +[ungrammar-tools]: https://github.com/azdavis/ungrammar-tools From 1523dde223b7f4cee84ef62f5dbcc84e6eb2fcb2 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 26 Sep 2021 12:00:56 +0300 Subject: [PATCH 69/84] avoid attribute ambiguity in ExprStmt Both expr stmt and expr can have attributes, which doesn't make sense. Let's say that statement's attrs are those of the expression. --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 52bdf323af..5a8e116137 100644 --- a/rust.ungram +++ b/rust.ungram @@ -318,7 +318,7 @@ LetStmt = '=' initializer:Expr ';' ExprStmt = - Attr* Expr ';'? + Expr ';'? Expr = ArrayExpr From 2da46ff08c55a5940ce3005fba37b774ed6d023b Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 26 Sep 2021 12:02:26 +0300 Subject: [PATCH 70/84] canonical grammar for block expressions Historically, we struggled with formulating the right grammar for block expressions. Today's EffectExpr is the best we've come up so far, but, if you are thinking "WTF is an effect expression?", you are not wrong. I think in this commit I've come up with what can be called a reasonable grammar for block expressions. Observe that *all* things in `{}` we call list: item list, assoc item list, match arm list, record field list, record expr field list. In fact, `BlockExpr` is the only exception. So, let's just call the stuff in `{}` a statement list. This isn't forced: *all* things inside a block are statements, and `;` is a statement separator, just like `,`. Trailing `;` is allowed, but not required. Then, statement list with modifiers such as `async` or attributes or labels is just a block expression. Why haven't I thought of it from the start? --- rust.ungram | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust.ungram b/rust.ungram index 5a8e116137..938843ffce 100644 --- a/rust.ungram +++ b/rust.ungram @@ -331,7 +331,6 @@ Expr = | CastExpr | ClosureExpr | ContinueExpr -| EffectExpr | FieldExpr | ForExpr | IfExpr @@ -366,7 +365,7 @@ Literal = PathExpr = Attr* Path -BlockExpr = +StmtList = '{' Attr* statements:Stmt* @@ -379,8 +378,8 @@ RefExpr = TryExpr = Attr* Expr '?' -EffectExpr = - Attr* Label? ('try' | 'unsafe' | 'async' | 'const') BlockExpr +BlockExpr = + Attr* Label? ('try' | 'unsafe' | 'async' | 'const') StmtList PrefixExpr = Attr* op:('-' | '!' | '*') Expr From 91eb93c349b7a441baa1524915b84428585554e8 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 26 Sep 2021 19:13:22 +0300 Subject: [PATCH 71/84] bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 79f3c39e93..af7ec004fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.4" +version = "1.14.5" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" From aee806c64c2daca6c6009ba4e97958342cb20187 Mon Sep 17 00:00:00 2001 From: zhoufan <1247714429@qq.com> Date: Sun, 3 Oct 2021 08:49:18 +0800 Subject: [PATCH 72/84] add Attr to RestPat --- Cargo.toml | 2 +- rust.ungram | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index af7ec004fd..25a10b3a54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.5" +version = "1.14.6" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index 938843ffce..81323d27ca 100644 --- a/rust.ungram +++ b/rust.ungram @@ -618,7 +618,7 @@ RecordPat = RecordPatFieldList = '{' fields:(RecordPatField (',' RecordPatField)* ','?)? - '..'? + RestPat? '}' RecordPatField = @@ -646,7 +646,7 @@ BoxPat = 'box' Pat RestPat = - '..' + Attr* '..' MacroPat = MacroCall From 7fdea9fdc978a9419682e3319bcb047a5540fb9d Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 6 Oct 2021 18:15:06 +0200 Subject: [PATCH 73/84] Add support for `let ... else` --- Cargo.toml | 2 +- rust.ungram | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 25a10b3a54..f0172821bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.6" +version = "1.14.7" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index 81323d27ca..c9a36079f2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -315,7 +315,9 @@ Stmt = LetStmt = Attr* 'let' Pat (':' Type)? - '=' initializer:Expr ';' + '=' initializer:Expr + ('else' else_branch:BlockExpr)? + ';' ExprStmt = Expr ';'? From f1db2cac1a4498eff74dc8541f754b48d1a0265b Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 7 Oct 2021 16:59:49 +0200 Subject: [PATCH 74/84] Make `LetElse` its own node --- Cargo.toml | 2 +- rust.ungram | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f0172821bf..050459e109 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.7" +version = "1.14.8" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index c9a36079f2..0af0f2521c 100644 --- a/rust.ungram +++ b/rust.ungram @@ -316,9 +316,12 @@ Stmt = LetStmt = Attr* 'let' Pat (':' Type)? '=' initializer:Expr - ('else' else_branch:BlockExpr)? + LetElse? ';' +LetElse = + 'else' BlockExpr + ExprStmt = Expr ';'? From c0478a4433923af678b2f9b20ef916061be294ae Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 19 Oct 2021 14:14:31 +0200 Subject: [PATCH 75/84] Add `~const` bounds to rust grammar --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 050459e109..8e732c8dd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.8" +version = "1.14.9" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index 0af0f2521c..d2ef552ae9 100644 --- a/rust.ungram +++ b/rust.ungram @@ -573,7 +573,7 @@ TypeBoundList = TypeBound = Lifetime -| '?'? Type +| ('?' | '~' 'const')? Type //************************// // Patterns // From 7479c18dc9ca8c2f5fda60f4b19dc8595a611427 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Mon, 24 Jan 2022 03:30:36 +0200 Subject: [PATCH 76/84] Support if- and while-let chains RFC 2497 https://github.com/rust-lang/rfcs/blob/master/text/2497-if-let-chains.md. --- Cargo.toml | 2 +- rust.ungram | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8e732c8dd9..bbfb0f5855 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.14.9" +version = "1.15.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index d2ef552ae9..7d7f184852 100644 --- a/rust.ungram +++ b/rust.ungram @@ -357,6 +357,7 @@ Expr = | TupleExpr | WhileExpr | YieldExpr +| LetExpr Literal = Attr* value:( @@ -448,13 +449,9 @@ ClosureExpr = body:Expr IfExpr = - Attr* 'if' Condition then_branch:BlockExpr + Attr* 'if' condition:Expr then_branch:BlockExpr ('else' else_branch:(IfExpr | BlockExpr))? -Condition = - 'let' Pat '=' Expr -| Expr - LoopExpr = Attr* Label? 'loop' loop_body:BlockExpr @@ -464,7 +461,7 @@ ForExpr = loop_body:BlockExpr WhileExpr = - Attr* Label? 'while' Condition + Attr* Label? 'while' condition:Expr loop_body:BlockExpr Label = @@ -492,7 +489,7 @@ MatchArm = Attr* Pat guard:MatchGuard? '=>' Expr ','? MatchGuard = - 'if' ('let' Pat '=')? Expr + 'if' condition:Expr ReturnExpr = Attr* 'return' Expr? @@ -500,6 +497,9 @@ ReturnExpr = YieldExpr = Attr* 'yield' Expr? +LetExpr = + Attr* 'let' Pat '=' Expr + AwaitExpr = Attr* Expr '.' 'await' From 800de5830d99c1fd7dade0440c5bfc464e09d566 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Tue, 8 Feb 2022 13:51:44 +0100 Subject: [PATCH 77/84] Fix some rust.ungram inconsistencies --- rust.ungram | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/rust.ungram b/rust.ungram index 7d7f184852..6b31f98ec0 100644 --- a/rust.ungram +++ b/rust.ungram @@ -102,7 +102,7 @@ Item = | Use MacroRules = - Attr* Visibility? + Attr* 'macro_rules' '!' Name TokenTree @@ -173,7 +173,7 @@ TypeAlias = Attr* Visibility? 'default'? 'type' Name GenericParamList? (':' TypeBoundList?)? WhereClause? - '=' Type ';' + ('=' Type)? ';' Struct = Attr* Visibility? @@ -210,7 +210,7 @@ VariantList = Variant = Attr* Visibility? - Name FieldList ('=' Expr)? + Name FieldList? ('=' Expr)? Union = Attr* Visibility? @@ -229,17 +229,17 @@ Const = Attr* Visibility? 'default'? 'const' (Name | '_') ':' Type - '=' body:Expr ';' + ('=' body:Expr)? ';' Static = Attr* Visibility? - 'static'? 'mut'? Name ':' Type - '=' body:Expr ';' + 'static' 'mut'? Name ':' Type + ('=' body:Expr)? ';' Trait = Attr* Visibility? 'unsafe'? 'auto'? - 'trait' Name GenericParamList (':' TypeBoundList?)? WhereClause + 'trait' Name GenericParamList? (':' TypeBoundList?)? WhereClause? AssocItemList AssocItemList = @@ -258,7 +258,7 @@ Impl = AssocItemList ExternBlock = - Attr* Abi ExternItemList + Attr* 'unsafe'? Abi ExternItemList ExternItemList = '{' Attr* ExternItem* '}' @@ -292,7 +292,7 @@ WhereClause = 'where' predicates:(WherePred (',' WherePred)* ','?) WherePred = - ('for' GenericParamList)? (Lifetime | Type) ':' TypeBoundList + ('for' GenericParamList)? (Lifetime | Type) ':' TypeBoundList? Visibility = 'pub' ('(' 'in'? Path ')')? From 11206f1fb048301e2f907a6474c84976df4a004d Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Thu, 24 Feb 2022 14:20:36 +0200 Subject: [PATCH 78/84] Revert the visibility removal of macro_rules visibility (#46) As discussed in https://github.com/rust-analyzer/ungrammar/pull/46#issuecomment-1049758027. --- rust.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust.ungram b/rust.ungram index 6b31f98ec0..572afe2e55 100644 --- a/rust.ungram +++ b/rust.ungram @@ -102,7 +102,7 @@ Item = | Use MacroRules = - Attr* + Attr* Visibility? 'macro_rules' '!' Name TokenTree From 039a27440743b068f811c6f0bc06dbdd0d1de503 Mon Sep 17 00:00:00 2001 From: Chayim Refael Friedman Date: Thu, 24 Feb 2022 02:34:24 +0000 Subject: [PATCH 79/84] Support destructuring assignments (RFC 2909) The supported patterns are already valid as expressions, except the rest pattern (`..`) and the wildcard pattern (`_`). --- Cargo.toml | 2 +- rust.ungram | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bbfb0f5855..e9a5145779 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.15.0" +version = "1.16.0" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index 572afe2e55..8c85c45ec4 100644 --- a/rust.ungram +++ b/rust.ungram @@ -358,6 +358,7 @@ Expr = | WhileExpr | YieldExpr | LetExpr +| UnderscoreExpr Literal = Attr* value:( @@ -426,7 +427,7 @@ RecordExprFieldList = '{' Attr* fields:(RecordExprField (',' RecordExprField)* ','?)? - ('..' spread:Expr)? + ('..' spread:Expr?)? '}' RecordExprField = @@ -500,6 +501,9 @@ YieldExpr = LetExpr = Attr* 'let' Pat '=' Expr +UnderscoreExpr = + Attr* '_' + AwaitExpr = Attr* Expr '.' 'await' From 6d2f3ec8b0339baf8508b5faf2f917b7627d5cfc Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Sat, 5 Mar 2022 22:48:00 +0100 Subject: [PATCH 80/84] Include `Self` specifically in NameRef --- Cargo.toml | 2 +- rust.ungram | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e9a5145779..833d47990a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ungrammar" description = "A DSL for describing concrete syntax trees" -version = "1.16.0" +version = "1.16.1" license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/ungrammar" edition = "2018" diff --git a/rust.ungram b/rust.ungram index 8c85c45ec4..cb58486eff 100644 --- a/rust.ungram +++ b/rust.ungram @@ -24,7 +24,7 @@ Name = 'ident' | 'self' NameRef = - 'ident' | 'int_number' | 'self' | 'super' | 'crate' + 'ident' | 'int_number' | 'self' | 'super' | 'crate' | 'Self' Lifetime = 'lifetime_ident' From 11fecff8dec9879129083d89de64864f8a5cb903 Mon Sep 17 00:00:00 2001 From: Eli Bendersky Date: Mon, 19 Jun 2023 08:42:25 -0700 Subject: [PATCH 81/84] Update ungrammar.ungram with proper labeled rule Labels are followed by rules The ungrammar listed in https://rust-analyzer.github.io/blog/2020/10/24/introducing-ungrammar.html seems to be correct --- ungrammar.ungram | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ungrammar.ungram b/ungrammar.ungram index 6cb4e10fb1..856a6cede0 100644 --- a/ungrammar.ungram +++ b/ungrammar.ungram @@ -13,4 +13,4 @@ Rule = | Rule '?' | Rule '*' | '(' Rule ')' -| label:'ident' ':' +| label:'ident' ':' Rule From 5289a56013181cbdbf31e99c7b2ff611e0d0200a Mon Sep 17 00:00:00 2001 From: Alona Enraght-Moony Date: Thu, 29 Jun 2023 14:51:33 +0100 Subject: [PATCH 82/84] implement `Clone` for `Rule` --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 7aa0ce9c88..6adf8ef8ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -101,7 +101,7 @@ pub struct TokenData { } /// A production rule. -#[derive(Debug, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq)] pub enum Rule { /// A labeled rule, like `a:B` (`"a"` is the label, `B` is the rule). Labeled { From 1d4189c0afa157dbff7d8f9efa73cd77020f3fe5 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 1 Jul 2023 15:13:32 +0800 Subject: [PATCH 83/84] Update repo link in Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 833d47990a..920d9ef49d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "ungrammar" description = "A DSL for describing concrete syntax trees" version = "1.16.1" license = "MIT OR Apache-2.0" -repository = "https://github.com/matklad/ungrammar" +repository = "https://github.com/rust-analyzer/ungrammar" edition = "2018" exclude = ["/bors.toml", "/.github"] From 73678ce217b4b78df4484d9c32eca6c7dff34496 Mon Sep 17 00:00:00 2001 From: Lukas Wirth Date: Thu, 18 Apr 2024 17:12:18 +0200 Subject: [PATCH 84/84] Update rust.ungram --- rust.ungram | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust.ungram b/rust.ungram index cb58486eff..7781e719e2 100644 --- a/rust.ungram +++ b/rust.ungram @@ -1,3 +1,5 @@ +// Note this grammar file does not reflect the current language as this file is no longer maintained. + // Rust Un-Grammar. // // This grammar specifies the structure of Rust's concrete syntax tree.