Merge #41

41: G: unsafe impl & trait r=matklad a=matklad

bors r+

grammar.ron

@@ -91,6 +91,8 @@ Grammar(
         "USE_ITEM",
         "STATIC_ITEM",
         "CONST_ITEM",
+        "TRAIT_ITEM",
+        "IMPL_ITEM",
 
         "EXTERN_BLOCK",
         "ENUM_VARIANT",

src/parser/event.rs

@@ -0,0 +1,140 @@
+use {File, FileBuilder, Sink, SyntaxKind, Token};
+use syntax_kinds::TOMBSTONE;
+use super::is_insignificant;
+
+/// `Parser` produces a flat list of `Event`s.
+/// They are converted to a tree-structure in
+/// a separate pass, via `TreeBuilder`.
+#[derive(Debug)]
+pub(crate) enum Event {
+    /// This event signifies the start of the node.
+    /// It should be either abandoned (in which case the
+    /// `kind` is `TOMBSTONE`, and the event is ignored),
+    /// or completed via a `Finish` event.
+    ///
+    /// All tokens between a `Start` and a `Finish` would
+    /// become the children of the respective node.
+    ///
+    /// For left-recursive syntactic constructs, the parser produces
+    /// a child node before it sees a parent. `forward_parent`
+    /// exists to allow to tweak parent-child relationships.
+    ///
+    /// Consider this path
+    ///
+    /// foo::bar
+    ///
+    /// The events for it would look like this:
+    ///
+    ///
+    /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
+    ///       |                          /\
+    ///       |                          |
+    ///       +------forward-parent------+
+    ///
+    /// And the tree would look like this
+    ///
+    ///    +--PATH---------+
+    ///    |   |           |
+    ///    |   |           |
+    ///    |  '::'       'bar'
+    ///    |
+    ///   PATH
+    ///    |
+    ///   'foo'
+    ///
+    /// See also `CompleteMarker::precede`.
+    Start {
+        kind: SyntaxKind,
+        forward_parent: Option<u32>,
+    },
+
+    /// Complete the previous `Start` event
+    Finish,
+
+    /// Produce a single leaf-element.
+    /// `n_raw_tokens` is used to glue complex contextual tokens.
+    /// For example, lexer tokenizes `>>` as `>`, `>`, and
+    /// `n_raw_tokens = 2` is used to produced a single `>>`.
+    Token {
+        kind: SyntaxKind,
+        n_raw_tokens: u8,
+    },
+
+    Error {
+        message: String,
+    },
+}
+
+pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
+    let mut builder = FileBuilder::new(text);
+    let mut idx = 0;
+
+    let mut holes = Vec::new();
+    let mut forward_parents = Vec::new();
+
+    for (i, event) in events.iter().enumerate() {
+        if holes.last() == Some(&i) {
+            holes.pop();
+            continue;
+        }
+
+        match event {
+            &Event::Start {
+                kind: TOMBSTONE, ..
+            } => (),
+
+            &Event::Start { .. } => {
+                forward_parents.clear();
+                let mut idx = i;
+                loop {
+                    let (kind, fwd) = match events[idx] {
+                        Event::Start {
+                            kind,
+                            forward_parent,
+                        } => (kind, forward_parent),
+                        _ => unreachable!(),
+                    };
+                    forward_parents.push((idx, kind));
+                    if let Some(fwd) = fwd {
+                        idx += fwd as usize;
+                    } else {
+                        break;
+                    }
+                }
+                for &(idx, kind) in forward_parents.iter().into_iter().rev() {
+                    builder.start_internal(kind);
+                    holes.push(idx);
+                }
+                holes.pop();
+            }
+            &Event::Finish => {
+                while idx < tokens.len() {
+                    let token = tokens[idx];
+                    if is_insignificant(token.kind) {
+                        idx += 1;
+                        builder.leaf(token.kind, token.len);
+                    } else {
+                        break;
+                    }
+                }
+                builder.finish_internal()
+            }
+            &Event::Token {
+                kind: _,
+                mut n_raw_tokens,
+            } => loop {
+                let token = tokens[idx];
+                if !is_insignificant(token.kind) {
+                    n_raw_tokens -= 1;
+                }
+                idx += 1;
+                builder.leaf(token.kind, token.len);
+                if n_raw_tokens == 0 {
+                    break;
+                }
+            },
+            &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
+        }
+    }
+    builder.finish()
+}

src/parser/event_parser/mod.rs

@@ -1,74 +0,0 @@
-use {SyntaxKind, Token};
-
-#[macro_use]
-mod parser;
-mod grammar;
-
-/// `Parser` produces a flat list of `Event`s.
-/// They are converted to a tree-structure in
-/// a separate pass, via `TreeBuilder`.
-#[derive(Debug)]
-pub(crate) enum Event {
-    /// This event signifies the start of the node.
-    /// It should be either abandoned (in which case the
-    /// `kind` is `TOMBSTONE`, and the event is ignored),
-    /// or completed via a `Finish` event.
-    ///
-    /// All tokens between a `Start` and a `Finish` would
-    /// become the children of the respective node.
-    ///
-    /// For left-recursive syntactic constructs, the parser produces
-    /// a child node before it sees a parent. `forward_parent`
-    /// exists to allow to tweak parent-child relationships.
-    ///
-    /// Consider this path
-    ///
-    /// foo::bar
-    ///
-    /// The events for it would look like this:
-    ///
-    ///
-    /// START(PATH) IDENT('foo') FINISH START(PATH) COLONCOLON IDENT('bar') FINISH
-    ///       |                          /\
-    ///       |                          |
-    ///       +------forward-parent------+
-    ///
-    /// And the tree would look like this
-    ///
-    ///    +--PATH---------+
-    ///    |   |           |
-    ///    |   |           |
-    ///    |  '::'       'bar'
-    ///    |
-    ///   PATH
-    ///    |
-    ///   'foo'
-    ///
-    /// See also `CompleteMarker::precede`.
-    Start {
-        kind: SyntaxKind,
-        forward_parent: Option<u32>,
-    },
-
-    /// Complete the previous `Start` event
-    Finish,
-
-    /// Produce a single leaf-element.
-    /// `n_raw_tokens` is used to glue complex contextual tokens.
-    /// For example, lexer tokenizes `>>` as `>`, `>`, and
-    /// `n_raw_tokens = 2` is used to produced a single `>>`.
-    Token {
-        kind: SyntaxKind,
-        n_raw_tokens: u8,
-    },
-
-    Error {
-        message: String,
-    },
-}
-
-pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
-    let mut parser = parser::Parser::new(text, raw_tokens);
-    grammar::file(&mut parser);
-    parser.into_events()
-}

src/parser/grammar/items/mod.rs

@@ -3,6 +3,7 @@ use super::*;
 mod structs;
 mod use_item;
 mod consts;
+mod traits;
 
 pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
     attributes::inner_attributes(p);
@@ -80,6 +81,22 @@ fn item(p: &mut Parser) {
                 CONST_ITEM
             }
         },
+        // TODO: auto trait
+        // test unsafe_trait
+        // unsafe trait T {}
+        UNSAFE_KW if la == TRAIT_KW => {
+            p.bump();
+            traits::trait_item(p);
+            TRAIT_ITEM
+        }
+        // TODO: default impl
+        // test unsafe_impl
+        // unsafe impl Foo {}
+        UNSAFE_KW if la == IMPL_KW => {
+            p.bump();
+            traits::impl_item(p);
+            IMPL_ITEM
+        }
         MOD_KW => {
             mod_item(p);
             MOD_ITEM
@@ -131,6 +148,7 @@ fn extern_block(p: &mut Parser) {
     p.bump();
     p.expect(R_CURLY);
 }
+
 fn mod_item(p: &mut Parser) {
     assert!(p.at(MOD_KW));
     p.bump();

src/parser/grammar/items/traits.rs

@@ -0,0 +1,17 @@
+use super::*;
+
+pub(super) fn trait_item(p: &mut Parser) {
+    assert!(p.at(TRAIT_KW));
+    p.bump();
+    p.expect(IDENT);
+    p.expect(L_CURLY);
+    p.expect(R_CURLY);
+}
+
+pub(super) fn impl_item(p: &mut Parser) {
+    assert!(p.at(IMPL_KW));
+    p.bump();
+    p.expect(IDENT);
+    p.expect(L_CURLY);
+    p.expect(R_CURLY);
+}

src/parser/mod.rs

@@ -1,88 +1,21 @@
-use {File, FileBuilder, Sink, SyntaxKind, Token};
+use {File, SyntaxKind, Token};
 
 use syntax_kinds::*;
 
-mod event_parser;
+#[macro_use]
-use self::event_parser::Event;
+mod parser;
+mod event;
+mod grammar;
+use self::event::Event;
 
 /// Parse a sequence of tokens into the representative node tree
 pub fn parse(text: String, tokens: &[Token]) -> File {
-    let events = event_parser::parse(&text, tokens);
+    let events = {
-    from_events_to_file(text, tokens, events)
+        let mut parser = parser::Parser::new(&text, tokens);
-}
+        grammar::file(&mut parser);
-
+        parser.into_events()
-fn from_events_to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
+    };
-    let mut builder = FileBuilder::new(text);
+    event::to_file(text, tokens, events)
-    let mut idx = 0;
-
-    let mut holes = Vec::new();
-    let mut forward_parents = Vec::new();
-
-    for (i, event) in events.iter().enumerate() {
-        if holes.last() == Some(&i) {
-            holes.pop();
-            continue;
-        }
-
-        match event {
-            &Event::Start {
-                kind: TOMBSTONE, ..
-            } => (),
-
-            &Event::Start { .. } => {
-                forward_parents.clear();
-                let mut idx = i;
-                loop {
-                    let (kind, fwd) = match events[idx] {
-                        Event::Start {
-                            kind,
-                            forward_parent,
-                        } => (kind, forward_parent),
-                        _ => unreachable!(),
-                    };
-                    forward_parents.push((idx, kind));
-                    if let Some(fwd) = fwd {
-                        idx += fwd as usize;
-                    } else {
-                        break;
-                    }
-                }
-                for &(idx, kind) in forward_parents.iter().into_iter().rev() {
-                    builder.start_internal(kind);
-                    holes.push(idx);
-                }
-                holes.pop();
-            }
-            &Event::Finish => {
-                while idx < tokens.len() {
-                    let token = tokens[idx];
-                    if is_insignificant(token.kind) {
-                        idx += 1;
-                        builder.leaf(token.kind, token.len);
-                    } else {
-                        break;
-                    }
-                }
-                builder.finish_internal()
-            }
-            &Event::Token {
-                kind: _,
-                mut n_raw_tokens,
-            } => loop {
-                let token = tokens[idx];
-                if !is_insignificant(token.kind) {
-                    n_raw_tokens -= 1;
-                }
-                idx += 1;
-                builder.leaf(token.kind, token.len);
-                if n_raw_tokens == 0 {
-                    break;
-                }
-            },
-            &Event::Error { ref message } => builder.error().message(message.clone()).emit(),
-        }
-    }
-    builder.finish()
 }
 
 fn is_insignificant(kind: SyntaxKind) -> bool {

src/parser/parser.rs

@@ -1,6 +1,6 @@
 use {SyntaxKind, TextUnit, Token};
 use super::Event;
-use super::super::is_insignificant;
+use super::is_insignificant;
 use SyntaxKind::{EOF, TOMBSTONE};
 
 pub(crate) struct Marker {

src/syntax_kinds.rs

@@ -92,6 +92,8 @@ pub enum SyntaxKind {
     USE_ITEM,
     STATIC_ITEM,
     CONST_ITEM,
+    TRAIT_ITEM,
+    IMPL_ITEM,
     EXTERN_BLOCK,
     ENUM_VARIANT,
     NAMED_FIELD,
@@ -207,6 +209,8 @@ impl SyntaxKind {
             USE_ITEM => &SyntaxInfo { name: "USE_ITEM" },
             STATIC_ITEM => &SyntaxInfo { name: "STATIC_ITEM" },
             CONST_ITEM => &SyntaxInfo { name: "CONST_ITEM" },
+            TRAIT_ITEM => &SyntaxInfo { name: "TRAIT_ITEM" },
+            IMPL_ITEM => &SyntaxInfo { name: "IMPL_ITEM" },
             EXTERN_BLOCK => &SyntaxInfo { name: "EXTERN_BLOCK" },
             ENUM_VARIANT => &SyntaxInfo { name: "ENUM_VARIANT" },
             NAMED_FIELD => &SyntaxInfo { name: "NAMED_FIELD" },

tests/data/parser/inline/0007_unsafe_trait.rs

@@ -0,0 +1 @@
+unsafe trait T {}

tests/data/parser/inline/0007_unsafe_trait.txt

@@ -0,0 +1,11 @@
+FILE@[0; 18)
+  TRAIT_ITEM@[0; 18)
+    UNSAFE_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    TRAIT_KW@[7; 12)
+    WHITESPACE@[12; 13)
+    IDENT@[13; 14) "T"
+    WHITESPACE@[14; 15)
+    L_CURLY@[15; 16)
+    R_CURLY@[16; 17)
+    WHITESPACE@[17; 18)

tests/data/parser/inline/0008_unsafe_impl.rs

@@ -0,0 +1 @@
+unsafe impl Foo {}

tests/data/parser/inline/0008_unsafe_impl.txt

@@ -0,0 +1,11 @@
+FILE@[0; 19)
+  IMPL_ITEM@[0; 19)
+    UNSAFE_KW@[0; 6)
+    WHITESPACE@[6; 7)
+    IMPL_KW@[7; 11)
+    WHITESPACE@[11; 12)
+    IDENT@[12; 15) "Foo"
+    WHITESPACE@[15; 16)
+    L_CURLY@[16; 17)
+    R_CURLY@[17; 18)
+    WHITESPACE@[18; 19)

tools/src/bin/collect-tests.rs

@@ -79,16 +79,19 @@ fn collect_tests(s: &str) -> Vec<Test> {
         .map(str::trim_left)
         .group_by(|line| line.starts_with(prefix));
 
-    for (is_comment, block) in comment_blocks.into_iter() {
+    'outer: for (is_comment, block) in comment_blocks.into_iter() {
         if !is_comment {
             continue;
         }
         let mut block = block.map(|line| &line[prefix.len()..]);
-        let first = block.next().unwrap();
+
-        if !first.starts_with("test ") {
+        let name = loop {
-            continue;
+            match block.next() {
-        }
+                Some(line) if line.starts_with("test ") => break line["test ".len()..].to_string(),
-        let name = first["test ".len()..].to_string();
+                Some(_) => (),
+                None => continue 'outer,
+            }
+        };
         let text: String = itertools::join(block.chain(::std::iter::once("")), "\n");
         assert!(!text.trim().is_empty() && text.ends_with("\n"));
         res.push(Test { name, text })
@@ -121,7 +124,7 @@ fn inline_tests_dir() -> PathBuf {
 }
 
 fn grammar_dir() -> PathBuf {
-    base_dir().join("src/parser/event_parser/grammar")
+    base_dir().join("src/parser/grammar")
 }
 
 fn base_dir() -> PathBuf {