Merge pull request #140 from GuillaumeGomez/literal-prefixes

Add support for b'a', b"a" and c"a"

rinja_derive/src/generator.rs

@@ -9,7 +9,7 @@ use std::{cmp, hash, mem, str};
 use parser::node::{
     Call, Comment, Cond, CondTest, FilterBlock, If, Include, Let, Lit, Loop, Match, Whitespace, Ws,
 };
-use parser::{Expr, Filter, Node, Target, WithSpan};
+use parser::{CharLit, CharPrefix, Expr, Filter, Node, StrLit, StrPrefix, Target, WithSpan};
 use quote::quote;
 use rustc_hash::FxBuildHasher;
 
@@ -1270,9 +1270,18 @@ impl<'a> Generator<'a> {
             }
 
             // for now, we only escape strings and chars at compile time
-            let lit = match &**s {
+            let (lit, escape_prefix) = match &**s {
-                Expr::StrLit(input) => InputKind::StrLit(input),
+                Expr::StrLit(StrLit { prefix, content }) => {
-                Expr::CharLit(input) => InputKind::CharLit(input),
+                    (InputKind::StrLit(content), prefix.map(|p| p.to_char()))
+                }
+                Expr::CharLit(CharLit { prefix, content }) => (
+                    InputKind::CharLit(content),
+                    if *prefix == Some(CharPrefix::Binary) {
+                        Some('b')
+                    } else {
+                        None
+                    },
+                ),
                 _ => return None,
             };
 
@@ -1314,6 +1323,9 @@ impl<'a> Generator<'a> {
                 OutputKind::Text => unescaped,
                 OutputKind::Html => {
                     let mut escaped = String::with_capacity(unescaped.len() + 20);
+                    if let Some(escape_prefix) = escape_prefix {
+                        escaped.push(escape_prefix);
+                    }
                     write_escaped_str(&mut escaped, &unescaped).ok()?;
                     match escaped == unescaped {
                         true => unescaped,
@@ -1478,8 +1490,8 @@ impl<'a> Generator<'a> {
         Ok(match **expr {
             Expr::BoolLit(s) => self.visit_bool_lit(buf, s),
             Expr::NumLit(s) => self.visit_num_lit(buf, s),
-            Expr::StrLit(s) => self.visit_str_lit(buf, s),
+            Expr::StrLit(ref s) => self.visit_str_lit(buf, s),
-            Expr::CharLit(s) => self.visit_char_lit(buf, s),
+            Expr::CharLit(ref s) => self.visit_char_lit(buf, s),
             Expr::Var(s) => self.visit_var(buf, s),
             Expr::Path(ref path) => self.visit_path(buf, path),
             Expr::Array(ref elements) => self.visit_array(ctx, buf, elements)?,
@@ -1699,7 +1711,22 @@ impl<'a> Generator<'a> {
             return Err(ctx.generate_error("only two arguments allowed to escape filter", node));
         }
         let opt_escaper = match args.get(1).map(|expr| &**expr) {
-            Some(Expr::StrLit(name)) => Some(*name),
+            Some(Expr::StrLit(StrLit { prefix, content })) => {
+                if let Some(prefix) = prefix {
+                    let kind = if *prefix == StrPrefix::Binary {
+                        "slice"
+                    } else {
+                        "CStr"
+                    };
+                    return Err(ctx.generate_error(
+                        &format!(
+                            "invalid escaper `b{content:?}`. Expected a string, found a {kind}"
+                        ),
+                        &args[1],
+                    ));
+                }
+                Some(content)
+            }
             Some(_) => {
                 return Err(ctx.generate_error("invalid escaper type for escape filter", node));
             }
@@ -1741,7 +1768,7 @@ impl<'a> Generator<'a> {
         node: &WithSpan<'_, T>,
     ) -> Result<DisplayWrap, CompileError> {
         if !args.is_empty() {
-            if let Expr::StrLit(fmt) = *args[0] {
+            if let Expr::StrLit(ref fmt) = *args[0] {
                 buf.write("::std::format!(");
                 self.visit_str_lit(buf, fmt);
                 if args.len() > 1 {
@@ -1763,7 +1790,7 @@ impl<'a> Generator<'a> {
         node: &WithSpan<'_, T>,
     ) -> Result<DisplayWrap, CompileError> {
         if let [_, arg2] = args {
-            if let Expr::StrLit(fmt) = **arg2 {
+            if let Expr::StrLit(ref fmt) = **arg2 {
                 buf.write("::std::format!(");
                 self.visit_str_lit(buf, fmt);
                 buf.write(',');
@@ -2077,13 +2104,19 @@ impl<'a> Generator<'a> {
         DisplayWrap::Unwrapped
     }
 
-    fn visit_str_lit(&mut self, buf: &mut Buffer, s: &str) -> DisplayWrap {
+    fn visit_str_lit(&mut self, buf: &mut Buffer, s: &StrLit<'_>) -> DisplayWrap {
-        buf.write(format_args!("\"{s}\""));
+        if let Some(prefix) = s.prefix {
+            buf.write(prefix.to_char());
+        }
+        buf.write(format_args!("\"{}\"", s.content));
         DisplayWrap::Unwrapped
     }
 
-    fn visit_char_lit(&mut self, buf: &mut Buffer, s: &str) -> DisplayWrap {
+    fn visit_char_lit(&mut self, buf: &mut Buffer, c: &CharLit<'_>) -> DisplayWrap {
-        buf.write(format_args!("'{s}'"));
+        if c.prefix == Some(CharPrefix::Binary) {
+            buf.write('b');
+        }
+        buf.write(format_args!("'{}'", c.content));
         DisplayWrap::Unwrapped
     }
 

rinja_parser/src/expr.rs

@@ -10,11 +10,10 @@ use nom::error_position;
 use nom::multi::{fold_many0, many0, separated_list0};
 use nom::sequence::{pair, preceded, terminated, tuple};
 
-use super::{
+use crate::{
-    char_lit, filter, identifier, keyword, not_ws, num_lit, path_or_identifier, str_lit, ws, Level,
+    char_lit, filter, identifier, keyword, not_ws, num_lit, path_or_identifier, str_lit, ws,
-    PathOrIdentifier,
+    CharLit, ErrorContext, Level, ParseResult, PathOrIdentifier, StrLit, WithSpan,
 };
-use crate::{ErrorContext, ParseResult, WithSpan};
 
 macro_rules! expr_prec_layer {
     ( $name:ident, $inner:ident, $op:expr ) => {
@@ -37,8 +36,8 @@ macro_rules! expr_prec_layer {
 pub enum Expr<'a> {
     BoolLit(bool),
     NumLit(&'a str),
-    StrLit(&'a str),
+    StrLit(StrLit<'a>),
-    CharLit(&'a str),
+    CharLit(CharLit<'a>),
     Var(&'a str),
     Path(Vec<&'a str>),
     Array(Vec<WithSpan<'a, Expr<'a>>>),

rinja_parser/src/lib.rs

@@ -416,7 +416,36 @@ fn separated_digits(radix: u32, start: bool) -> impl Fn(&str) -> ParseResult<'_>
     }
 }
 
-fn str_lit(i: &str) -> ParseResult<'_> {
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum StrPrefix {
+    Binary,
+    CLike,
+}
+
+impl StrPrefix {
+    pub fn to_char(self) -> char {
+        match self {
+            Self::Binary => 'b',
+            Self::CLike => 'c',
+        }
+    }
+}
+
+impl fmt::Display for StrPrefix {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use std::fmt::Write;
+
+        f.write_char(self.to_char())
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct StrLit<'a> {
+    pub prefix: Option<StrPrefix>,
+    pub content: &'a str,
+}
+
+fn str_lit_without_prefix(i: &str) -> ParseResult<'_> {
     let (i, s) = delimited(
         char('"'),
         opt(escaped(is_not("\\\""), '\\', anychar)),
@@ -425,15 +454,40 @@ fn str_lit(i: &str) -> ParseResult<'_> {
     Ok((i, s.unwrap_or_default()))
 }
 
+fn str_lit(i: &str) -> Result<(&str, StrLit<'_>), ParseErr<'_>> {
+    let (i, (prefix, content)) =
+        tuple((opt(alt((char('b'), char('c')))), str_lit_without_prefix))(i)?;
+    let prefix = match prefix {
+        Some('b') => Some(StrPrefix::Binary),
+        Some('c') => Some(StrPrefix::CLike),
+        _ => None,
+    };
+    Ok((i, StrLit { prefix, content }))
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum CharPrefix {
+    Binary,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct CharLit<'a> {
+    pub prefix: Option<CharPrefix>,
+    pub content: &'a str,
+}
+
 // Information about allowed character escapes is available at:
 // <https://doc.rust-lang.org/reference/tokens.html#character-literals>.
-fn char_lit(i: &str) -> ParseResult<'_> {
+fn char_lit(i: &str) -> Result<(&str, CharLit<'_>), ParseErr<'_>> {
     let start = i;
-    let (i, s) = delimited(
+    let (i, (b_prefix, s)) = tuple((
-        char('\''),
+        opt(char('b')),
-        opt(escaped(is_not("\\\'"), '\\', anychar)),
+        delimited(
-        char('\''),
+            char('\''),
-    )(i)?;
+            opt(escaped(is_not("\\\'"), '\\', anychar)),
+            char('\''),
+        ),
+    ))(i)?;
 
     let Some(s) = s else {
         return Err(nom::Err::Failure(ErrorContext::new(
@@ -449,7 +503,15 @@ fn char_lit(i: &str) -> ParseResult<'_> {
     };
 
     let (nb, max_value, err1, err2) = match c {
-        Char::Literal | Char::Escaped => return Ok((i, s)),
+        Char::Literal | Char::Escaped => {
+            return Ok((
+                i,
+                CharLit {
+                    prefix: b_prefix.map(|_| CharPrefix::Binary),
+                    content: s,
+                },
+            ));
+        }
         Char::AsciiEscape(nb) => (
             nb,
             // `0x7F` is the maximum value for a `\x` escaped character.
@@ -473,7 +535,13 @@ fn char_lit(i: &str) -> ParseResult<'_> {
         return Err(nom::Err::Failure(ErrorContext::new(err2, start)));
     }
 
-    Ok((i, s))
+    Ok((
+        i,
+        CharLit {
+            prefix: b_prefix.map(|_| CharPrefix::Binary),
+            content: s,
+        },
+    ))
 }
 
 /// Represents the different kinds of char declarations:
@@ -775,7 +843,7 @@ const PRIMITIVE_TYPES: &[&str] = &{
 mod test {
     use std::path::Path;
 
-    use super::{char_lit, num_lit, strip_common};
+    use super::{char_lit, num_lit, str_lit, strip_common, StrLit, StrPrefix};
 
     #[test]
     fn test_strip_common() {
@@ -820,26 +888,43 @@ mod test {
 
     #[test]
     fn test_char_lit() {
-        assert_eq!(char_lit("'a'").unwrap(), ("", "a"));
+        let lit = |s: &'static str| crate::CharLit {
-        assert_eq!(char_lit("'字'").unwrap(), ("", "字"));
+            prefix: None,
+            content: s,
+        };
+
+        assert_eq!(char_lit("'a'").unwrap(), ("", lit("a")));
+        assert_eq!(char_lit("'字'").unwrap(), ("", lit("字")));
 
         // Escaped single characters.
-        assert_eq!(char_lit("'\\\"'").unwrap(), ("", "\\\""));
+        assert_eq!(char_lit("'\\\"'").unwrap(), ("", lit("\\\"")));
-        assert_eq!(char_lit("'\\''").unwrap(), ("", "\\'"));
+        assert_eq!(char_lit("'\\''").unwrap(), ("", lit("\\'")));
-        assert_eq!(char_lit("'\\t'").unwrap(), ("", "\\t"));
+        assert_eq!(char_lit("'\\t'").unwrap(), ("", lit("\\t")));
-        assert_eq!(char_lit("'\\n'").unwrap(), ("", "\\n"));
+        assert_eq!(char_lit("'\\n'").unwrap(), ("", lit("\\n")));
-        assert_eq!(char_lit("'\\r'").unwrap(), ("", "\\r"));
+        assert_eq!(char_lit("'\\r'").unwrap(), ("", lit("\\r")));
-        assert_eq!(char_lit("'\\0'").unwrap(), ("", "\\0"));
+        assert_eq!(char_lit("'\\0'").unwrap(), ("", lit("\\0")));
         // Escaped ascii characters (up to `0x7F`).
-        assert_eq!(char_lit("'\\x12'").unwrap(), ("", "\\x12"));
+        assert_eq!(char_lit("'\\x12'").unwrap(), ("", lit("\\x12")));
-        assert_eq!(char_lit("'\\x02'").unwrap(), ("", "\\x02"));
+        assert_eq!(char_lit("'\\x02'").unwrap(), ("", lit("\\x02")));
-        assert_eq!(char_lit("'\\x6a'").unwrap(), ("", "\\x6a"));
+        assert_eq!(char_lit("'\\x6a'").unwrap(), ("", lit("\\x6a")));
-        assert_eq!(char_lit("'\\x7F'").unwrap(), ("", "\\x7F"));
+        assert_eq!(char_lit("'\\x7F'").unwrap(), ("", lit("\\x7F")));
         // Escaped unicode characters (up to `0x10FFFF`).
-        assert_eq!(char_lit("'\\u{A}'").unwrap(), ("", "\\u{A}"));
+        assert_eq!(char_lit("'\\u{A}'").unwrap(), ("", lit("\\u{A}")));
-        assert_eq!(char_lit("'\\u{10}'").unwrap(), ("", "\\u{10}"));
+        assert_eq!(char_lit("'\\u{10}'").unwrap(), ("", lit("\\u{10}")));
-        assert_eq!(char_lit("'\\u{aa}'").unwrap(), ("", "\\u{aa}"));
+        assert_eq!(char_lit("'\\u{aa}'").unwrap(), ("", lit("\\u{aa}")));
-        assert_eq!(char_lit("'\\u{10FFFF}'").unwrap(), ("", "\\u{10FFFF}"));
+        assert_eq!(char_lit("'\\u{10FFFF}'").unwrap(), ("", lit("\\u{10FFFF}")));
+
+        // Check with `b` prefix.
+        assert_eq!(
+            char_lit("b'a'").unwrap(),
+            (
+                "",
+                crate::CharLit {
+                    prefix: Some(crate::CharPrefix::Binary),
+                    content: "a"
+                }
+            )
+        );
 
         // Should fail.
         assert!(char_lit("''").is_err());
@@ -851,4 +936,29 @@ mod test {
         assert!(char_lit("'\\u{}'").is_err());
         assert!(char_lit("'\\u{110000}'").is_err());
     }
+
+    #[test]
+    fn test_str_lit() {
+        assert_eq!(
+            str_lit(r#"b"hello""#).unwrap(),
+            (
+                "",
+                StrLit {
+                    prefix: Some(StrPrefix::Binary),
+                    content: "hello"
+                }
+            )
+        );
+        assert_eq!(
+            str_lit(r#"c"hello""#).unwrap(),
+            (
+                "",
+                StrLit {
+                    prefix: Some(StrPrefix::CLike),
+                    content: "hello"
+                }
+            )
+        );
+        assert!(str_lit(r#"d"hello""#).is_err());
+    }
 }

rinja_parser/src/node.rs

@@ -9,8 +9,8 @@ use nom::sequence::{delimited, pair, preceded, tuple};
 
 use crate::memchr_splitter::{Splitter1, Splitter2, Splitter3};
 use crate::{
-    filter, identifier, is_ws, keyword, not_ws, skip_till, str_lit, ws, ErrorContext, Expr, Filter,
+    filter, identifier, is_ws, keyword, not_ws, skip_till, str_lit_without_prefix, ws,
-    ParseResult, State, Target, WithSpan,
+    ErrorContext, Expr, Filter, ParseResult, State, Target, WithSpan,
 };
 
 #[derive(Debug, PartialEq)]
@@ -562,7 +562,7 @@ impl<'a> Import<'a> {
             opt(Whitespace::parse),
             ws(keyword("import")),
             cut(tuple((
-                ws(str_lit),
+                ws(str_lit_without_prefix),
                 ws(keyword("as")),
                 cut(pair(ws(identifier), opt(Whitespace::parse))),
             ))),
@@ -938,7 +938,7 @@ impl<'a> Include<'a> {
         let mut p = tuple((
             opt(Whitespace::parse),
             ws(keyword("include")),
-            cut(pair(ws(str_lit), opt(Whitespace::parse))),
+            cut(pair(ws(str_lit_without_prefix), opt(Whitespace::parse))),
         ));
         let (i, (pws, _, (path, nws))) = p(i)?;
         Ok((
@@ -966,7 +966,7 @@ impl<'a> Extends<'a> {
         let (i, (pws, _, (path, nws))) = tuple((
             opt(Whitespace::parse),
             ws(keyword("extends")),
-            cut(pair(ws(str_lit), opt(Whitespace::parse))),
+            cut(pair(ws(str_lit_without_prefix), opt(Whitespace::parse))),
         ))(i)?;
         match (pws, nws) {
             (None, None) => Ok((i, WithSpan::new(Self { path }, start))),

rinja_parser/src/target.rs

@@ -6,8 +6,8 @@ use nom::multi::separated_list1;
 use nom::sequence::{pair, preceded, tuple};
 
 use crate::{
-    bool_lit, char_lit, identifier, keyword, num_lit, path_or_identifier, str_lit, ws,
+    bool_lit, char_lit, identifier, keyword, num_lit, path_or_identifier, str_lit, ws, CharLit,
-    ErrorContext, ParseErr, ParseResult, PathOrIdentifier, State, WithSpan,
+    ErrorContext, ParseErr, ParseResult, PathOrIdentifier, State, StrLit, WithSpan,
 };
 
 #[derive(Clone, Debug, PartialEq)]
@@ -17,8 +17,8 @@ pub enum Target<'a> {
     Array(Vec<&'a str>, Vec<Target<'a>>),
     Struct(Vec<&'a str>, Vec<(&'a str, Target<'a>)>),
     NumLit(&'a str),
-    StrLit(&'a str),
+    StrLit(StrLit<'a>),
-    CharLit(&'a str),
+    CharLit(CharLit<'a>),
     BoolLit(&'a str),
     Path(Vec<&'a str>),
     OrChain(Vec<Target<'a>>),

rinja_parser/src/tests.rs

@@ -1,5 +1,5 @@
-use super::node::{Lit, Whitespace, Ws};
+use crate::node::{Lit, Whitespace, Ws};
-use super::{Ast, Expr, Filter, Node, Syntax, WithSpan};
+use crate::{Ast, Expr, Filter, Node, StrLit, Syntax, WithSpan};
 
 impl<T> WithSpan<'static, T> {
     fn no_span(inner: T) -> Self {
@@ -218,7 +218,10 @@ fn test_parse_var_call() {
             WithSpan::no_span(Expr::Call(
                 Box::new(WithSpan::no_span(Expr::Var("function"))),
                 vec![
-                    WithSpan::no_span(Expr::StrLit("123")),
+                    WithSpan::no_span(Expr::StrLit(StrLit {
+                        content: "123",
+                        prefix: None,
+                    })),
                     WithSpan::no_span(Expr::NumLit("3"))
                 ]
             )),
@@ -259,7 +262,10 @@ fn test_parse_path_call() {
             WithSpan::no_span(Expr::Call(
                 Box::new(WithSpan::no_span(Expr::Path(vec!["self", "function"]))),
                 vec![
-                    WithSpan::no_span(Expr::StrLit("123")),
+                    WithSpan::no_span(Expr::StrLit(StrLit {
+                        content: "123",
+                        prefix: None,
+                    })),
                     WithSpan::no_span(Expr::NumLit("3"))
                 ],
             ),)

testing/tests/literal.rs

@@ -0,0 +1,61 @@
+use rinja::Template;
+
+#[derive(Template)]
+#[template(source = "{% if x == b'a' %}bc{% endif %}", ext = "txt")]
+struct Expr {
+    x: u8,
+}
+
+#[test]
+fn test_prefix_char_literal_in_expr() {
+    let t = Expr { x: b'a' };
+    assert_eq!(t.render().unwrap(), "bc");
+}
+
+#[derive(Template)]
+#[template(
+    source = "{% if let Some(b'a') = Some(b'a') %}bc{% endif %}
+{%- if data == [b'h', b'i'] %} hoy{% endif %}",
+    ext = "txt"
+)]
+struct Target {
+    data: &'static [u8],
+}
+
+#[test]
+fn test_prefix_char_literal_in_target() {
+    let t = Target { data: b"hi" };
+    assert_eq!(t.render().unwrap(), "bc hoy");
+}
+
+#[derive(Template)]
+#[template(
+    source = r#"{% if x == b"hi".as_slice() %}bc{% endif %}
+{%- if c"a".to_bytes_with_nul() == b"a\0" %} hoy{% endif %}"#,
+    ext = "txt"
+)]
+struct ExprStr {
+    x: &'static [u8],
+}
+
+#[test]
+fn test_prefix_str_literal_in_expr() {
+    let t = ExprStr { x: b"hi" };
+    assert_eq!(t.render().unwrap(), "bc hoy");
+}
+
+#[derive(Template)]
+#[template(
+    source = r#"{% if let Some(b"hi") = Some(data) %}bc{% endif %}
+{%- if let x = c"hi" %} hoy{% endif %}"#,
+    ext = "txt"
+)]
+struct TargetStr {
+    data: [u8; 2],
+}
+
+#[test]
+fn test_prefix_str_literal_in_target() {
+    let t = TargetStr { data: *b"hi" };
+    assert_eq!(t.render().unwrap(), "bc hoy");
+}

testing/tests/ui/escape-filter-invalid-kind.rs

@@ -0,0 +1,17 @@
+use rinja::Template;
+
+#[derive(Template)]
+#[template(
+    source = r#"{{ "a"|escape(b"none") }}"#,
+    ext = "txt",
+)]
+struct BadEscapeKind;
+
+#[derive(Template)]
+#[template(
+    source = r#"{{ "a"|escape(c"none") }}"#,
+    ext = "txt",
+)]
+struct BadEscapeKind2;
+
+fn main() {}

testing/tests/ui/escape-filter-invalid-kind.stderr

@@ -0,0 +1,15 @@
+error: invalid escaper `b"none"`. Expected a string, found a slice
+ --> BadEscapeKind.txt:1:14
+       "b\"none\") }}"
+ --> tests/ui/escape-filter-invalid-kind.rs:5:14
+  |
+5 |     source = r#"{{ "a"|escape(b"none") }}"#,
+  |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: invalid escaper `b"none"`. Expected a string, found a CStr
+ --> BadEscapeKind2.txt:1:14
+       "c\"none\") }}"
+  --> tests/ui/escape-filter-invalid-kind.rs:12:14
+   |
+12 |     source = r#"{{ "a"|escape(c"none") }}"#,
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^