itsscb/askama
1
0
Fork 0
mirror of https://github.com/askama-rs/askama.git synced 2025-09-30 14:31:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #140 from GuillaumeGomez/literal-prefixes

Browse Source 
Add support for b'a', b"a" and c"a"
This commit is contained in:
René Kijewski 2024-08-19 00:26:14 +02:00 committed by GitHub
commit cd3ff13e05
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 299 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
rinja_derive/src/generator.rs
View File

@ -9,7 +9,7 @@ use std::{cmp, hash, mem, str};
use parser::node::{
Call, Comment, Cond, CondTest, FilterBlock, If, Include, Let, Lit, Loop, Match, Whitespace, Ws,
};
use parser::{Expr, Filter, Node, Target, WithSpan};
use parser::{CharLit, CharPrefix, Expr, Filter, Node, StrLit, StrPrefix, Target, WithSpan};
use quote::quote;
use rustc_hash::FxBuildHasher;
@ -1270,9 +1270,18 @@ impl<'a> Generator<'a> {
}
// for now, we only escape strings and chars at compile time
let lit = match &**s {
Expr::StrLit(input) => InputKind::StrLit(input),
Expr::CharLit(input) => InputKind::CharLit(input),
let (lit, escape_prefix) = match &**s {
Expr::StrLit(StrLit { prefix, content }) => {
(InputKind::StrLit(content), prefix.map(|p| p.to_char()))
}
Expr::CharLit(CharLit { prefix, content }) => (
InputKind::CharLit(content),
if *prefix == Some(CharPrefix::Binary) {
Some('b')
} else {
None
},
),
_ => return None,
};
@ -1314,6 +1323,9 @@ impl<'a> Generator<'a> {
OutputKind::Text => unescaped,
OutputKind::Html => {
let mut escaped = String::with_capacity(unescaped.len() + 20);
if let Some(escape_prefix) = escape_prefix {
escaped.push(escape_prefix);
}
write_escaped_str(&mut escaped, &unescaped).ok()?;
match escaped == unescaped {
true => unescaped,
@ -1478,8 +1490,8 @@ impl<'a> Generator<'a> {
Ok(match **expr {
Expr::BoolLit(s) => self.visit_bool_lit(buf, s),
Expr::NumLit(s) => self.visit_num_lit(buf, s),
Expr::StrLit(s) => self.visit_str_lit(buf, s),
Expr::CharLit(s) => self.visit_char_lit(buf, s),
Expr::StrLit(ref s) => self.visit_str_lit(buf, s),
Expr::CharLit(ref s) => self.visit_char_lit(buf, s),
Expr::Var(s) => self.visit_var(buf, s),
Expr::Path(ref path) => self.visit_path(buf, path),
Expr::Array(ref elements) => self.visit_array(ctx, buf, elements)?,
@ -1699,7 +1711,22 @@ impl<'a> Generator<'a> {
return Err(ctx.generate_error("only two arguments allowed to escape filter", node));
}
let opt_escaper = match args.get(1).map(|expr| &**expr) {
Some(Expr::StrLit(name)) => Some(*name),
Some(Expr::StrLit(StrLit { prefix, content })) => {
if let Some(prefix) = prefix {
let kind = if *prefix == StrPrefix::Binary {
"slice"
} else {
"CStr"
};
return Err(ctx.generate_error(
&format!(
"invalid escaper `b{content:?}`. Expected a string, found a {kind}"
),
&args[1],
));
}
Some(content)
}
Some(_) => {
return Err(ctx.generate_error("invalid escaper type for escape filter", node));
}
@ -1741,7 +1768,7 @@ impl<'a> Generator<'a> {
node: &WithSpan<'_, T>,
) -> Result<DisplayWrap, CompileError> {
if !args.is_empty() {
if let Expr::StrLit(fmt) = *args[0] {
if let Expr::StrLit(ref fmt) = *args[0] {
buf.write("::std::format!(");
self.visit_str_lit(buf, fmt);
if args.len() > 1 {
@ -1763,7 +1790,7 @@ impl<'a> Generator<'a> {
node: &WithSpan<'_, T>,
) -> Result<DisplayWrap, CompileError> {
if let [_, arg2] = args {
if let Expr::StrLit(fmt) = **arg2 {
if let Expr::StrLit(ref fmt) = **arg2 {
buf.write("::std::format!(");
self.visit_str_lit(buf, fmt);
buf.write(',');
@ -2077,13 +2104,19 @@ impl<'a> Generator<'a> {
DisplayWrap::Unwrapped
}
fn visit_str_lit(&mut self, buf: &mut Buffer, s: &str) -> DisplayWrap {
buf.write(format_args!("\"{s}\""));
fn visit_str_lit(&mut self, buf: &mut Buffer, s: &StrLit<'_>) -> DisplayWrap {
if let Some(prefix) = s.prefix {
buf.write(prefix.to_char());
}
buf.write(format_args!("\"{}\"", s.content));
DisplayWrap::Unwrapped
}
fn visit_char_lit(&mut self, buf: &mut Buffer, s: &str) -> DisplayWrap {
buf.write(format_args!("'{s}'"));
fn visit_char_lit(&mut self, buf: &mut Buffer, c: &CharLit<'_>) -> DisplayWrap {
if c.prefix == Some(CharPrefix::Binary) {
buf.write('b');
}
buf.write(format_args!("'{}'", c.content));
DisplayWrap::Unwrapped
}

11
rinja_parser/src/expr.rs
View File

@ -10,11 +10,10 @@ use nom::error_position;
use nom::multi::{fold_many0, many0, separated_list0};
use nom::sequence::{pair, preceded, terminated, tuple};
use super::{
char_lit, filter, identifier, keyword, not_ws, num_lit, path_or_identifier, str_lit, ws, Level,
PathOrIdentifier,
use crate::{
char_lit, filter, identifier, keyword, not_ws, num_lit, path_or_identifier, str_lit, ws,
CharLit, ErrorContext, Level, ParseResult, PathOrIdentifier, StrLit, WithSpan,
};
use crate::{ErrorContext, ParseResult, WithSpan};
macro_rules! expr_prec_layer {
( $name:ident, $inner:ident, $op:expr ) => {
@ -37,8 +36,8 @@ macro_rules! expr_prec_layer {
pub enum Expr<'a> {
BoolLit(bool),
NumLit(&'a str),
StrLit(&'a str),
CharLit(&'a str),
StrLit(StrLit<'a>),
CharLit(CharLit<'a>),
Var(&'a str),
Path(Vec<&'a str>),
Array(Vec<WithSpan<'a, Expr<'a>>>),

162
rinja_parser/src/lib.rs
View File

@ -416,7 +416,36 @@ fn separated_digits(radix: u32, start: bool) -> impl Fn(&str) -> ParseResult<'_>
}
}
fn str_lit(i: &str) -> ParseResult<'_> {
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum StrPrefix {
Binary,
CLike,
}
impl StrPrefix {
pub fn to_char(self) -> char {
match self {
Self::Binary => 'b',
Self::CLike => 'c',
}
}
}
impl fmt::Display for StrPrefix {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use std::fmt::Write;
f.write_char(self.to_char())
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct StrLit<'a> {
pub prefix: Option<StrPrefix>,
pub content: &'a str,
}
fn str_lit_without_prefix(i: &str) -> ParseResult<'_> {
let (i, s) = delimited(
char('"'),
opt(escaped(is_not("\\\""), '\\', anychar)),
@ -425,15 +454,40 @@ fn str_lit(i: &str) -> ParseResult<'_> {
Ok((i, s.unwrap_or_default()))
}
fn str_lit(i: &str) -> Result<(&str, StrLit<'_>), ParseErr<'_>> {
let (i, (prefix, content)) =
tuple((opt(alt((char('b'), char('c')))), str_lit_without_prefix))(i)?;
let prefix = match prefix {
Some('b') => Some(StrPrefix::Binary),
Some('c') => Some(StrPrefix::CLike),
_ => None,
};
Ok((i, StrLit { prefix, content }))
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum CharPrefix {
Binary,
}
#[derive(Clone, Debug, PartialEq)]
pub struct CharLit<'a> {
pub prefix: Option<CharPrefix>,
pub content: &'a str,
}
// Information about allowed character escapes is available at:
// <https://doc.rust-lang.org/reference/tokens.html#character-literals>.
fn char_lit(i: &str) -> ParseResult<'_> {
fn char_lit(i: &str) -> Result<(&str, CharLit<'_>), ParseErr<'_>> {
let start = i;
let (i, s) = delimited(
char('\''),
opt(escaped(is_not("\\\'"), '\\', anychar)),
char('\''),
)(i)?;
let (i, (b_prefix, s)) = tuple((
opt(char('b')),
delimited(
char('\''),
opt(escaped(is_not("\\\'"), '\\', anychar)),
char('\''),
),
))(i)?;
let Some(s) = s else {
return Err(nom::Err::Failure(ErrorContext::new(
@ -449,7 +503,15 @@ fn char_lit(i: &str) -> ParseResult<'_> {
};
let (nb, max_value, err1, err2) = match c {
Char::Literal | Char::Escaped => return Ok((i, s)),
Char::Literal | Char::Escaped => {
return Ok((
i,
CharLit {
prefix: b_prefix.map(|_| CharPrefix::Binary),
content: s,
},
));
}
Char::AsciiEscape(nb) => (
nb,
// `0x7F` is the maximum value for a `\x` escaped character.
@ -473,7 +535,13 @@ fn char_lit(i: &str) -> ParseResult<'_> {
return Err(nom::Err::Failure(ErrorContext::new(err2, start)));
}
Ok((i, s))
Ok((
i,
CharLit {
prefix: b_prefix.map(|_| CharPrefix::Binary),
content: s,
},
))
}
/// Represents the different kinds of char declarations:
@ -775,7 +843,7 @@ const PRIMITIVE_TYPES: &[&str] = &{
mod test {
use std::path::Path;
use super::{char_lit, num_lit, strip_common};
use super::{char_lit, num_lit, str_lit, strip_common, StrLit, StrPrefix};
#[test]
fn test_strip_common() {
@ -820,26 +888,43 @@ mod test {
#[test]
fn test_char_lit() {
assert_eq!(char_lit("'a'").unwrap(), ("", "a"));
assert_eq!(char_lit("'字'").unwrap(), ("", "字"));
let lit = |s: &'static str| crate::CharLit {
prefix: None,
content: s,
};
assert_eq!(char_lit("'a'").unwrap(), ("", lit("a")));
assert_eq!(char_lit("'字'").unwrap(), ("", lit("字")));
// Escaped single characters.
assert_eq!(char_lit("'\\\"'").unwrap(), ("", "\\\""));
assert_eq!(char_lit("'\\''").unwrap(), ("", "\\'"));
assert_eq!(char_lit("'\\t'").unwrap(), ("", "\\t"));
assert_eq!(char_lit("'\\n'").unwrap(), ("", "\\n"));
assert_eq!(char_lit("'\\r'").unwrap(), ("", "\\r"));
assert_eq!(char_lit("'\\0'").unwrap(), ("", "\\0"));
assert_eq!(char_lit("'\\\"'").unwrap(), ("", lit("\\\"")));
assert_eq!(char_lit("'\\''").unwrap(), ("", lit("\\'")));
assert_eq!(char_lit("'\\t'").unwrap(), ("", lit("\\t")));
assert_eq!(char_lit("'\\n'").unwrap(), ("", lit("\\n")));
assert_eq!(char_lit("'\\r'").unwrap(), ("", lit("\\r")));
assert_eq!(char_lit("'\\0'").unwrap(), ("", lit("\\0")));
// Escaped ascii characters (up to `0x7F`).
assert_eq!(char_lit("'\\x12'").unwrap(), ("", "\\x12"));
assert_eq!(char_lit("'\\x02'").unwrap(), ("", "\\x02"));
assert_eq!(char_lit("'\\x6a'").unwrap(), ("", "\\x6a"));
assert_eq!(char_lit("'\\x7F'").unwrap(), ("", "\\x7F"));
assert_eq!(char_lit("'\\x12'").unwrap(), ("", lit("\\x12")));
assert_eq!(char_lit("'\\x02'").unwrap(), ("", lit("\\x02")));
assert_eq!(char_lit("'\\x6a'").unwrap(), ("", lit("\\x6a")));
assert_eq!(char_lit("'\\x7F'").unwrap(), ("", lit("\\x7F")));
// Escaped unicode characters (up to `0x10FFFF`).
assert_eq!(char_lit("'\\u{A}'").unwrap(), ("", "\\u{A}"));
assert_eq!(char_lit("'\\u{10}'").unwrap(), ("", "\\u{10}"));
assert_eq!(char_lit("'\\u{aa}'").unwrap(), ("", "\\u{aa}"));
assert_eq!(char_lit("'\\u{10FFFF}'").unwrap(), ("", "\\u{10FFFF}"));
assert_eq!(char_lit("'\\u{A}'").unwrap(), ("", lit("\\u{A}")));
assert_eq!(char_lit("'\\u{10}'").unwrap(), ("", lit("\\u{10}")));
assert_eq!(char_lit("'\\u{aa}'").unwrap(), ("", lit("\\u{aa}")));
assert_eq!(char_lit("'\\u{10FFFF}'").unwrap(), ("", lit("\\u{10FFFF}")));
// Check with `b` prefix.
assert_eq!(
char_lit("b'a'").unwrap(),
(
"",
crate::CharLit {
prefix: Some(crate::CharPrefix::Binary),
content: "a"
}
)
);
// Should fail.
assert!(char_lit("''").is_err());
@ -851,4 +936,29 @@ mod test {
assert!(char_lit("'\\u{}'").is_err());
assert!(char_lit("'\\u{110000}'").is_err());
}
#[test]
fn test_str_lit() {
assert_eq!(
str_lit(r#"b"hello""#).unwrap(),
(
"",
StrLit {
prefix: Some(StrPrefix::Binary),
content: "hello"
}
)
);
assert_eq!(
str_lit(r#"c"hello""#).unwrap(),
(
"",
StrLit {
prefix: Some(StrPrefix::CLike),
content: "hello"
}
)
);
assert!(str_lit(r#"d"hello""#).is_err());
}
}

10
rinja_parser/src/node.rs
View File

@ -9,8 +9,8 @@ use nom::sequence::{delimited, pair, preceded, tuple};
use crate::memchr_splitter::{Splitter1, Splitter2, Splitter3};
use crate::{
filter, identifier, is_ws, keyword, not_ws, skip_till, str_lit, ws, ErrorContext, Expr, Filter,
ParseResult, State, Target, WithSpan,
filter, identifier, is_ws, keyword, not_ws, skip_till, str_lit_without_prefix, ws,
ErrorContext, Expr, Filter, ParseResult, State, Target, WithSpan,
};
#[derive(Debug, PartialEq)]
@ -562,7 +562,7 @@ impl<'a> Import<'a> {
opt(Whitespace::parse),
ws(keyword("import")),
cut(tuple((
ws(str_lit),
ws(str_lit_without_prefix),
ws(keyword("as")),
cut(pair(ws(identifier), opt(Whitespace::parse))),
))),
@ -938,7 +938,7 @@ impl<'a> Include<'a> {
let mut p = tuple((
opt(Whitespace::parse),
ws(keyword("include")),
cut(pair(ws(str_lit), opt(Whitespace::parse))),
cut(pair(ws(str_lit_without_prefix), opt(Whitespace::parse))),
));
let (i, (pws, _, (path, nws))) = p(i)?;
Ok((
@ -966,7 +966,7 @@ impl<'a> Extends<'a> {
let (i, (pws, _, (path, nws))) = tuple((
opt(Whitespace::parse),
ws(keyword("extends")),
cut(pair(ws(str_lit), opt(Whitespace::parse))),
cut(pair(ws(str_lit_without_prefix), opt(Whitespace::parse))),
))(i)?;
match (pws, nws) {
(None, None) => Ok((i, WithSpan::new(Self { path }, start))),

8
rinja_parser/src/target.rs
View File

@ -6,8 +6,8 @@ use nom::multi::separated_list1;
use nom::sequence::{pair, preceded, tuple};
use crate::{
bool_lit, char_lit, identifier, keyword, num_lit, path_or_identifier, str_lit, ws,
ErrorContext, ParseErr, ParseResult, PathOrIdentifier, State, WithSpan,
bool_lit, char_lit, identifier, keyword, num_lit, path_or_identifier, str_lit, ws, CharLit,
ErrorContext, ParseErr, ParseResult, PathOrIdentifier, State, StrLit, WithSpan,
};
#[derive(Clone, Debug, PartialEq)]
@ -17,8 +17,8 @@ pub enum Target<'a> {
Array(Vec<&'a str>, Vec<Target<'a>>),
Struct(Vec<&'a str>, Vec<(&'a str, Target<'a>)>),
NumLit(&'a str),
StrLit(&'a str),
CharLit(&'a str),
StrLit(StrLit<'a>),
CharLit(CharLit<'a>),
BoolLit(&'a str),
Path(Vec<&'a str>),
OrChain(Vec<Target<'a>>),

14
rinja_parser/src/tests.rs
View File

@ -1,5 +1,5 @@
use super::node::{Lit, Whitespace, Ws};
use super::{Ast, Expr, Filter, Node, Syntax, WithSpan};
use crate::node::{Lit, Whitespace, Ws};
use crate::{Ast, Expr, Filter, Node, StrLit, Syntax, WithSpan};
impl<T> WithSpan<'static, T> {
fn no_span(inner: T) -> Self {
@ -218,7 +218,10 @@ fn test_parse_var_call() {
WithSpan::no_span(Expr::Call(
Box::new(WithSpan::no_span(Expr::Var("function"))),
vec![
WithSpan::no_span(Expr::StrLit("123")),
WithSpan::no_span(Expr::StrLit(StrLit {
content: "123",
prefix: None,
})),
WithSpan::no_span(Expr::NumLit("3"))
]
)),
@ -259,7 +262,10 @@ fn test_parse_path_call() {
WithSpan::no_span(Expr::Call(
Box::new(WithSpan::no_span(Expr::Path(vec!["self", "function"]))),
vec![
WithSpan::no_span(Expr::StrLit("123")),
WithSpan::no_span(Expr::StrLit(StrLit {
content: "123",
prefix: None,
})),
WithSpan::no_span(Expr::NumLit("3"))
],
),)

61
testing/tests/literal.rs Normal file
View File

@ -0,0 +1,61 @@
use rinja::Template;
#[derive(Template)]
#[template(source = "{% if x == b'a' %}bc{% endif %}", ext = "txt")]
struct Expr {
x: u8,
}
#[test]
fn test_prefix_char_literal_in_expr() {
let t = Expr { x: b'a' };
assert_eq!(t.render().unwrap(), "bc");
}
#[derive(Template)]
#[template(
source = "{% if let Some(b'a') = Some(b'a') %}bc{% endif %}
{%- if data == [b'h', b'i'] %} hoy{% endif %}",
ext = "txt"
)]
struct Target {
data: &'static [u8],
}
#[test]
fn test_prefix_char_literal_in_target() {
let t = Target { data: b"hi" };
assert_eq!(t.render().unwrap(), "bc hoy");
}
#[derive(Template)]
#[template(
source = r#"{% if x == b"hi".as_slice() %}bc{% endif %}
{%- if c"a".to_bytes_with_nul() == b"a\0" %} hoy{% endif %}"#,
ext = "txt"
)]
struct ExprStr {
x: &'static [u8],
}
#[test]
fn test_prefix_str_literal_in_expr() {
let t = ExprStr { x: b"hi" };
assert_eq!(t.render().unwrap(), "bc hoy");
}
#[derive(Template)]
#[template(
source = r#"{% if let Some(b"hi") = Some(data) %}bc{% endif %}
{%- if let x = c"hi" %} hoy{% endif %}"#,
ext = "txt"
)]
struct TargetStr {
data: [u8; 2],
}
#[test]
fn test_prefix_str_literal_in_target() {
let t = TargetStr { data: *b"hi" };
assert_eq!(t.render().unwrap(), "bc hoy");
}

17
testing/tests/ui/escape-filter-invalid-kind.rs Normal file
View File

@ -0,0 +1,17 @@
use rinja::Template;
#[derive(Template)]
#[template(
source = r#"{{ "a"|escape(b"none") }}"#,
ext = "txt",
)]
struct BadEscapeKind;
#[derive(Template)]
#[template(
source = r#"{{ "a"|escape(c"none") }}"#,
ext = "txt",
)]
struct BadEscapeKind2;
fn main() {}

15
testing/tests/ui/escape-filter-invalid-kind.stderr Normal file
View File

@ -0,0 +1,15 @@
error: invalid escaper `b"none"`. Expected a string, found a slice
--> BadEscapeKind.txt:1:14
"b\"none\") }}"
--> tests/ui/escape-filter-invalid-kind.rs:5:14
|
5 | source = r#"{{ "a"|escape(b"none") }}"#,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
error: invalid escaper `b"none"`. Expected a string, found a CStr
--> BadEscapeKind2.txt:1:14
"c\"none\") }}"
--> tests/ui/escape-filter-invalid-kind.rs:12:14
|
12 | source = r#"{{ "a"|escape(c"none") }}"#,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^