itsscb/askama
1
0
Fork 0
mirror of https://github.com/askama-rs/askama.git synced 2025-09-27 04:50:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

parser: make State part of InputStream

Browse Source
This commit is contained in:
René Kijewski 2025-08-14 08:10:01 +02:00 committed by René Kijewski
parent e5189b933d
commit 81252cd0a2
5 changed files with 615 additions and 734 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

382
askama_parser/src/expr.rs
View File

@ -8,35 +8,30 @@ use winnow::token::{any, one_of, take, take_until};
use crate::node::CondTest;
use crate::{
CharLit, ErrorContext, HashSet, InputStream, Level, Num, ParseResult, PathOrIdentifier, StrLit,
StrPrefix, WithSpan, can_be_variable_name, char_lit, cut_error, filter, identifier, keyword,
not_suffix_with_hash, num_lit, path_or_identifier, skip_ws0, skip_ws1, str_lit, ws,
CharLit, ErrorContext, HashSet, InputStream, Num, ParseResult, PathOrIdentifier, StrLit,
StrPrefix, WithSpan, can_be_variable_name, char_lit, cut_error, filter, identifier,
is_rust_keyword, keyword, not_suffix_with_hash, num_lit, path_or_identifier, skip_ws0,
skip_ws1, str_lit, ws,
};
macro_rules! expr_prec_layer {
( $name:ident, $inner:ident, $op:expr ) => {
fn $name(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
expr_prec_layer(i, level, Expr::$inner, |i: &mut _| $op.parse_next(i))
fn $name(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
expr_prec_layer(i, Expr::$inner, |i: &mut _| $op.parse_next(i))
}
};
}
fn expr_prec_layer<'a>(
i: &mut InputStream<'a>,
level: Level<'_>,
inner: fn(&mut InputStream<'a>, Level<'_>) -> ParseResult<'a, WithSpan<Box<Expr<'a>>>>,
op: fn(&mut InputStream<'a>) -> ParseResult<'a>,
fn expr_prec_layer<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
inner: fn(&mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Expr<'a>>>>,
op: fn(&mut InputStream<'a, 'l>) -> ParseResult<'a>,
) -> ParseResult<'a, WithSpan<Box<Expr<'a>>>> {
let mut expr = inner(i, level)?;
let mut expr = inner(i)?;
let mut i_before = *i;
let mut level_guard = level.guard();
while let Some(((op, span), rhs)) =
opt((ws(op.with_span()), |i: &mut _| inner(i, level))).parse_next(i)?
{
let mut level_guard = i.state.level.guard();
while let Some(((op, span), rhs)) = opt((ws(op.with_span()), inner)).parse_next(i)? {
level_guard.nest(&i_before)?;
expr = WithSpan::new(Box::new(Expr::BinOp(BinOp { op, lhs: expr, rhs })), span);
i_before = *i;
@ -165,7 +160,7 @@ pub struct PathComponent<'a> {
pub generics: Option<WithSpan<Vec<WithSpan<TyGenerics<'a>>>>>,
}
impl<'a> PathComponent<'a> {
impl<'a: 'l, 'l> PathComponent<'a> {
#[inline]
pub fn new_with_name(name: WithSpan<&'a str>) -> Self {
Self {
@ -174,10 +169,10 @@ impl<'a> PathComponent<'a> {
}
}
pub(crate) fn parse(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, Self> {
pub(crate) fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
let mut p = (
identifier.with_span(),
opt(preceded(ws("::"), |i: &mut _| TyGenerics::args(i, level))),
opt(preceded(ws("::"), TyGenerics::args)),
);
let ((name, name_span), generics) = p.parse_next(i)?;
Ok(Self {
@ -242,12 +237,11 @@ pub struct BinOp<'a> {
pub rhs: WithSpan<Box<Expr<'a>>>,
}
impl<'a> Expr<'a> {
impl<'a: 'l, 'l> Expr<'a> {
pub(super) fn arguments(
i: &mut InputStream<'a>,
level: Level<'_>,
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, WithSpan<Vec<WithSpan<Box<Self>>>>> {
let _level_guard = level.nest(i)?;
let _level_guard = i.state.level.nest(i)?;
let mut named_arguments = HashSet::default();
let mut p = (
ws('('.span()),
@ -261,8 +255,8 @@ impl<'a> Expr<'a> {
let has_named_arguments = !named_arguments.is_empty();
let mut p = alt((
move |i: &mut _| Self::named_argument(i, level, named_arguments),
move |i: &mut _| Self::parse(i, level, false),
move |i: &mut _| Self::named_argument(i, named_arguments),
move |i: &mut _| Self::parse(i, false),
));
let expr = p.parse_next(i)?;
if has_named_arguments && !matches!(**expr, Self::NamedArgument(..)) {
@ -287,13 +281,12 @@ impl<'a> Expr<'a> {
}
fn named_argument(
i: &mut InputStream<'a>,
level: Level<'_>,
i: &mut InputStream<'a, 'l>,
named_arguments: &mut HashSet<&'a str>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (((argument, arg_span), _, value), span) =
(identifier.with_span(), ws('='), move |i: &mut _| {
Self::parse(i, level, false)
Self::parse(i, false)
})
.with_span()
.parse_next(i)?;
@ -317,25 +310,20 @@ impl<'a> Expr<'a> {
}
pub(super) fn parse(
i: &mut InputStream<'a>,
level: Level<'_>,
i: &mut InputStream<'a, 'l>,
allow_underscore: bool,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let _level_guard = level.nest(i)?;
Self::range(i, level, allow_underscore)
let _level_guard = i.state.level.nest(i)?;
Self::range(i, allow_underscore)
}
fn range(
i: &mut InputStream<'a>,
level: Level<'_>,
i: &mut InputStream<'a, 'l>,
allow_underscore: bool,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let range_right = move |i: &mut InputStream<'a>| {
let ((op, span), rhs) = (
ws(alt(("..=", "..")).with_span()),
opt(move |i: &mut _| Self::or(i, level)),
)
.parse_next(i)?;
let range_right = move |i: &mut InputStream<'a, 'l>| {
let ((op, span), rhs) =
(ws(alt(("..=", "..")).with_span()), opt(Self::or)).parse_next(i)?;
Ok((op, rhs, span))
};
@ -345,19 +333,17 @@ impl<'a> Expr<'a> {
});
// `expr..expr` or `expr..`
let range_from = (move |i: &mut _| Self::or(i, level), opt(range_right)).map(
move |(lhs, rhs)| match rhs {
Some((op, rhs, span)) => WithSpan::new(
Box::new(Self::Range(Range {
op,
lhs: Some(lhs),
rhs,
})),
span,
),
None => lhs,
},
);
let range_from = (Self::or, opt(range_right)).map(move |(lhs, rhs)| match rhs {
Some((op, rhs, span)) => WithSpan::new(
Box::new(Self::Range(Range {
op,
lhs: Some(lhs),
rhs,
})),
span,
),
None => lhs,
});
let expr = alt((range_to, range_from)).parse_next(i)?;
check_expr(
@ -373,14 +359,10 @@ impl<'a> Expr<'a> {
expr_prec_layer!(or, and, "||");
expr_prec_layer!(and, compare, "&&");
fn compare(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn compare(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let mut parse_op = ws(alt(("==", "!=", ">=", ">", "<=", "<")).with_span());
let (expr, rhs) = (
|i: &mut _| Self::bor(i, level),
opt((parse_op.by_ref(), |i: &mut _| Self::bor(i, level))),
)
.parse_next(i)?;
let (expr, rhs) = (Self::bor, opt((parse_op.by_ref(), Self::bor))).parse_next(i)?;
let Some(((op, span), rhs)) = rhs else {
return Ok(expr);
};
@ -405,15 +387,14 @@ impl<'a> Expr<'a> {
expr_prec_layer!(shifts, addsub, alt((">>", "<<")));
expr_prec_layer!(addsub, concat, alt(("+", "-")));
fn concat(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn concat(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
#[allow(clippy::type_complexity)]
fn concat_expr<'a>(
i: &mut InputStream<'a>,
level: Level<'_>,
fn concat_expr<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, Option<(WithSpan<Box<Expr<'a>>>, std::ops::Range<usize>)>> {
let ws1 = |i: &mut _| opt(skip_ws1).parse_next(i);
let tilde = (ws1, '~', ws1).with_span();
let data = opt((tilde, |i: &mut _| Expr::muldivmod(i, level))).parse_next(i)?;
let data = opt((tilde, Expr::muldivmod)).parse_next(i)?;
let Some((((t1, _, t2), span), expr)) = data else {
return Ok(None);
@ -425,11 +406,11 @@ impl<'a> Expr<'a> {
Ok(Some((expr, span)))
}
let expr = Self::muldivmod(i, level)?;
let expr2 = concat_expr(i, level)?;
let expr = Self::muldivmod(i)?;
let expr2 = concat_expr(i)?;
if let Some((expr2, span)) = expr2 {
let mut exprs = vec![expr, expr2];
while let Some((expr, _)) = concat_expr(i, level)? {
while let Some((expr, _)) = concat_expr(i)? {
exprs.push(expr);
}
Ok(WithSpan::new(Box::new(Self::Concat(exprs)), span))
@ -440,13 +421,13 @@ impl<'a> Expr<'a> {
expr_prec_layer!(muldivmod, is_as, alt(("*", "/", "%")));
fn is_as(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let lhs = Self::filtered(i, level)?;
fn is_as(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let lhs = Self::filtered(i)?;
let checkpoint = i.checkpoint();
let rhs = opt(ws(identifier.with_span())).parse_next(i)?;
match rhs {
Some(("is", span)) => Self::is_as_handle_is(i, lhs, span),
Some(("as", span)) => Self::is_as_handle_as(i, level, lhs, span),
Some(("as", span)) => Self::is_as_handle_as(i, lhs, span),
_ => {
i.reset(&checkpoint);
Ok(lhs)
@ -455,7 +436,7 @@ impl<'a> Expr<'a> {
}
fn is_as_handle_is(
i: &mut InputStream<'a>,
i: &mut InputStream<'a, 'l>,
lhs: WithSpan<Box<Expr<'a>>>,
span: std::ops::Range<usize>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
@ -483,12 +464,11 @@ impl<'a> Expr<'a> {
}
fn is_as_handle_as(
i: &mut InputStream<'a>,
level: Level<'_>,
i: &mut InputStream<'a, 'l>,
lhs: WithSpan<Box<Expr<'a>>>,
span: std::ops::Range<usize>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let target = opt(|i: &mut _| path_or_identifier(i, level)).parse_next(i)?;
let target = opt(path_or_identifier).parse_next(i)?;
let Some(PathOrIdentifier::Identifier(target)) = target else {
return cut_error!(
"`as` operator expects the name of a primitive type on its right-hand side, \
@ -516,14 +496,12 @@ impl<'a> Expr<'a> {
}
}
fn filtered(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let mut res = Self::prefix(i, level)?;
fn filtered(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let mut res = Self::prefix(i)?;
let mut level_guard = level.guard();
let mut level_guard = i.state.level.guard();
let mut i_before = *i;
while let Some((mut filter, span)) =
opt(ws((|i: &mut _| filter(i, level)).with_span())).parse_next(i)?
{
while let Some((mut filter, span)) = opt(ws(filter.with_span())).parse_next(i)? {
level_guard.nest(&i_before)?;
filter.arguments.insert(0, res);
res = WithSpan::new(Box::new(Self::Filter(filter)), span);
@ -532,11 +510,11 @@ impl<'a> Expr<'a> {
Ok(res)
}
fn prefix(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn prefix(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
// This is a rare place where we create recursion in the parsed AST
// without recursing the parser call stack. However, this can lead
// to stack overflows in drop glue when the AST is very deep.
let mut level_guard = level.guard();
let mut level_guard = i.state.level.guard();
let mut i_before = *i;
let mut ops = vec![];
while let Some(op) = opt(ws(alt(("!", "-", "*", "&")).with_span())).parse_next(i)? {
@ -545,7 +523,7 @@ impl<'a> Expr<'a> {
i_before = *i;
}
let mut expr = Suffix::parse(i, level)?;
let mut expr = Suffix::parse(i)?;
for (op, span) in ops.into_iter().rev() {
expr = WithSpan::new(Box::new(Self::Unary(op, expr)), span);
}
@ -553,44 +531,36 @@ impl<'a> Expr<'a> {
Ok(expr)
}
fn single(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn single(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
alt((
Self::num,
Self::str,
Self::char,
move |i: &mut _| Self::path_var_bool(i, level),
move |i: &mut _| Self::array(i, level),
move |i: &mut _| Self::group(i, level),
Self::path_var_bool,
Self::array,
Self::group,
))
.parse_next(i)
}
fn group(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn group(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
(skip_ws0, peek('(')).parse_next(i)?;
Self::group_actually(i, level)
Self::group_actually(i)
}
// `Self::group()` is quite big. Let's only put it on the stack if needed.
#[inline(never)]
fn group_actually(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (expr, span) = cut_err(preceded('(', |i: &mut _| {
Self::group_actually_inner(i, level)
}))
.with_span()
.parse_next(i)?;
fn group_actually(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (expr, span) = cut_err(preceded('(', Self::group_actually_inner))
.with_span()
.parse_next(i)?;
Ok(WithSpan::new(expr, span))
}
#[inline]
fn group_actually_inner(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, Box<Self>> {
fn group_actually_inner(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Self>> {
let (expr, comma, closing) = (
ws(opt(|i: &mut _| Self::parse(i, level, true))),
ws(opt(|i: &mut _| Self::parse(i, true))),
opt(terminated(','.span(), skip_ws0)),
opt(')'),
)
@ -617,7 +587,7 @@ impl<'a> Expr<'a> {
let collect_items = opt(separated(
1..,
|i: &mut _| {
exprs.push(Self::parse(i, level, true)?);
exprs.push(Self::parse(i, true)?);
Ok(())
},
ws(','),
@ -639,12 +609,12 @@ impl<'a> Expr<'a> {
cut_error!(msg, span)
}
fn array(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn array(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (array, span) = preceded(
ws('['),
cut_err(terminated(
opt(terminated(
separated(1.., ws(move |i: &mut _| Self::parse(i, level, true)), ','),
separated(1.., ws(move |i: &mut _| Self::parse(i, true)), ','),
ws(opt(',')),
)),
']',
@ -658,13 +628,8 @@ impl<'a> Expr<'a> {
))
}
fn path_var_bool(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (ret, span) = (|i: &mut _| path_or_identifier(i, level))
.with_span()
.parse_next(i)?;
fn path_var_bool(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (ret, span) = path_or_identifier.with_span().parse_next(i)?;
let ret = match ret {
PathOrIdentifier::Path(v) => Box::new(Self::Path(v)),
PathOrIdentifier::Identifier(v) if *v == "true" => Box::new(Self::BoolLit(true)),
@ -674,17 +639,17 @@ impl<'a> Expr<'a> {
Ok(WithSpan::new(ret, span))
}
fn str(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn str(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (s, span) = str_lit.with_span().parse_next(i)?;
Ok(WithSpan::new(Box::new(Self::StrLit(s)), span))
}
fn num(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn num(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let ((num, full), span) = num_lit.with_taken().with_span().parse_next(i)?;
Ok(WithSpan::new(Box::new(Expr::NumLit(full, num)), span))
}
fn char(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Box<Self>>> {
fn char(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Self>>> {
let (c, span) = char_lit.with_span().parse_next(i)?;
Ok(WithSpan::new(Box::new(Self::CharLit(c)), span))
}
@ -722,7 +687,7 @@ impl<'a> Expr<'a> {
}
}
fn token_xor<'a>(i: &mut InputStream<'a>) -> ParseResult<'a> {
fn token_xor<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
let (good, span) = alt((keyword("xor").value(true), '^'.value(false)))
.with_span()
.parse_next(i)?;
@ -733,7 +698,7 @@ fn token_xor<'a>(i: &mut InputStream<'a>) -> ParseResult<'a> {
}
}
fn token_bitand<'a>(i: &mut InputStream<'a>) -> ParseResult<'a> {
fn token_bitand<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
let (good, span) = alt((keyword("bitand").value(true), ('&', not('&')).value(false)))
.with_span()
.parse_next(i)?;
@ -750,12 +715,9 @@ pub struct Filter<'a> {
pub arguments: Vec<WithSpan<Box<Expr<'a>>>>,
}
impl<'a> Filter<'a> {
pub(crate) fn parse(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, Self> {
let mut p = (
ws(|i: &mut _| path_or_identifier(i, level)),
opt(|i: &mut _| Expr::arguments(i, level)),
);
impl<'a: 'l, 'l> Filter<'a> {
pub(crate) fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
let mut p = (ws(path_or_identifier), opt(Expr::arguments));
let (name, arguments) = p.parse_next(i)?;
Ok(Self {
name,
@ -782,17 +744,14 @@ enum Suffix<'a> {
Try,
}
impl<'a> Suffix<'a> {
fn parse(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, WithSpan<Box<Expr<'a>>>> {
let mut level_guard = level.guard();
let mut expr = Expr::single(i, level)?;
impl<'a: 'l, 'l> Suffix<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Box<Expr<'a>>>> {
let mut level_guard = i.state.level.guard();
let mut expr = Expr::single(i)?;
let mut right = opt(alt((
|i: &mut _| Self::associated_item(i, level),
|i: &mut _| Self::index(i, level),
|i: &mut _| Self::call(i, level),
Self::associated_item,
Self::index,
Self::call,
Self::r#try,
Self::r#macro,
)));
@ -852,7 +811,7 @@ impl<'a> Suffix<'a> {
Ok(expr)
}
fn r#macro(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Self>> {
fn r#macro(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Token {
SomeOther,
@ -877,14 +836,14 @@ impl<'a> Suffix<'a> {
}
}
fn macro_arguments<'a>(
i: &mut InputStream<'a>,
fn macro_arguments<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
open_token: Group,
) -> ParseResult<'a, Suffix<'a>> {
fn inner<'a>(
i: &mut InputStream<'a>,
fn inner<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
open_token: Group,
) -> ParseResult<'a, <InputStream<'a> as Stream>::Checkpoint> {
) -> ParseResult<'a, <InputStream<'a, 'l> as Stream>::Checkpoint> {
let mut open_list = vec![open_token];
loop {
let before = i.checkpoint();
@ -925,36 +884,50 @@ impl<'a> Suffix<'a> {
Ok(Suffix::MacroCall(inner))
}
fn lifetime<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
// Before the 2021 edition, we can have whitespace characters between "r#" and the
// identifier so we allow it here.
let start = *i;
'\''.parse_next(i)?;
let Some((is_raw, identifier)) = opt(alt((
('r', '#', identifier).map(|(_, _, ident)| (true, ident)),
(identifier, not(peek('#'))).map(|(ident, _)| (false, ident)),
)))
.parse_next(i)?
else {
return cut_error!("wrong lifetime format", **start);
};
if !is_raw {
if crate::is_rust_keyword(identifier) {
return cut_error!(
"a non-raw lifetime cannot be named like an existing keyword",
**start,
);
fn lifetime<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
// this code assumes that we tried to match char literals before calling this function
let p = (
'\''.void(),
identifier,
opt((repeat(1.., '#'), opt(identifier))),
opt('\'').map(|o| o.is_some()),
);
let ((_, front, back, quot), span) = p.with_span().parse_next(i)?;
match (front, back, quot) {
// this case should never be encountered
(_, _, true) => cut_error!(
"cannot have multiple characters in a character literal, \
use `\"...\"` to write a string",
span
),
// a normal lifetime
(identifier, None, _) => {
if !is_rust_keyword(identifier) {
Ok(())
} else {
cut_error!(
"a non-raw lifetime cannot be named like an existing keyword",
span,
)
}
}
} else if ["Self", "self", "crate", "super", "_"].contains(&identifier) {
return cut_error!(format!("`{identifier}` cannot be a raw lifetime"), **start,);
// a raw lifetime
("r", Some((1, Some(identifier))), _) => {
if matches!(identifier, "Self" | "self" | "crate" | "super" | "_") {
cut_error!(
format!("`{}` cannot be a raw lifetime", identifier.escape_debug()),
span,
)
} else {
Ok(())
}
}
// an illegal prefix (not `'r#..`, multiple `#` or no identifier after `#`)
(_, Some(_), _) => cut_error!("wrong lifetime format", span),
}
if opt(peek('\'')).parse_next(i)?.is_some() {
return cut_error!("unexpected `'` after lifetime", **start);
}
Ok(())
}
fn token<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Token> {
fn token<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Token> {
// <https://doc.rust-lang.org/reference/tokens.html>
let some_other = alt((
// literals
@ -974,8 +947,8 @@ impl<'a> Suffix<'a> {
alt((open.map(Token::Open), close.map(Token::Close), some_other)).parse_next(i)
}
fn line_comment<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn inner<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, bool> {
fn line_comment<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
fn inner<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, bool> {
let mut p = (
"//".span(),
alt((
@ -1000,8 +973,8 @@ impl<'a> Suffix<'a> {
doc_comment_no_bare_cr(i, inner)
}
fn block_comment<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn inner<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, bool> {
fn block_comment<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
fn inner<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, bool> {
let is_doc_comment = alt((
('*', not(peek(one_of(['*', '/'])))).value(true),
'!'.value(true),
@ -1034,7 +1007,9 @@ impl<'a> Suffix<'a> {
doc_comment_no_bare_cr(i, inner)
}
fn identifier_or_prefixed_string<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn identifier_or_prefixed_string<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, ()> {
// <https://doc.rust-lang.org/reference/tokens.html#r-lex.token.literal.str-raw.syntax>
let ((prefix, hashes, quot), prefix_span): ((_, usize, _), _) =
@ -1145,7 +1120,7 @@ impl<'a> Suffix<'a> {
}
}
fn hash<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Token> {
fn hash<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Token> {
let (quot, span) = preceded('#', opt('"')).with_span().parse_next(i)?;
if quot.is_some() {
return cut_error!(
@ -1156,7 +1131,7 @@ impl<'a> Suffix<'a> {
Ok(Token::SomeOther)
}
fn punctuation<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn punctuation<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
// <https://doc.rust-lang.org/reference/tokens.html#punctuation>
// hash '#' omitted
@ -1200,7 +1175,7 @@ impl<'a> Suffix<'a> {
alt((three_chars, two_chars, one_char)).parse_next(i)
}
fn open<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Group> {
fn open<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Group> {
alt((
'('.value(Group::Paren),
'{'.value(Group::Brace),
@ -1209,7 +1184,7 @@ impl<'a> Suffix<'a> {
.parse_next(i)
}
fn close<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Group> {
fn close<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Group> {
alt((
')'.value(Group::Paren),
'}'.value(Group::Brace),
@ -1223,10 +1198,7 @@ impl<'a> Suffix<'a> {
Ok(WithSpan::new(inner, span))
}
fn associated_item(
i: &mut InputStream<'a>,
level: Level<'_>,
) -> ParseResult<'a, WithSpan<Self>> {
fn associated_item(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (
ws(terminated('.'.span(), not('.'))),
cut_err((
@ -1240,7 +1212,7 @@ impl<'a> Suffix<'a> {
}
Ok(WithSpan::new(name, span))
},
opt(|i: &mut _| call_generics(i, level)),
opt(call_generics),
)),
);
let (span, (name, generics)) = p.parse_next(i)?;
@ -1250,10 +1222,10 @@ impl<'a> Suffix<'a> {
))
}
fn index(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Self>> {
fn index(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (
ws('['.span()),
cut_err((ws(move |i: &mut _| Expr::parse(i, level, true)), opt(']'))),
cut_err((ws(move |i: &mut _| Expr::parse(i, true)), opt(']'))),
);
let (span, (expr, closed)) = p.parse_next(i)?;
if closed.is_none() {
@ -1262,24 +1234,22 @@ impl<'a> Suffix<'a> {
Ok(WithSpan::new(Self::Index(expr), span))
}
fn call(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (opt(|i: &mut _| call_generics(i, level)), |i: &mut _| {
Expr::arguments(i, level)
});
fn call(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (opt(call_generics), Expr::arguments);
let (generics, args) = p.parse_next(i)?;
let (args, span) = args.deconstruct();
Ok(WithSpan::new(Self::Call { generics, args }, span))
}
fn r#try(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Self>> {
fn r#try(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let span = preceded(skip_ws0, '?'.span()).parse_next(i)?;
Ok(WithSpan::new(Self::Try, span))
}
}
fn doc_comment_no_bare_cr<'a>(
i: &mut InputStream<'a>,
inner: fn(i: &mut InputStream<'a>) -> ParseResult<'a, bool>,
fn doc_comment_no_bare_cr<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
inner: fn(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, bool>,
) -> ParseResult<'a, ()> {
let ((is_doc_comment, comment), span) = inner.with_taken().with_span().parse_next(i)?;
if is_doc_comment && comment.split('\r').skip(1).any(|s| !s.starts_with('\n')) {
@ -1307,8 +1277,8 @@ pub struct TyGenerics<'a> {
pub args: Option<WithSpan<Vec<WithSpan<TyGenerics<'a>>>>>,
}
impl<'i> TyGenerics<'i> {
fn parse(i: &mut InputStream<'i>, level: Level<'_>) -> ParseResult<'i, WithSpan<Self>> {
impl<'a: 'l, 'l> TyGenerics<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let path = separated(
1..,
ws(identifier
@ -1318,11 +1288,7 @@ impl<'i> TyGenerics<'i> {
)
.map(|v: Vec<_>| v);
let p = ws((
repeat(0.., ws('&')),
path,
opt(|i: &mut _| Self::args(i, level)),
));
let p = ws((repeat(0.., ws('&')), path, opt(Self::args)));
let ((refs, path, args), span) = p.with_span().parse_next(i)?;
if let [name] = path.as_slice() {
@ -1347,27 +1313,25 @@ impl<'i> TyGenerics<'i> {
}
fn args(
i: &mut InputStream<'i>,
level: Level<'_>,
) -> ParseResult<'i, WithSpan<Vec<WithSpan<TyGenerics<'i>>>>> {
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, WithSpan<Vec<WithSpan<TyGenerics<'a>>>>> {
let mut p = cut_err(terminated(
opt(terminated(
separated(1.., |i: &mut _| TyGenerics::parse(i, level), ws(',')),
separated(1.., TyGenerics::parse, ws(',')),
ws(opt(',')),
)),
'>',
));
let span = ws('<'.span()).parse_next(i)?;
let _level_guard = level.nest(i)?;
let _level_guard = i.state.level.nest(i)?;
let args = p.parse_next(i)?;
Ok(WithSpan::new(args.unwrap_or_default(), span))
}
}
pub(crate) fn call_generics<'i>(
i: &mut InputStream<'i>,
level: Level<'_>,
) -> ParseResult<'i, WithSpan<Vec<WithSpan<TyGenerics<'i>>>>> {
preceded(ws("::"), cut_err(|i: &mut _| TyGenerics::args(i, level))).parse_next(i)
pub(crate) fn call_generics<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, WithSpan<Vec<WithSpan<TyGenerics<'a>>>>> {
preceded(ws("::"), cut_err(TyGenerics::args)).parse_next(i)
}

300
askama_parser/src/lib.rs
View File

@ -103,7 +103,7 @@ mod _parsed {
pub use _parsed::Parsed;
type InputStream<'a> = Stateful<LocatingSlice<&'a str>, ()>;
type InputStream<'a, 'l> = Stateful<LocatingSlice<&'a str>, &'l State<'l>>;
#[derive(Debug, Default)]
pub struct Ast<'a> {
@ -117,18 +117,16 @@ impl<'a> Ast<'a> {
src: &'a str,
file_path: Option<Arc<Path>>,
syntax: &Syntax<'_>,
) -> Result<Self, ParseError> {
let level = Cell::new(Level::MAX_DEPTH);
) -> Result<Ast<'a>, ParseError> {
let state = State {
syntax,
loop_depth: Cell::new(0),
level: Level(&level),
syntax: *syntax,
..State::default()
};
let mut src = InputStream {
input: LocatingSlice::new(src),
state: (),
state: &state,
};
match Node::parse_template(&mut src, &state) {
match Node::parse_template(&mut src) {
Ok(nodes) if src.is_empty() => Ok(Self { nodes }),
Ok(_) | Err(ErrMode::Incomplete(_)) => unreachable!(),
Err(
@ -164,13 +162,6 @@ pub struct Span {
end: usize,
}
impl From<&InputStream<'_>> for Span {
#[inline]
fn from(i: &InputStream<'_>) -> Self {
(*i).into()
}
}
impl Default for Span {
#[inline]
fn default() -> Self {
@ -178,9 +169,16 @@ impl Default for Span {
}
}
impl From<InputStream<'_>> for Span {
impl From<&InputStream<'_, '_>> for Span {
#[inline]
fn from(mut i: InputStream<'_>) -> Self {
fn from(i: &InputStream<'_, '_>) -> Self {
(*i).into()
}
}
impl From<InputStream<'_, '_>> for Span {
#[inline]
fn from(mut i: InputStream<'_, '_>) -> Self {
let start = i.current_token_start();
i.finish();
Self {
@ -471,11 +469,11 @@ impl ErrorContext {
}
}
impl<'a> winnow::error::ParserError<InputStream<'a>> for ErrorContext {
impl<'a: 'l, 'l> winnow::error::ParserError<InputStream<'a, 'l>> for ErrorContext {
type Inner = Self;
#[inline]
fn from_input(input: &InputStream<'a>) -> Self {
fn from_input(input: &InputStream<'a, 'l>) -> Self {
Self {
span: input.into(),
message: None,
@ -488,35 +486,35 @@ impl<'a> winnow::error::ParserError<InputStream<'a>> for ErrorContext {
}
}
fn skip_ws0<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn skip_ws0<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
take_while(0.., |c: char| c.is_ascii_whitespace())
.void()
.parse_next(i)
}
fn skip_ws1<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn skip_ws1<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
take_while(1.., |c: char| c.is_ascii_whitespace())
.void()
.parse_next(i)
}
fn ws<'a, O>(
inner: impl ModalParser<InputStream<'a>, O, ErrorContext>,
) -> impl ModalParser<InputStream<'a>, O, ErrorContext> {
fn ws<'a: 'l, 'l, O>(
inner: impl ModalParser<InputStream<'a, 'l>, O, ErrorContext>,
) -> impl ModalParser<InputStream<'a, 'l>, O, ErrorContext> {
delimited(skip_ws0, inner, skip_ws0)
}
fn keyword<'a>(k: &str) -> impl ModalParser<InputStream<'a>, &'a str, ErrorContext> {
fn keyword<'a: 'l, 'l>(k: &str) -> impl ModalParser<InputStream<'a, 'l>, &'a str, ErrorContext> {
identifier.verify(move |v: &str| v == k)
}
fn identifier<'i>(input: &mut InputStream<'i>) -> ParseResult<'i> {
fn identifier<'a: 'l, 'l>(input: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
let head = any.verify(|&c| c == '_' || unicode_ident::is_xid_start(c));
let tail = take_while(.., unicode_ident::is_xid_continue);
(head, tail).take().parse_next(input)
}
fn bool_lit<'i>(i: &mut InputStream<'i>) -> ParseResult<'i> {
fn bool_lit<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
alt((keyword("false"), keyword("true"))).parse_next(i)
}
@ -526,11 +524,11 @@ pub enum Num<'a> {
Float(&'a str, Option<FloatKind>),
}
fn num_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Num<'a>> {
fn num_lit_suffix<'a, T: Copy>(
fn num_lit<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Num<'a>> {
fn num_lit_suffix<'a: 'l, 'l, T: Copy>(
kind: &'a str,
list: &[(&str, T)],
i: &mut InputStream<'a>,
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, T> {
let (suffix, span) = identifier.with_span().parse_next(i)?;
if let Some(value) = list
@ -558,7 +556,7 @@ fn num_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Num<'a>> {
// Equivalent to <https://github.com/rust-lang/rust/blob/e3f909b2bbd0b10db6f164d466db237c582d3045/compiler/rustc_lexer/src/lib.rs#L626-L653>:
// no `_` directly after the decimal point `.`, or between `e` and `+/-`.
let float = |i: &mut InputStream<'a>| -> ParseResult<'a, ()> {
let float = |i: &mut InputStream<'a, 'l>| -> ParseResult<'a, ()> {
let has_dot = opt(('.', separated_digits(10, true))).parse_next(i)?;
let has_exp = opt(|i: &mut _| {
let ((kind, op), span) = (one_of(['e', 'E']), opt(one_of(['+', '-'])))
@ -605,10 +603,10 @@ fn num_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, Num<'a>> {
/// Underscore separated digits of the given base, unless `start` is true this may start
/// with an underscore.
fn separated_digits<'a>(
fn separated_digits<'a: 'l, 'l>(
radix: u32,
start: bool,
) -> impl ModalParser<InputStream<'a>, &'a str, ErrorContext> {
) -> impl ModalParser<InputStream<'a, 'l>, &'a str, ErrorContext> {
(
cond(!start, repeat(0.., '_').map(|()| ())),
one_of(move |ch: char| ch.is_digit(radix)),
@ -661,10 +659,10 @@ pub struct StrLit<'a> {
pub contains_high_ascii: bool,
}
fn str_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, StrLit<'a>> {
fn str_lit<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, StrLit<'a>> {
// <https://doc.rust-lang.org/reference/tokens.html#r-lex.token.literal.str.syntax>
fn inner<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, StrLit<'a>> {
fn inner<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, StrLit<'a>> {
#[derive(Debug, Clone, PartialEq, Eq)]
enum Sequence<'a> {
Text(&'a str),
@ -800,7 +798,7 @@ fn str_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, StrLit<'a>> {
Ok(lit)
}
fn not_suffix_with_hash<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
fn not_suffix_with_hash<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
if let Some(span) = opt(identifier.span()).parse_next(i)? {
return cut_error!(
"you are missing a space to separate two string literals",
@ -810,7 +808,7 @@ fn not_suffix_with_hash<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, ()> {
Ok(())
}
fn str_lit_without_prefix<'a>(i: &mut InputStream<'a>) -> ParseResult<'a> {
fn str_lit_without_prefix<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
let (lit, span) = str_lit.with_span().parse_next(i)?;
let kind = match lit.prefix {
@ -841,7 +839,7 @@ pub struct CharLit<'a> {
// Information about allowed character escapes is available at:
// <https://doc.rust-lang.org/reference/tokens.html#character-literals>.
fn char_lit<'a>(i: &mut InputStream<'a>) -> ParseResult<'a, CharLit<'a>> {
fn char_lit<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, CharLit<'a>> {
let ((prefix, _, content, is_closed), span) = (
alt(('b'.value(Some(CharPrefix::Binary)), empty.value(None))),
'\'',
@ -985,18 +983,13 @@ pub enum PathOrIdentifier<'a> {
Identifier(WithSpan<&'a str>),
}
fn path_or_identifier<'a>(
i: &mut InputStream<'a>,
level: Level<'_>,
fn path_or_identifier<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, PathOrIdentifier<'a>> {
let mut p = |i: &mut _| {
let root = ws(opt(terminated(empty.span(), "::")));
let start = move |i: &mut _| PathComponent::parse(i, level);
let tail = opt(repeat(
1..,
preceded(ws("::"), move |i: &mut _| PathComponent::parse(i, level)),
)
.map(|v: Vec<_>| v));
let start = PathComponent::parse;
let tail = opt(repeat(1.., preceded(ws("::"), PathComponent::parse)).map(|v: Vec<_>| v));
let (root, start, rest) = (root, start, tail).parse_next(i)?;
Ok((root, start, rest.unwrap_or_default()))
@ -1037,41 +1030,50 @@ fn path_or_identifier<'a>(
}
}
struct State<'a, 'l> {
syntax: &'l Syntax<'a>,
#[derive(Debug, Clone, Default)]
struct State<'a> {
syntax: Syntax<'a>,
loop_depth: Cell<usize>,
level: Level<'l>,
level: Level,
}
impl State<'_, '_> {
fn tag_block_start<'i>(&self, i: &mut InputStream<'i>) -> ParseResult<'i, ()> {
self.syntax.block_start.value(()).parse_next(i)
}
fn block_start<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
i.state.syntax.block_start.void().parse_next(i)
}
fn tag_block_end<'i>(&self, i: &mut InputStream<'i>) -> ParseResult<'i, ()> {
let (control, span) = alt((
self.syntax.block_end.value(None),
peek(delimited('%', alt(('-', '~', '+')).map(Some), '}')),
fail, // rollback on partial matches in the previous line
))
.with_span()
.parse_next(i)?;
fn block_end<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
let (control, span) = alt((
i.state.syntax.block_end.value(None),
peek(delimited('%', alt(('-', '~', '+')).map(Some), '}')),
fail, // rollback on partial matches in the previous line
))
.with_span()
.parse_next(i)?;
let Some(control) = control else {
return Ok(());
};
let Some(control) = control else {
return Ok(());
};
let err = ErrorContext::new(
format!(
"unclosed block, you likely meant to apply whitespace control: \"{}{}\"",
control.escape_default(),
self.syntax.block_end.escape_default(),
),
span,
);
Err(err.backtrack())
}
let err = ErrorContext::new(
format!(
"unclosed block, you likely meant to apply whitespace control: \"{}{}\"",
control.escape_default(),
i.state.syntax.block_end.escape_default(),
),
span,
);
Err(err.backtrack())
}
fn expr_start<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
i.state.syntax.expr_start.void().parse_next(i)
}
fn expr_end<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
i.state.syntax.expr_end.void().parse_next(i)
}
impl State<'_> {
fn enter_loop(&self) {
self.loop_depth.set(self.loop_depth.get() + 1);
}
@ -1244,27 +1246,34 @@ impl<'a> SyntaxBuilder<'a> {
/// [`Level::nest()`] / [`LevelGuard::nest()`] will return an error. The same [`Level`] instance is
/// shared across all usages in a [`Parsed::new()`] / [`Ast::from_str()`] call, using a reference
/// to an interior mutable counter.
#[derive(Debug, Clone, Copy)]
struct Level<'l>(&'l Cell<usize>);
#[derive(Debug, Clone)]
struct Level(Cell<usize>);
impl Level<'_> {
impl Default for Level {
#[inline]
fn default() -> Self {
Self(Cell::new(Level::MAX_DEPTH))
}
}
impl Level {
const MAX_DEPTH: usize = 128;
/// Acquire a [`LevelGuard`] without decrementing the counter, to be used with loops.
fn guard(&self) -> LevelGuard<'_> {
LevelGuard {
level: *self,
level: self,
count: 0,
}
}
/// Decrement the remaining level counter, and return a [`LevelGuard`] that increments it again
/// when it's dropped.
fn nest<'a>(&self, i: &InputStream<'a>) -> ParseResult<'a, LevelGuard<'_>> {
fn nest<'a: 'l, 'l>(&self, i: &InputStream<'a, 'l>) -> ParseResult<'a, LevelGuard<'_>> {
if let Some(new_level) = self.0.get().checked_sub(1) {
self.0.set(new_level);
Ok(LevelGuard {
level: *self,
level: self,
count: 1,
})
} else {
@ -1274,7 +1283,7 @@ impl Level<'_> {
#[cold]
#[inline(never)]
fn _fail<'a, T>(i: &InputStream<'a>) -> ParseResult<'a, T> {
fn _fail<'a: 'l, 'l, T>(i: &InputStream<'a, 'l>) -> ParseResult<'a, T> {
let msg = "your template code is too deeply nested, or the last expression is too complex";
Err(ErrorContext::new(msg, i).cut())
}
@ -1284,7 +1293,7 @@ impl Level<'_> {
/// remaining level counter when it is dropped / falls out of scope.
#[must_use]
struct LevelGuard<'l> {
level: Level<'l>,
level: &'l Level,
count: usize,
}
@ -1296,7 +1305,7 @@ impl Drop for LevelGuard<'_> {
impl LevelGuard<'_> {
/// Used to decrement the level multiple times, e.g. for every iteration of a loop.
fn nest<'a>(&mut self, i: &InputStream<'a>) -> ParseResult<'a, ()> {
fn nest<'a: 'l, 'l>(&mut self, i: &InputStream<'a, 'l>) -> ParseResult<'a, ()> {
if let Some(new_level) = self.level.0.get().checked_sub(1) {
self.level.0.set(new_level);
self.count += 1;
@ -1307,12 +1316,8 @@ impl LevelGuard<'_> {
}
}
fn filter<'a>(i: &mut InputStream<'a>, level: Level<'_>) -> ParseResult<'a, Filter<'a>> {
preceded(
('|', not('|')),
cut_err(|i: &mut _| Filter::parse(i, level)),
)
.parse_next(i)
fn filter<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Filter<'a>> {
preceded(('|', not('|')), cut_err(Filter::parse)).parse_next(i)
}
/// Returns the common parts of two paths.
@ -1602,78 +1607,82 @@ mod test {
assert_eq!(strip_common(&cwd, Path::new("/a/b/c")), "/a/b/c");
}
fn parse_peek<'a, T>(
mut parser: impl ModalParser<InputStream<'a>, T, ErrorContext>,
#[track_caller]
fn parse_peek<'a: 'l, 'l, T>(
state: &'l State<'l>,
parser: impl FnOnce(&mut InputStream<'a, 'l>) -> ParseResult<'a, T>,
input: &'a str,
) -> ParseResult<'a, (&'a str, T)> {
let mut i = InputStream {
input: LocatingSlice::new(input),
state: (),
state,
};
let value = parser.parse_next(&mut i)?;
let value = parser(&mut i)?;
Ok((**i, value))
}
#[test]
fn test_num_lit() {
let s = State::default();
// Should fail.
assert!(parse_peek(num_lit, ".").is_err());
assert!(parse_peek(&s, num_lit, ".").is_err());
// Should succeed.
assert_eq!(
parse_peek(num_lit, "1.2E-02").unwrap(),
parse_peek(&s, num_lit, "1.2E-02").unwrap(),
("", Num::Float("1.2E-02", None))
);
assert_eq!(
parse_peek(num_lit, "4e3").unwrap(),
parse_peek(&s, num_lit, "4e3").unwrap(),
("", Num::Float("4e3", None)),
);
assert_eq!(
parse_peek(num_lit, "4e+_3").unwrap(),
parse_peek(&s, num_lit, "4e+_3").unwrap(),
("", Num::Float("4e+_3", None)),
);
// Not supported because Rust wants a number before the `.`.
assert!(parse_peek(num_lit, ".1").is_err());
assert!(parse_peek(num_lit, ".1E-02").is_err());
assert!(parse_peek(&s, num_lit, ".1").is_err());
assert!(parse_peek(&s, num_lit, ".1E-02").is_err());
// A `_` directly after the `.` denotes a field.
assert_eq!(
parse_peek(num_lit, "1._0").unwrap(),
parse_peek(&s, num_lit, "1._0").unwrap(),
("._0", Num::Int("1", None))
);
assert_eq!(
parse_peek(num_lit, "1_.0").unwrap(),
parse_peek(&s, num_lit, "1_.0").unwrap(),
("", Num::Float("1_.0", None))
);
// Not supported (voluntarily because of `1..` syntax).
assert_eq!(
parse_peek(num_lit, "1.").unwrap(),
parse_peek(&s, num_lit, "1.").unwrap(),
(".", Num::Int("1", None))
);
assert_eq!(
parse_peek(num_lit, "1_.").unwrap(),
parse_peek(&s, num_lit, "1_.").unwrap(),
(".", Num::Int("1_", None))
);
assert_eq!(
parse_peek(num_lit, "1_2.").unwrap(),
parse_peek(&s, num_lit, "1_2.").unwrap(),
(".", Num::Int("1_2", None))
);
// Numbers with suffixes
assert_eq!(
parse_peek(num_lit, "-1usize").unwrap(),
parse_peek(&s, num_lit, "-1usize").unwrap(),
("", Num::Int("-1", Some(IntKind::Usize)))
);
assert_eq!(
parse_peek(num_lit, "123_f32").unwrap(),
parse_peek(&s, num_lit, "123_f32").unwrap(),
("", Num::Float("123_", Some(FloatKind::F32)))
);
assert_eq!(
parse_peek(num_lit, "1_.2_e+_3_f64|into_isize").unwrap(),
parse_peek(&s, num_lit, "1_.2_e+_3_f64|into_isize").unwrap(),
(
"|into_isize",
Num::Float("1_.2_e+_3_", Some(FloatKind::F64))
)
);
assert_eq!(
parse_peek(num_lit, "4e3f128").unwrap(),
parse_peek(&s, num_lit, "4e3f128").unwrap(),
("", Num::Float("4e3", Some(FloatKind::F128))),
);
}
@ -1684,43 +1693,59 @@ mod test {
prefix: None,
content: s,
};
let s = State::default();
assert_eq!(parse_peek(char_lit, "'a'").unwrap(), ("", lit("a")));
assert_eq!(parse_peek(char_lit, "'字'").unwrap(), ("", lit("字")));
assert_eq!(parse_peek(&s, char_lit, "'a'").unwrap(), ("", lit("a")));
assert_eq!(parse_peek(&s, char_lit, "'字'").unwrap(), ("", lit("字")));
// Escaped single characters.
assert_eq!(parse_peek(char_lit, "'\\\"'").unwrap(), ("", lit("\\\"")));
assert_eq!(parse_peek(char_lit, "'\\''").unwrap(), ("", lit("\\'")));
assert_eq!(parse_peek(char_lit, "'\\t'").unwrap(), ("", lit("\\t")));
assert_eq!(parse_peek(char_lit, "'\\n'").unwrap(), ("", lit("\\n")));
assert_eq!(parse_peek(char_lit, "'\\r'").unwrap(), ("", lit("\\r")));
assert_eq!(parse_peek(char_lit, "'\\0'").unwrap(), ("", lit("\\0")));
assert_eq!(
parse_peek(&s, char_lit, "'\\\"'").unwrap(),
("", lit("\\\""))
);
assert_eq!(parse_peek(&s, char_lit, "'\\''").unwrap(), ("", lit("\\'")));
assert_eq!(parse_peek(&s, char_lit, "'\\t'").unwrap(), ("", lit("\\t")));
assert_eq!(parse_peek(&s, char_lit, "'\\n'").unwrap(), ("", lit("\\n")));
assert_eq!(parse_peek(&s, char_lit, "'\\r'").unwrap(), ("", lit("\\r")));
assert_eq!(parse_peek(&s, char_lit, "'\\0'").unwrap(), ("", lit("\\0")));
// Escaped ascii characters (up to `0x7F`).
assert_eq!(parse_peek(char_lit, "'\\x12'").unwrap(), ("", lit("\\x12")));
assert_eq!(parse_peek(char_lit, "'\\x02'").unwrap(), ("", lit("\\x02")));
assert_eq!(parse_peek(char_lit, "'\\x6a'").unwrap(), ("", lit("\\x6a")));
assert_eq!(parse_peek(char_lit, "'\\x7F'").unwrap(), ("", lit("\\x7F")));
assert_eq!(
parse_peek(&s, char_lit, "'\\x12'").unwrap(),
("", lit("\\x12"))
);
assert_eq!(
parse_peek(&s, char_lit, "'\\x02'").unwrap(),
("", lit("\\x02"))
);
assert_eq!(
parse_peek(&s, char_lit, "'\\x6a'").unwrap(),
("", lit("\\x6a"))
);
assert_eq!(
parse_peek(&s, char_lit, "'\\x7F'").unwrap(),
("", lit("\\x7F"))
);
// Escaped unicode characters (up to `0x10FFFF`).
assert_eq!(
parse_peek(char_lit, "'\\u{A}'").unwrap(),
parse_peek(&s, char_lit, "'\\u{A}'").unwrap(),
("", lit("\\u{A}"))
);
assert_eq!(
parse_peek(char_lit, "'\\u{10}'").unwrap(),
parse_peek(&s, char_lit, "'\\u{10}'").unwrap(),
("", lit("\\u{10}"))
);
assert_eq!(
parse_peek(char_lit, "'\\u{aa}'").unwrap(),
parse_peek(&s, char_lit, "'\\u{aa}'").unwrap(),
("", lit("\\u{aa}"))
);
assert_eq!(
parse_peek(char_lit, "'\\u{10FFFF}'").unwrap(),
parse_peek(&s, char_lit, "'\\u{10FFFF}'").unwrap(),
("", lit("\\u{10FFFF}"))
);
// Check with `b` prefix.
assert_eq!(
parse_peek(char_lit, "b'a'").unwrap(),
parse_peek(&s, char_lit, "b'a'").unwrap(),
(
"",
crate::CharLit {
@ -1731,20 +1756,21 @@ mod test {
);
// Should fail.
assert!(parse_peek(char_lit, "''").is_err());
assert!(parse_peek(char_lit, "'\\o'").is_err());
assert!(parse_peek(char_lit, "'\\x'").is_err());
assert!(parse_peek(char_lit, "'\\x1'").is_err());
assert!(parse_peek(char_lit, "'\\x80'").is_err());
assert!(parse_peek(char_lit, "'\\u'").is_err());
assert!(parse_peek(char_lit, "'\\u{}'").is_err());
assert!(parse_peek(char_lit, "'\\u{110000}'").is_err());
assert!(parse_peek(&s, char_lit, "''").is_err());
assert!(parse_peek(&s, char_lit, "'\\o'").is_err());
assert!(parse_peek(&s, char_lit, "'\\x'").is_err());
assert!(parse_peek(&s, char_lit, "'\\x1'").is_err());
assert!(parse_peek(&s, char_lit, "'\\x80'").is_err());
assert!(parse_peek(&s, char_lit, "'\\u'").is_err());
assert!(parse_peek(&s, char_lit, "'\\u{}'").is_err());
assert!(parse_peek(&s, char_lit, "'\\u{110000}'").is_err());
}
#[test]
fn test_str_lit() {
let s = State::default();
assert_eq!(
parse_peek(str_lit, r#"b"hello""#).unwrap(),
parse_peek(&s, str_lit, r#"b"hello""#).unwrap(),
(
"",
StrLit {
@ -1758,7 +1784,7 @@ mod test {
)
);
assert_eq!(
parse_peek(str_lit, r#"c"hello""#).unwrap(),
parse_peek(&s, str_lit, r#"c"hello""#).unwrap(),
(
"",
StrLit {
@ -1771,7 +1797,7 @@ mod test {
}
)
);
assert!(parse_peek(str_lit, r#"d"hello""#).is_err());
assert!(parse_peek(&s, str_lit, r#"d"hello""#).is_err());
}
#[test]

417
askama_parser/src/node.rs
View File

@ -10,9 +10,9 @@ use winnow::token::{any, literal, rest, take, take_until};
use winnow::{ModalParser, Parser};
use crate::{
ErrorContext, Expr, Filter, HashSet, InputStream, ParseErr, ParseResult, Span, State, Target,
WithSpan, cut_error, filter, identifier, is_rust_keyword, keyword, skip_ws0,
str_lit_without_prefix, ws,
ErrorContext, Expr, Filter, HashSet, InputStream, ParseErr, ParseResult, Span, Target,
WithSpan, block_end, block_start, cut_error, expr_end, expr_start, filter, identifier,
is_rust_keyword, keyword, skip_ws0, str_lit_without_prefix, ws,
};
#[derive(Debug, PartialEq)]
@ -36,18 +36,12 @@ pub enum Node<'a> {
FilterBlock(WithSpan<FilterBlock<'a>>),
}
impl<'a> Node<'a> {
pub(super) fn parse_template(
i: &mut InputStream<'a>,
s: &State<'_, '_>,
) -> ParseResult<'a, Vec<Box<Self>>> {
let mut p = parse_with_unexpected_fallback(
|i: &mut _| Self::many(i, s),
|i: &mut _| unexpected_tag(i, s),
);
impl<'a: 'l, 'l> Node<'a> {
pub(super) fn parse_template(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Vec<Box<Self>>> {
let mut p = parse_with_unexpected_fallback(Self::many, unexpected_tag);
let nodes = p.parse_next(i)?;
if !i.is_empty() {
opt(|i: &mut _| unexpected_tag(i, s)).parse_next(i)?;
opt(unexpected_tag).parse_next(i)?;
return cut_error!(
"cannot parse entire template\n\
you should never encounter this error\n\
@ -58,24 +52,19 @@ impl<'a> Node<'a> {
Ok(nodes)
}
fn many(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Vec<Box<Self>>> {
fn many(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Vec<Box<Self>>> {
repeat(
0..,
alt((
|i: &mut _| Lit::parse(i, s),
|i: &mut _| Comment::parse(i, s),
|i: &mut _| Self::expr(i, s),
|i: &mut _| Self::parse(i, s),
)),
alt((Lit::parse, Comment::parse, Self::expr, Self::parse)),
)
.map(|v: Vec<_>| v)
.parse_next(i)
}
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Self>> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Self>> {
let start = i.checkpoint();
let (span, tag) = (
s.syntax.block_start.span(),
block_start.span(),
peek(preceded((opt(Whitespace::parse), skip_ws0), identifier)),
)
.parse_next(i)?;
@ -86,9 +75,9 @@ impl<'a> Node<'a> {
"if" => If::parse,
"for" => Loop::parse,
"match" => Match::parse,
"extends" => |i: &mut InputStream<'a>, _s: &State<'_, '_>| Extends::parse(i),
"include" => |i: &mut InputStream<'a>, _s: &State<'_, '_>| Include::parse(i),
"import" => |i: &mut InputStream<'a>, _s: &State<'_, '_>| Import::parse(i),
"extends" => Extends::parse,
"include" => Include::parse,
"import" => Import::parse,
"block" => BlockDef::parse,
"macro" => Macro::parse,
"raw" => Raw::parse,
@ -101,23 +90,22 @@ impl<'a> Node<'a> {
}
};
let _level_guard = s.level.nest(i)?;
let node = func(i, s)?;
let closed = cut_node(
None,
alt((
ws(eof).value(false),
(|i: &mut _| s.tag_block_end(i)).value(true),
)),
)
.parse_next(i)?;
let _level_guard = i.state.level.nest(i)?;
let node = func(i)?;
let closed =
cut_node(None, alt((ws(eof).value(false), block_end.value(true)))).parse_next(i)?;
match closed {
true => Ok(node),
false => Err(ErrorContext::unclosed("block", s.syntax.block_end, span).cut()),
false => {
Err(
ErrorContext::unclosed("block", i.state.syntax.block_end, Span::new(span))
.cut(),
)
}
}
}
fn r#break(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn r#break(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut p = (
opt(Whitespace::parse),
ws(keyword("break").span()),
@ -125,13 +113,13 @@ impl<'a> Node<'a> {
);
let (pws, span, nws) = p.parse_next(i)?;
if !s.is_in_loop() {
if !i.state.is_in_loop() {
return cut_error!("you can only `break` inside a `for` loop", span);
}
Ok(Box::new(Self::Break(WithSpan::new(Ws(pws, nws), span))))
}
fn r#continue(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn r#continue(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut p = (
opt(Whitespace::parse),
ws(keyword("continue").span()),
@ -139,27 +127,27 @@ impl<'a> Node<'a> {
);
let (pws, span, nws) = p.parse_next(i)?;
if !s.is_in_loop() {
if !i.state.is_in_loop() {
return cut_error!("you can only `continue` inside a `for` loop", span);
}
Ok(Box::new(Self::Continue(WithSpan::new(Ws(pws, nws), span))))
}
fn expr(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Self>> {
fn expr(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Self>> {
let mut p = (
s.syntax.expr_start.span(),
expr_start.span(),
cut_node(
None,
(
opt(Whitespace::parse),
ws(|i: &mut _| Expr::parse(i, s.level, false)),
ws(|i: &mut _| Expr::parse(i, false)),
),
),
cut_node(
None,
(
opt(Whitespace::parse),
alt((s.syntax.expr_end.value(true), ws(eof).value(false))),
alt((expr_end.value(true), ws(eof).value(false))),
),
),
);
@ -168,7 +156,7 @@ impl<'a> Node<'a> {
if closed {
Ok(Box::new(Self::Expr(Ws(pws, nws), expr)))
} else {
Err(ErrorContext::unclosed("expression", s.syntax.expr_end, start).cut())
Err(ErrorContext::unclosed("expression", i.state.syntax.expr_end, start).cut())
}
}
@ -197,15 +185,15 @@ impl<'a> Node<'a> {
}
#[inline]
fn parse_with_unexpected_fallback<'a, O>(
mut parser: impl ModalParser<InputStream<'a>, O, ErrorContext>,
mut unexpected_parser: impl FnMut(&mut InputStream<'a>) -> ParseResult<'a, ()>,
) -> impl ModalParser<InputStream<'a>, O, ErrorContext> {
fn parse_with_unexpected_fallback<'a: 'l, 'l, O>(
mut parser: impl ModalParser<InputStream<'a, 'l>, O, ErrorContext>,
mut unexpected_parser: impl FnMut(&mut InputStream<'a, 'l>) -> ParseResult<'a, ()>,
) -> impl ModalParser<InputStream<'a, 'l>, O, ErrorContext> {
#[cold]
#[inline(never)]
fn try_assign_fallback_error<'a>(
i: &mut InputStream<'a>,
unexpected_parser: &mut dyn FnMut(&mut InputStream<'a>) -> ParseResult<'a, ()>,
fn try_assign_fallback_error<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
unexpected_parser: &mut dyn FnMut(&mut InputStream<'a, 'l>) -> ParseResult<'a, ()>,
err: &mut ErrMode<ErrorContext>,
) {
let (ErrMode::Backtrack(err_ctx) | ErrMode::Cut(err_ctx)) = &err else {
@ -227,7 +215,7 @@ fn parse_with_unexpected_fallback<'a, O>(
i.reset(&checkpoint);
}
move |i: &mut InputStream<'a>| {
move |i: &mut InputStream<'a, 'l>| {
let mut result = parser.parse_next(i);
if let Err(err) = &mut result {
try_assign_fallback_error(i, &mut unexpected_parser, err);
@ -237,26 +225,24 @@ fn parse_with_unexpected_fallback<'a, O>(
}
#[inline]
fn cut_node<'a, O>(
fn cut_node<'a: 'l, 'l, O>(
kind: Option<&'static str>,
inner: impl ModalParser<InputStream<'a>, O, ErrorContext>,
) -> impl ModalParser<InputStream<'a>, O, ErrorContext> {
inner: impl ModalParser<InputStream<'a, 'l>, O, ErrorContext>,
) -> impl ModalParser<InputStream<'a, 'l>, O, ErrorContext> {
parse_with_unexpected_fallback(cut_err(inner), move |i: &mut _| unexpected_raw_tag(kind, i))
}
fn unexpected_tag<'a>(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, ()> {
(
|i: &mut _| s.tag_block_start(i),
opt(Whitespace::parse),
|i: &mut _| unexpected_raw_tag(None, i),
)
fn unexpected_tag<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
(block_start, opt(Whitespace::parse), |i: &mut _| {
unexpected_raw_tag(None, i)
})
.void()
.parse_next(i)
}
fn unexpected_raw_tag<'a>(
fn unexpected_raw_tag<'a: 'l, 'l>(
kind: Option<&'static str>,
i: &mut InputStream<'a>,
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, ()> {
let (tag, span) = peek(ws(identifier.with_span())).parse_next(i)?;
cut_error!(
@ -281,18 +267,18 @@ pub struct When<'a> {
pub nodes: Vec<Box<Node<'a>>>,
}
impl<'a> When<'a> {
fn r#else(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, WithSpan<Self>> {
impl<'a: 'l, 'l> When<'a> {
fn r#else(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (
|i: &mut _| s.tag_block_start(i),
block_start,
opt(Whitespace::parse),
ws(keyword("else").span()),
cut_node(
Some("match-else"),
(
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
cut_node(Some("match-else"), |i: &mut _| Node::many(i, s)),
block_end,
cut_node(Some("match-else"), Node::many),
),
),
);
@ -308,19 +294,19 @@ impl<'a> When<'a> {
}
#[allow(clippy::self_named_constructors)]
fn when(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, WithSpan<Self>> {
fn when(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (
|i: &mut _| s.tag_block_start(i),
block_start,
opt(Whitespace::parse),
ws(keyword("when").span()),
cut_node(
Some("match-when"),
(
separated(1.., ws(|i: &mut _| Target::parse(i, s)), '|'),
separated(1.., ws(Target::parse), '|'),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
cut_node(Some("match-when"), |i: &mut _| Node::many(i, s)),
opt(|i: &mut _| Self::endwhen(i, s)),
block_end,
cut_node(Some("match-when"), Node::many),
opt(Self::endwhen),
),
),
);
@ -338,10 +324,10 @@ impl<'a> When<'a> {
))
}
fn endwhen(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn endwhen(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut p = ws(terminated(
(
|i: &mut _| s.tag_block_start(i),
block_start,
opt(Whitespace::parse),
ws(keyword("endwhen").span()),
),
@ -349,8 +335,8 @@ impl<'a> When<'a> {
Some("match-endwhen"),
(
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
repeat(0.., ws(|i: &mut _| Comment::parse(i, s)).void()).map(|()| ()),
block_end,
repeat(0.., ws(Comment::parse).void()).map(|()| ()),
),
),
));
@ -372,28 +358,25 @@ pub struct Cond<'a> {
pub nodes: Vec<Box<Node<'a>>>,
}
impl<'a> Cond<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, WithSpan<Self>> {
let alt_else = (
ws(keyword("else").span()),
opt(|i: &mut _| CondTest::parse(i, s)),
);
impl<'a: 'l, 'l> Cond<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let alt_else = (ws(keyword("else").span()), opt(CondTest::parse));
let alt_elif = |i: &mut _| {
let mut p = (
ws(keyword("elif").span()),
cut_node(Some("if-elif"), |i: &mut _| CondTest::parse_cond(i, s)),
cut_node(Some("if-elif"), CondTest::parse_cond),
);
let (span, cond) = p.parse_next(i)?;
Ok((span.clone(), Some(WithSpan::new(cond, span))))
};
let (_, pws, (span, cond), nws, _, nodes) = (
|i: &mut _| s.tag_block_start(i),
block_start,
opt(Whitespace::parse),
alt((alt_else, alt_elif)),
opt(Whitespace::parse),
cut_node(Some("if"), |i: &mut _| s.tag_block_end(i)),
cut_node(Some("if"), |i: &mut _| Node::many(i, s)),
cut_node(Some("if"), block_end),
cut_node(Some("if"), Node::many),
)
.parse_next(i)?;
@ -415,39 +398,37 @@ pub struct CondTest<'a> {
pub contains_bool_lit_or_is_defined: bool,
}
impl<'a> CondTest<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, WithSpan<Self>> {
impl<'a: 'l, 'l> CondTest<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, WithSpan<Self>> {
let mut p = (
ws(keyword("if").span()),
cut_node(Some("if"), |i: &mut _| Self::parse_cond(i, s)),
cut_node(Some("if"), Self::parse_cond),
);
let (span, cond) = p.parse_next(i)?;
Ok(WithSpan::new(cond, span))
}
fn parse_cond(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Self> {
fn parse_cond(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
let (target, expr) = (
opt(delimited(
ws(alt((keyword("let"), keyword("set")))),
ws(|i: &mut _| Target::parse(i, s)),
ws(Target::parse),
ws('='),
)),
ws(|i: &mut InputStream<'a>| {
ws(|i: &mut InputStream<'a, 'l>| {
let start_checkpoint = i.checkpoint();
let start_offset = i.current_token_start();
let mut expr = Expr::parse(i, s.level, false)?;
let mut expr = Expr::parse(i, false)?;
if let Expr::BinOp(v) = &mut *expr.inner
&& matches!(*v.rhs.inner, Expr::Var("set" | "let"))
{
let _level_guard = s.level.nest(i)?;
let _level_guard = i.state.level.nest(i)?;
i.reset(&start_checkpoint);
i.next_slice(v.rhs.span.start - start_offset);
let (new_right, span) = (|i: &mut _| Self::parse_cond(i, s))
.with_span()
.parse_next(i)?;
let (new_right, span) = Self::parse_cond.with_span().parse_next(i)?;
v.rhs.inner = Box::new(Expr::LetCond(WithSpan::new(new_right, span)));
}
Ok(expr)
@ -474,7 +455,7 @@ pub enum Whitespace {
}
impl Whitespace {
fn parse<'i>(i: &mut InputStream<'i>) -> ParseResult<'i, Self> {
fn parse<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
any.verify_map(Self::parse_char).parse_next(i)
}
@ -509,10 +490,9 @@ impl FromStr for Whitespace {
}
}
fn check_block_start<'a>(
i: &mut InputStream<'a>,
fn check_block_start<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
start: Span,
s: &State<'_, '_>,
node: &str,
expected: &str,
) -> ParseResult<'a, ()> {
@ -522,7 +502,7 @@ fn check_block_start<'a>(
start,
);
}
(|i: &mut _| s.tag_block_start(i)).parse_next(i)
i.state.syntax.block_start.void().parse_next(i)
}
#[derive(Debug, PartialEq)]
@ -537,15 +517,12 @@ pub struct Loop<'a> {
pub ws3: Ws,
}
impl<'a> Loop<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn content<'a>(
i: &mut InputStream<'a>,
s: &State<'_, '_>,
) -> ParseResult<'a, Vec<Box<Node<'a>>>> {
s.enter_loop();
let result = (|i: &mut _| Node::many(i, s)).parse_next(i);
s.leave_loop();
impl<'a: 'l, 'l> Loop<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
fn content<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Vec<Box<Node<'a>>>> {
i.state.enter_loop();
let result = Node::many(i);
i.state.leave_loop();
result
}
@ -554,24 +531,17 @@ impl<'a> Loop<'a> {
let if_cond = preceded(
ws(keyword("if")),
cut_node(
Some("for-if"),
ws(|i: &mut _| Expr::parse(i, s.level, true)),
),
cut_node(Some("for-if"), ws(|i: &mut _| Expr::parse(i, true))),
);
let else_block = |i: &mut _| {
let else_block = |i: &mut InputStream<'a, 'l>| {
let mut p = preceded(
ws(keyword("else")),
cut_node(
Some("for-else"),
(
opt(Whitespace::parse),
delimited(
|i: &mut _| s.tag_block_end(i),
|i: &mut _| Node::many(i, s),
|i: &mut _| s.tag_block_start(i),
),
delimited(block_end, Node::many, block_start),
opt(Whitespace::parse),
),
),
@ -584,11 +554,11 @@ impl<'a> Loop<'a> {
let (body, (_, pws, else_block, _, nws)) = cut_node(
Some("for"),
(
|i: &mut _| content(i, s),
content,
cut_node(
Some("for"),
(
|i: &mut _| check_block_start(i, span, s, "for", "endfor"),
|i: &mut _| check_block_start(i, span, "for", "endfor"),
opt(Whitespace::parse),
opt(else_block),
end_node("for", "endfor"),
@ -604,15 +574,15 @@ impl<'a> Loop<'a> {
let mut p = cut_node(
Some("for"),
(
ws(|i: &mut _| Target::parse(i, s)),
ws(Target::parse),
ws(keyword("in")),
cut_node(
Some("for"),
(
ws(|i: &mut _| Expr::parse(i, s.level, true)),
ws(|i: &mut _| Expr::parse(i, true)),
opt(if_cond),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
block_end,
body_and_end,
),
),
@ -664,8 +634,8 @@ fn check_duplicated_name<'a>(
Ok(())
}
impl<'a> Macro<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Macro<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws1, keyword_span, (name, name_span)) = (
opt(Whitespace::parse),
ws(keyword("macro").span()),
@ -683,10 +653,7 @@ impl<'a> Macro<'a> {
let macro_arg = |i: &mut _| {
let mut p = (
ws(identifier.with_span()),
opt(preceded(
'=',
ws(|i: &mut _| Expr::parse(i, s.level, false)),
)),
opt(preceded('=', ws(|i: &mut _| Expr::parse(i, false)))),
);
let ((name, name_span), default) = p.parse_next(i)?;
Ok(MacroArg {
@ -709,7 +676,7 @@ impl<'a> Macro<'a> {
let (params, nws1, _) = cut_node(
Some("macro"),
(parameters, opt(Whitespace::parse), s.syntax.block_end),
(parameters, opt(Whitespace::parse), block_end),
)
.parse_next(i)?;
@ -745,11 +712,11 @@ impl<'a> Macro<'a> {
let mut end = cut_node(
Some("macro"),
(
|i: &mut _| Node::many(i, s),
Node::many,
cut_node(
Some("macro"),
(
|i: &mut _| check_block_start(i, keyword_span, s, "macro", "endmacro"),
|i: &mut _| check_block_start(i, keyword_span, "macro", "endmacro"),
opt(Whitespace::parse),
end_node("macro", "endmacro"),
check_end_name(name, "macro"),
@ -780,21 +747,19 @@ pub struct FilterBlock<'a> {
pub ws2: Ws,
}
impl<'a> FilterBlock<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> FilterBlock<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws1, span) = (opt(Whitespace::parse), ws(keyword("filter").span())).parse_next(i)?;
let span = Span::new(span);
fn filters<'a>(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Filter<'a>> {
let mut filter = opt(ws(|i: &mut InputStream<'a>| filter(i, s.level)).with_span());
fn filters<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Filter<'a>> {
let mut filter = opt(ws(filter.with_span()));
let (mut res, span) = (|i: &mut _| Filter::parse(i, s.level))
.with_span()
.parse_next(i)?;
let (mut res, span) = Filter::parse.with_span().parse_next(i)?;
res.arguments
.insert(0, WithSpan::new(Box::new(Expr::FilterSource), span));
let mut level_guard = s.level.guard();
let mut level_guard = i.state.level.guard();
let mut i_before = *i;
while let Some((mut filter, span)) = filter.parse_next(i)? {
level_guard.nest(&i_before)?;
@ -810,17 +775,13 @@ impl<'a> FilterBlock<'a> {
let mut p = (
cut_node(
Some("filter"),
(
ws(|i: &mut _| filters(i, s)),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
),
(ws(filters), opt(Whitespace::parse), block_end),
),
cut_node(Some("filter"), |i: &mut _| Node::many(i, s)),
cut_node(Some("filter"), Node::many),
cut_node(
Some("filter"),
(
|i: &mut _| check_block_start(i, span, s, "filter", "endfilter"),
|i: &mut _| check_block_start(i, span, "filter", "endfilter"),
opt(Whitespace::parse),
end_node("filter", "endfilter"),
opt(Whitespace::parse),
@ -848,8 +809,8 @@ pub struct Import<'a> {
pub scope: &'a str,
}
impl<'a> Import<'a> {
fn parse(i: &mut InputStream<'a>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Import<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut p = (
opt(Whitespace::parse),
ws(keyword("import").span()),
@ -885,8 +846,8 @@ pub struct Call<'a> {
pub ws2: Ws,
}
impl<'a> Call<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Call<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws, span) = (opt(Whitespace::parse), ws(keyword("call").span())).parse_next(i)?;
let keyword_span = Span::new(span);
@ -915,9 +876,9 @@ impl<'a> Call<'a> {
Ok(WithSpan::new(scope, span))
}),
ws(identifier.with_span()),
opt(ws(|nested: &mut _| Expr::arguments(nested, s.level))),
opt(ws(Expr::arguments)),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
block_end,
),
),
);
@ -926,11 +887,11 @@ impl<'a> Call<'a> {
let mut end = cut_node(
Some("call"),
(
|i: &mut _| Node::many(i, s),
Node::many,
cut_node(
Some("call"),
(
|i: &mut _| check_block_start(i, keyword_span, s, "call", "endcall"),
|i: &mut _| check_block_start(i, keyword_span, "call", "endcall"),
opt(Whitespace::parse),
end_node("call", "endcall"),
opt(Whitespace::parse),
@ -963,28 +924,28 @@ pub struct Match<'a> {
pub ws2: Ws,
}
impl<'a> Match<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Match<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws1, span) = (opt(Whitespace::parse), ws(keyword("match").span())).parse_next(i)?;
let span = Span::new(span);
let mut p = cut_node(
Some("match"),
(
ws(|i: &mut _| Expr::parse(i, s.level, false)),
ws(|i: &mut _| Expr::parse(i, false)),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
block_end,
cut_node(
Some("match"),
(
ws(repeat(0.., ws(|i: &mut _| Comment::parse(i, s)))).map(|()| ()),
repeat(0.., |i: &mut _| When::when(i, s)).map(|v: Vec<_>| v),
ws(repeat(0.., ws(Comment::parse))).map(|()| ()),
repeat(0.., When::when).map(|v: Vec<_>| v),
),
),
cut_node(Some("match"), opt(|i: &mut _| When::r#else(i, s))),
cut_node(Some("match"), opt(When::r#else)),
cut_node(
Some("match"),
(
ws(|i: &mut _| check_block_start(i, span, s, "match", "endmatch")),
ws(|i: &mut _| check_block_start(i, span, "match", "endmatch")),
opt(Whitespace::parse),
end_node("match", "endmatch"),
opt(Whitespace::parse),
@ -1024,8 +985,8 @@ pub struct BlockDef<'a> {
pub ws2: Ws,
}
impl<'a> BlockDef<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> BlockDef<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut start = (
opt(Whitespace::parse),
ws(keyword("block").span()),
@ -1034,7 +995,7 @@ impl<'a> BlockDef<'a> {
(
ws(identifier.with_span()),
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
block_end,
),
),
);
@ -1044,11 +1005,11 @@ impl<'a> BlockDef<'a> {
let mut end = cut_node(
Some("block"),
(
|i: &mut _| Node::many(i, s),
Node::many,
cut_node(
Some("block"),
(
|i: &mut _| check_block_start(i, keyword_span, s, "block", "endblock"),
|i: &mut _| check_block_start(i, keyword_span, "block", "endblock"),
opt(Whitespace::parse),
end_node("block", "endblock"),
check_end_name(name, "block"),
@ -1070,11 +1031,11 @@ impl<'a> BlockDef<'a> {
}
}
fn check_end_name<'a>(
fn check_end_name<'a: 'l, 'l>(
name: &'a str,
kind: &'static str,
) -> impl ModalParser<InputStream<'a>, Option<Whitespace>, ErrorContext> {
let name = move |i: &mut InputStream<'a>| {
) -> impl ModalParser<InputStream<'a, 'l>, Option<Whitespace>, ErrorContext> {
let name = move |i: &mut InputStream<'a, 'l>| {
let Some((end_name, span)) = ws(opt(identifier.with_span())).parse_next(i)? else {
return Ok(());
};
@ -1101,14 +1062,14 @@ pub struct Lit<'a> {
pub rws: WithSpan<&'a str>,
}
impl<'a> Lit<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Lit<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let content = take_until(
..,
(
s.syntax.block_start,
s.syntax.comment_start,
s.syntax.expr_start,
i.state.syntax.block_start,
i.state.syntax.comment_start,
i.state.syntax.expr_start,
),
);
let (content, span) = preceded(not(eof), alt((content, rest)))
@ -1153,11 +1114,10 @@ pub struct Raw<'a> {
pub ws2: Ws,
}
impl<'a> Raw<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn endraw<'a>(
i: &mut InputStream<'a>,
s: &State<'_, '_>,
impl<'a: 'l, 'l> Raw<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
fn endraw<'a: 'l, 'l>(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, (Ws, WithSpan<Lit<'a>>)> {
let start_i = ***i;
let start_idx = i.current_token_start();
@ -1178,7 +1138,7 @@ impl<'a> Raw<'a> {
(inner, None)
};
let Some(inner) = inner.strip_suffix(s.syntax.block_start) else {
let Some(inner) = inner.strip_suffix(i.state.syntax.block_start) else {
continue;
};
let span = start_idx..start_idx + inner.len();
@ -1187,7 +1147,7 @@ impl<'a> Raw<'a> {
skip_ws0(i)?;
let i_before_nws = i.checkpoint();
let nws = opt(Whitespace::parse).parse_next(i)?;
if opt(peek(s.syntax.block_end)).parse_next(i)?.is_none() {
if opt(peek(block_end)).parse_next(i)?.is_none() {
i.reset(&i_before_nws); // `block_start` might start with the `nws` character
continue;
}
@ -1204,11 +1164,7 @@ impl<'a> Raw<'a> {
ws(keyword("raw").span()),
cut_node(
Some("raw"),
separated_pair(
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
|i: &mut _| endraw(i, s),
),
separated_pair(opt(Whitespace::parse), block_end, endraw),
),
);
let (pws, span, (nws, (ws2, lit))) = p.parse_next(i)?;
@ -1228,8 +1184,8 @@ pub struct Let<'a> {
pub is_mutable: bool,
}
impl<'a> Let<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Let<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let mut p = (
opt(Whitespace::parse),
ws(alt((keyword("let"), keyword("set"))).span()),
@ -1237,11 +1193,8 @@ impl<'a> Let<'a> {
cut_node(
Some("let"),
(
ws((|i: &mut _| Target::parse(i, s)).with_span()),
opt(preceded(
ws('='),
ws(|i: &mut _| Expr::parse(i, s.level, false)),
)),
ws(Target::parse.with_span()),
opt(preceded(ws('='), ws(|i: &mut _| Expr::parse(i, false)))),
opt(Whitespace::parse),
),
),
@ -1300,15 +1253,14 @@ pub struct If<'a> {
pub branches: Vec<WithSpan<Cond<'a>>>,
}
impl<'a> If<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws1, cond) =
(opt(Whitespace::parse), (|i: &mut _| CondTest::parse(i, s))).parse_next(i)?;
impl<'a: 'l, 'l> If<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let (pws1, cond) = (opt(Whitespace::parse), CondTest::parse).parse_next(i)?;
let cond_span = cond.span;
let end_if = cut_node(
Some("if"),
(
|i: &mut _| check_block_start(i, cond_span, s, "if", "endif"),
|i: &mut _| check_block_start(i, cond_span, "if", "endif"),
opt(Whitespace::parse),
end_node("if", "endif"),
opt(Whitespace::parse),
@ -1318,12 +1270,12 @@ impl<'a> If<'a> {
Some("if"),
(
opt(Whitespace::parse),
|i: &mut _| s.tag_block_end(i),
block_end,
cut_node(
Some("if"),
(
|i: &mut _| Node::many(i, s),
repeat(0.., |i: &mut _| Cond::parse(i, s)).map(|v: Vec<_>| v),
Node::many,
repeat(0.., Cond::parse).map(|v: Vec<_>| v),
end_if,
),
),
@ -1357,8 +1309,8 @@ pub struct Include<'a> {
pub path: &'a str,
}
impl<'a> Include<'a> {
fn parse(i: &mut InputStream<'a>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Include<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let p = (
opt(Whitespace::parse),
ws(keyword("include")),
@ -1383,8 +1335,8 @@ pub struct Extends<'a> {
pub path: &'a str,
}
impl<'a> Extends<'a> {
fn parse(i: &mut InputStream<'a>) -> ParseResult<'a, Box<Node<'a>>> {
impl<'a: 'l, 'l> Extends<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
let p = preceded(
(opt(Whitespace::parse), ws(keyword("extends"))),
cut_node(
@ -1403,18 +1355,22 @@ pub struct Comment<'a> {
pub content: &'a str,
}
impl<'a> Comment<'a> {
fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Box<Node<'a>>> {
fn content<'a>(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, ()> {
impl<'a: 'l, 'l> Comment<'a> {
fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Box<Node<'a>>> {
fn content<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, ()> {
let mut depth = 0usize;
loop {
take_until(.., (s.syntax.comment_start, s.syntax.comment_end)).parse_next(i)?;
let is_open = opt(s.syntax.comment_start).parse_next(i)?.is_some();
take_until(
..,
(i.state.syntax.comment_start, i.state.syntax.comment_end),
)
.parse_next(i)?;
let is_open = opt(i.state.syntax.comment_start).parse_next(i)?.is_some();
if is_open {
// cannot overflow: `i` cannot be longer than `isize::MAX`, cf. [std::alloc::Layout]
depth += 1;
} else if let Some(new_depth) = depth.checked_sub(1) {
literal(s.syntax.comment_end).parse_next(i)?;
literal(i.state.syntax.comment_end).parse_next(i)?;
depth = new_depth;
} else {
return Ok(());
@ -1422,19 +1378,18 @@ impl<'a> Comment<'a> {
}
}
fn comment<'a>(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a> {
let start = s.syntax.comment_start.span().parse_next(i)?;
let mut content = opt(terminated(
(|i: &mut _| content(i, s)).take(),
s.syntax.comment_end,
));
fn comment<'a: 'l, 'l>(i: &mut InputStream<'a, 'l>) -> ParseResult<'a> {
let start = i.state.syntax.comment_start.span().parse_next(i)?;
let mut content = opt(terminated(content.take(), i.state.syntax.comment_end));
let Some(content) = content.parse_next(i)? else {
return Err(ErrorContext::unclosed("comment", s.syntax.comment_end, start).cut());
return Err(
ErrorContext::unclosed("comment", i.state.syntax.comment_end, start).cut(),
);
};
Ok(content)
}
let (content, span) = (|i: &mut _| comment(i, s)).with_span().parse_next(i)?;
let (content, span) = comment.with_span().parse_next(i)?;
let ws = match *content.as_bytes() {
[b'-' | b'+' | b'~'] => {
return cut_error!(
@ -1442,7 +1397,7 @@ impl<'a> Comment<'a> {
"ambiguous whitespace stripping\n\
use `{}{content} {content}{}` to apply the same whitespace stripping on \
both sides",
s.syntax.comment_start, s.syntax.comment_end,
i.state.syntax.comment_start, i.state.syntax.comment_end,
),
span,
);
@ -1463,11 +1418,11 @@ impl<'a> Comment<'a> {
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Ws(pub Option<Whitespace>, pub Option<Whitespace>);
fn end_node<'a, 'g: 'a>(
fn end_node<'a: 'l, 'g: 'a, 'l>(
node: &'g str,
expected: &'g str,
) -> impl ModalParser<InputStream<'a>, &'a str, ErrorContext> + 'g {
move |i: &mut InputStream<'a>| {
) -> impl ModalParser<InputStream<'a, 'l>, &'a str, ErrorContext> + 'g {
move |i: &mut InputStream<'a, 'l>| {
let start = i.checkpoint();
let (actual, span) = ws(identifier.with_span()).parse_next(i)?;
if actual == expected {

57
askama_parser/src/target.rs
View File

@ -6,8 +6,8 @@ use winnow::{ModalParser, Parser};
use crate::{
CharLit, ErrorContext, InputStream, Num, ParseErr, ParseResult, PathComponent,
PathOrIdentifier, Span, State, StrLit, WithSpan, bool_lit, can_be_variable_name, char_lit,
cut_error, identifier, is_rust_keyword, keyword, num_lit, path_or_identifier, str_lit, ws,
PathOrIdentifier, Span, StrLit, WithSpan, bool_lit, can_be_variable_name, char_lit, cut_error,
identifier, is_rust_keyword, keyword, num_lit, path_or_identifier, str_lit, ws,
};
#[derive(Clone, Debug, PartialEq)]
@ -40,7 +40,7 @@ impl<'a> From<(WithSpan<&'a str>, Target<'a>)> for NamedTarget<'a> {
}
}
impl<'a> Target<'a> {
impl<'a: 'l, 'l> Target<'a> {
pub fn span(&self) -> Span {
match self {
Target::Name(v) => v.span(),
@ -59,17 +59,15 @@ impl<'a> Target<'a> {
}
/// Parses multiple targets with `or` separating them
pub(super) fn parse(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Self> {
pub(super) fn parse(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
enum OneOrMany<'a> {
One(Target<'a>),
Many(Vec<Target<'a>>),
}
let mut or_more = opt(preceded(ws(keyword("or")), |i: &mut _| {
Self::parse_one(i, s)
}));
let mut or_more = opt(preceded(ws(keyword("or")), Self::parse_one));
let one_or_many = |i: &mut _| {
let target = Self::parse_one(i, s)?;
let target = Self::parse_one(i)?;
let Some(snd_target) = or_more.parse_next(i)? else {
return Ok(OneOrMany::One(target));
};
@ -81,7 +79,7 @@ impl<'a> Target<'a> {
Ok(OneOrMany::Many(targets))
};
let _level_guard = s.level.nest(i)?;
let _level_guard = i.state.level.nest(i)?;
let (inner, span) = one_or_many.with_span().parse_next(i)?;
match inner {
OneOrMany::One(target) => Ok(target),
@ -90,7 +88,7 @@ impl<'a> Target<'a> {
}
/// Parses a single target without an `or`, unless it is wrapped in parentheses.
fn parse_one(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Self> {
fn parse_one(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
let mut opt_opening_paren = opt(ws('(').span()).map(|o| o.is_some());
let mut opt_opening_brace = opt(ws('{').span()).map(|o| o.is_some());
let mut opt_opening_bracket = opt(ws('[').span()).map(|o| o.is_some());
@ -105,8 +103,7 @@ impl<'a> Target<'a> {
// match tuples
let target_is_tuple = opt_opening_paren.parse_next(i)?;
if target_is_tuple {
let (is_singleton, mut targets) =
collect_targets(i, ')', |i: &mut _| Self::unnamed(i, s))?;
let (is_singleton, mut targets) = collect_targets(i, ')', Self::unnamed)?;
if is_singleton && let Some(target) = targets.pop() {
return Ok(target);
}
@ -119,14 +116,14 @@ impl<'a> Target<'a> {
// match array
let target_is_array = opt_opening_bracket.parse_next(i)?;
if target_is_array {
let targets = collect_targets(i, ']', |i: &mut _| Self::unnamed(i, s))?.1;
let targets = collect_targets(i, ']', Self::unnamed)?.1;
let inner = only_one_rest_pattern(targets, true, "array")?;
let range = start..i.current_token_start();
return Ok(Self::Array(WithSpan::new(inner, range)));
}
// match structs
let path = (|i: &mut _| path_or_identifier(i, s.level)).verify_map(|r| match r {
let path = path_or_identifier.verify_map(|r| match r {
PathOrIdentifier::Path(v) => Some(v),
PathOrIdentifier::Identifier(_) => None,
});
@ -137,14 +134,14 @@ impl<'a> Target<'a> {
let is_unnamed_struct = opt_opening_paren.parse_next(i)?;
if is_unnamed_struct {
let targets = collect_targets(i, ')', |i: &mut _| Self::unnamed(i, s))?.1;
let targets = collect_targets(i, ')', Self::unnamed)?.1;
let inner = only_one_rest_pattern(targets, false, "struct")?;
return Ok(Self::Tuple(WithSpan::new((path, inner), path_span)));
}
let is_named_struct = opt_opening_brace.parse_next(i)?;
if is_named_struct {
let targets = collect_targets(i, '}', |i: &mut _| Self::named(i, s))?.1;
let targets = collect_targets(i, '}', Self::named)?.1;
return Ok(Self::Struct(WithSpan::new((path, targets), path_span)));
}
@ -184,7 +181,7 @@ impl<'a> Target<'a> {
}
}
fn lit(i: &mut InputStream<'a>) -> ParseResult<'a, Self> {
fn lit(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
enum Lit<'a> {
Str(StrLit<'a>),
Bool(&'a str),
@ -208,13 +205,12 @@ impl<'a> Target<'a> {
}
}
fn unnamed(i: &mut InputStream<'a>, s: &State<'_, '_>) -> ParseResult<'a, Self> {
alt((Self::rest, |i: &mut _| Self::parse(i, s))).parse_next(i)
fn unnamed(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
alt((Self::rest, Self::parse)).parse_next(i)
}
fn named<O: From<(WithSpan<&'a str>, Self)>>(
i: &mut InputStream<'a>,
s: &State<'_, '_>,
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, O> {
if let Some(rest) = opt(Self::rest_inner).parse_next(i)? {
let chr = peek(ws(opt(one_of([',', ':']).with_span()))).parse_next(i)?;
@ -233,11 +229,8 @@ impl<'a> Target<'a> {
}
Ok((WithSpan::new("..", rest.span), Target::Rest(rest)).into())
} else {
let ((src, span), target) = (
identifier.with_span(),
opt(preceded(ws(':'), |i: &mut _| Self::parse(i, s))),
)
.parse_next(i)?;
let ((src, span), target) =
(identifier.with_span(), opt(preceded(ws(':'), Self::parse))).parse_next(i)?;
let src = WithSpan::new(src, span);
if *src == "_" {
@ -255,11 +248,13 @@ impl<'a> Target<'a> {
}
}
fn rest(i: &mut InputStream<'a>) -> ParseResult<'a, Self> {
fn rest(i: &mut InputStream<'a, 'l>) -> ParseResult<'a, Self> {
Self::rest_inner.map(Self::Rest).parse_next(i)
}
fn rest_inner(i: &mut InputStream<'a>) -> ParseResult<'a, WithSpan<Option<WithSpan<&'a str>>>> {
fn rest_inner(
i: &mut InputStream<'a, 'l>,
) -> ParseResult<'a, WithSpan<Option<WithSpan<&'a str>>>> {
let p = |i: &mut _| {
let id =
terminated(opt(terminated(identifier.with_span(), ws('@'))), "..").parse_next(i)?;
@ -297,10 +292,10 @@ fn verify_name<'a>(name: WithSpan<&'a str>) -> Result<Target<'a>, ErrMode<ErrorC
}
}
fn collect_targets<'a, T>(
i: &mut InputStream<'a>,
fn collect_targets<'a: 'l, 'l, T>(
i: &mut InputStream<'a, 'l>,
delim: char,
one: impl ModalParser<InputStream<'a>, T, ErrorContext>,
one: impl ModalParser<InputStream<'a, 'l>, T, ErrorContext>,
) -> ParseResult<'a, (bool, Vec<T>)> {
let opt_comma = ws(opt(',')).map(|o| o.is_some());
let mut opt_end = ws(opt(one_of(delim))).map(|o| o.is_some());

193
askama_parser/src/tests.rs
View File

@ -1,10 +1,12 @@
use std::cell::Cell;
use winnow::{LocatingSlice, Parser};
use crate::expr::BinOp;
use crate::node::{Let, Lit, Raw, Whitespace, Ws};
use crate::{
Ast, Expr, Filter, InnerSyntax, InputStream, Node, Num, PathComponent, PathOrIdentifier,
StrLit, Syntax, SyntaxBuilder, Target, WithSpan,
Ast, Expr, Filter, InnerSyntax, InputStream, Level, Node, Num, PathComponent, PathOrIdentifier,
State, StrLit, Syntax, SyntaxBuilder, Target, WithSpan,
};
fn as_path<'a>(path: &'a [&'a str]) -> Vec<PathComponent<'a>> {
@ -36,7 +38,8 @@ fn test_ws_splitter() {
#[test]
#[should_panic]
fn test_invalid_block() {
Ast::from_str("{% extend \"blah\" %}", None, &Syntax::default()).unwrap();
let syntax = Syntax::default();
Ast::from_str("{% extend \"blah\" %}", None, &syntax).unwrap();
}
fn int_lit<'a>(i: &'a str) -> WithSpan<Box<Expr<'a>>> {
@ -133,17 +136,17 @@ fn test_parse_numbers() {
#[test]
fn test_parse_var() {
let s = Syntax::default();
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{{ foo }}", None, &s).unwrap().nodes,
Ast::from_str("{{ foo }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Var("foo")))
))]
);
assert_eq!(
Ast::from_str("{{ foo_bar }}", None, &s).unwrap().nodes,
Ast::from_str("{{ foo_bar }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Var("foo_bar")))
@ -151,7 +154,7 @@ fn test_parse_var() {
);
assert_eq!(
Ast::from_str("{{ none }}", None, &s).unwrap().nodes,
Ast::from_str("{{ none }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Var("none")))
@ -161,17 +164,17 @@ fn test_parse_var() {
#[test]
fn test_parse_const() {
let s = Syntax::default();
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{{ FOO }}", None, &s).unwrap().nodes,
Ast::from_str("{{ FOO }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Path(as_path(&["FOO"]))))
))]
);
assert_eq!(
Ast::from_str("{{ FOO_BAR }}", None, &s).unwrap().nodes,
Ast::from_str("{{ FOO_BAR }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Path(as_path(&["FOO_BAR"]))))
@ -179,7 +182,7 @@ fn test_parse_const() {
);
assert_eq!(
Ast::from_str("{{ NONE }}", None, &s).unwrap().nodes,
Ast::from_str("{{ NONE }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Path(as_path(&["NONE"]))))
@ -189,17 +192,19 @@ fn test_parse_const() {
#[test]
fn test_parse_path() {
let s = Syntax::default();
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{{ None }}", None, &s).unwrap().nodes,
Ast::from_str("{{ None }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Path(as_path(&["None"])))),
))]
);
assert_eq!(
Ast::from_str("{{ Some(123) }}", None, &s).unwrap().nodes,
Ast::from_str("{{ Some(123) }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
Ws(None, None),
call(
@ -210,7 +215,7 @@ fn test_parse_path() {
);
assert_eq!(
Ast::from_str("{{ Ok(123) }}", None, &s).unwrap().nodes,
Ast::from_str("{{ Ok(123) }}", None, &syntax).unwrap().nodes,
[Box::new(Node::Expr(
Ws(None, None),
call(
@ -220,7 +225,9 @@ fn test_parse_path() {
))],
);
assert_eq!(
Ast::from_str("{{ Err(123) }}", None, &s).unwrap().nodes,
Ast::from_str("{{ Err(123) }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
Ws(None, None),
call(
@ -233,8 +240,10 @@ fn test_parse_path() {
#[test]
fn test_parse_var_call() {
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{{ function(\"123\", 3) }}", None, &Syntax::default())
Ast::from_str("{{ function(\"123\", 3) }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
@ -259,17 +268,19 @@ fn test_parse_var_call() {
#[test]
fn test_parse_path_call() {
let s = Syntax::default();
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{{ Option::None }}", None, &s).unwrap().nodes,
Ast::from_str("{{ Option::None }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
Ws(None, None),
WithSpan::no_span(Box::new(Expr::Path(as_path(&["Option", "None"])))),
))],
);
assert_eq!(
Ast::from_str("{{ Option::Some(123) }}", None, &s)
Ast::from_str("{{ Option::Some(123) }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
@ -282,7 +293,7 @@ fn test_parse_path_call() {
);
assert_eq!(
Ast::from_str("{{ self::function(\"123\", 3) }}", None, &s)
Ast::from_str("{{ self::function(\"123\", 3) }}", None, &syntax)
.unwrap()
.nodes,
[Box::new(Node::Expr(
@ -741,8 +752,8 @@ fn test_odd_calls() {
fn test_parse_comments() {
#[track_caller]
fn one_comment_ws(source: &str, ws: Ws) {
let s = &Syntax::default();
let mut nodes = Ast::from_str(source, None, s).unwrap().nodes;
let syntax = Syntax::default();
let mut nodes = Ast::from_str(source, None, &syntax).unwrap().nodes;
assert_eq!(nodes.len(), 1, "expected to parse one node");
match *nodes.pop().unwrap() {
Node::Comment(comment) => assert_eq!(comment.ws, ws),
@ -1055,10 +1066,12 @@ fn fuzzed_unary_recursion() {
#[test]
fn fuzzed_comment_depth() {
let syntax = Syntax::default();
let (sender, receiver) = std::sync::mpsc::channel();
let test = std::thread::spawn(move || {
const TEMPLATE: &str = include_str!("../tests/comment-depth.txt");
assert!(Ast::from_str(TEMPLATE, None, &Syntax::default()).is_ok());
assert!(Ast::from_str(TEMPLATE, None, &syntax).is_ok());
sender.send(()).unwrap();
});
receiver
@ -1069,13 +1082,10 @@ fn fuzzed_comment_depth() {
#[test]
fn let_set() {
let syntax = Syntax::default();
assert_eq!(
Ast::from_str("{% let a %}", None, &Syntax::default())
.unwrap()
.nodes(),
Ast::from_str("{% set a %}", None, &Syntax::default())
.unwrap()
.nodes(),
Ast::from_str("{% let a %}", None, &syntax).unwrap().nodes(),
Ast::from_str("{% set a %}", None, &syntax).unwrap().nodes(),
);
}
@ -1181,110 +1191,35 @@ fn fuzzed_excessive_filter_block() {
#[test]
fn test_generics_parsing() {
let syntax = Syntax::default();
// Method call.
Ast::from_str("{{ a.b::<&str, H<B<C>>>() }}", None, &Syntax::default()).unwrap();
Ast::from_str(
"{{ a.b::<&str, H<B<C> , &u32>>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str("{{ a.b::<&str, H<B<C>>>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&str, H<B<C> , &u32>>() }}", None, &syntax).unwrap();
// Call.
Ast::from_str(
"{{ a::<&str, H<B<C> , &u32>>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str("{{ a::<&str, H<B<C> , &u32>>() }}", None, &syntax).unwrap();
// Filter.
Ast::from_str("{{ 12 | a::<&str> }}", None, &Syntax::default()).unwrap();
Ast::from_str("{{ 12 | a::<&str, u32>('a') }}", None, &Syntax::default()).unwrap();
Ast::from_str("{{ 12 | a::<&str> }}", None, &syntax).unwrap();
Ast::from_str("{{ 12 | a::<&str, u32>('a') }}", None, &syntax).unwrap();
// Unclosed `<`.
assert!(
Ast::from_str(
"{{ a.b::<&str, H<B<C> , &u32>() }}",
None,
&Syntax::default()
)
.is_err()
);
assert!(Ast::from_str("{{ a.b::<&str, H<B<C> , &u32>() }}", None, &syntax).is_err());
// With path and spaces
Ast::from_str(
"{{ a.b::<&&core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b ::<&&core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b:: <&&core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::< &&core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<& &core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&& core::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core ::primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core:: primitive::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core::primitive ::str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core::primitive:: str>() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core::primitive::str >() }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str(
"{{ a.b::<&&core::primitive::str> () }}",
None,
&Syntax::default(),
)
.unwrap();
Ast::from_str("{{ a.b::<&&core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b ::<&&core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b:: <&&core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::< &&core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<& &core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&& core::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core ::primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core:: primitive::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core::primitive ::str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core::primitive:: str>() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core::primitive::str >() }}", None, &syntax).unwrap();
Ast::from_str("{{ a.b::<&&core::primitive::str> () }}", None, &syntax).unwrap();
}
#[test]
@ -1365,9 +1300,14 @@ fn test_filter_with_path() {
#[test]
fn underscore_is_an_identifier() {
let state = State {
syntax: Syntax::default(),
loop_depth: Cell::new(0),
level: Level::default(),
};
let mut input = InputStream {
input: LocatingSlice::new("_"),
state: (),
state: &state,
};
let result = crate::identifier.parse_next(&mut input);
assert_eq!(result.unwrap(), "_");
@ -1512,6 +1452,7 @@ fn macro_comments_in_macro_calls() {
#[test]
fn test_raw() {
let syntax = Syntax::default();
let val = "hello {{ endraw %} my {%* endraw %} green {% endraw }} world";
assert_eq!(
Ast::from_str(