diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java
index 0e5df631c..6ec9dd259 100644
--- a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java
+++ b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java
@@ -608,11 +608,14 @@ private Token emitToken(int kind) {
(t instanceof TagToken || t instanceof NoteToken) &&
config.isTrimBlocks() &&
currPost < length &&
- is[currPost] == '\n'
+ (is[currPost] == '\n' ||
+ (is[currPost] == '\r' && currPost + 1 < length && is[currPost + 1] == '\n'))
) {
- lastNewlinePos = currPost + 1;
- ++currPost;
- ++tokenStart;
+ int newlineLength = is[currPost] == '\r' ? 2 : 1;
+ currPost += newlineLength;
+ tokenStart += newlineLength;
+ lastNewlinePos = currPost;
+ ++currLine;
}
if (t instanceof TagToken) {
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
index de3b6e040..e35a87a21 100644
--- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
+++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
@@ -287,11 +287,14 @@ private Token newToken(int kind) {
(t instanceof TagToken || t instanceof NoteToken) &&
config.isTrimBlocks() &&
currPost < length &&
- is[currPost] == '\n'
+ (is[currPost] == '\n' ||
+ (is[currPost] == '\r' && currPost + 1 < length && is[currPost + 1] == '\n'))
) {
+ int newlineLength = is[currPost] == '\r' ? 2 : 1;
+ currPost += newlineLength;
+ tokenStart += newlineLength;
lastNewlinePos = currPost;
- ++currPost;
- ++tokenStart;
+ ++currLine;
}
if (t instanceof TagToken) {
diff --git a/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java b/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java
index 7beba0733..4e5fb32ef 100644
--- a/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java
+++ b/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java
@@ -150,6 +150,41 @@ public void trimAndLstripBlocks() {
.isEqualTo("
\n" + " yay\n" + "
\n");
}
+ @Test
+ public void itTrimsCrLfAfterBlocks() {
+ interpreter =
+ new Jinjava(
+ BaseJinjavaTest
+ .newConfigBuilder()
+ .withLstripBlocks(true)
+ .withTrimBlocks(true)
+ .build()
+ )
+ .newInterpreter();
+ interpreter.getContext().put("bootefiDev", "sda");
+ interpreter.getContext().put("bootDev", "sda");
+ interpreter.getContext().put("rootDev", "sda");
+
+ String template =
+ "{% if bootefiDev %}\r\n" +
+ "part /boot/efi --fstype=\"efi\" --size=500 --ondisk={{ bootefiDev }}\r\n" +
+ "{% endif %}\r\n" +
+ "{% if bootDev %}\r\n" +
+ "part /boot --fstype ext4 --size=2048 --ondisk={{ bootDev }}\r\n" +
+ "{% endif %}\r\n" +
+ "{% if rootDev %}\r\n" +
+ "part / --fstype ext4 --size=1 --grow --ondisk={{ rootDev }}\r\n" +
+ "{% endif %}\r\n";
+
+ assertThat(interpreter.render(interpreter.parse(template)))
+ .isEqualTo(
+ "part /boot/efi --fstype=\"efi\" --size=500 --ondisk=sda\r\n" +
+ "part /boot --fstype ext4 --size=2048 --ondisk=sda\r\n" +
+ "part / --fstype ext4 --size=1 --grow --ondisk=sda\r\n"
+ );
+ assertThat(interpreter.getErrors()).isEmpty();
+ }
+
@Test
public void trimAndLstripCommentBlocks() {
interpreter =
diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java
index b7af6c074..c65851b23 100644
--- a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java
+++ b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java
@@ -323,6 +323,59 @@ public void itRespectsLstripBlocksWithLatexSymbols() {
assertThat(result).isEqualTo("hello\n");
}
+ @Test
+ public void itRespectsWhitespaceOptionsWithCrLfAndAngleSymbols() {
+ assertCrLfWhitespaceOptions(
+ ANGLE_SYMBOLS,
+ " <% if show %>\r\nhello\r\n <% endif %>\r\ntail"
+ );
+ }
+
+ @Test
+ public void itRespectsWhitespaceOptionsWithCrLfAndLatexSymbols() {
+ assertCrLfWhitespaceOptions(
+ LATEX_SYMBOLS,
+ " \\BLOCK{ if show }\r\nhello\r\n \\BLOCK{ endif }\r\ntail"
+ );
+ }
+
+ @Test
+ public void itTrimsCrLfAroundCommentsAndRawBlocksWithCustomSymbols() {
+ for (StringTokenScannerSymbols symbols : Lists.newArrayList(
+ ANGLE_SYMBOLS,
+ LATEX_SYMBOLS
+ )) {
+ Jinjava j = new Jinjava(
+ BaseJinjavaTest
+ .newConfigBuilder()
+ .withTokenScannerSymbols(symbols)
+ .withTrimBlocks(true)
+ .withKeepTrailingNewline(true)
+ .build()
+ );
+ String expression =
+ symbols.getExpressionStart() + " value " + symbols.getExpressionEnd();
+ String template =
+ symbols.getOpeningComment() +
+ " comment " +
+ symbols.getClosingComment() +
+ "\r\n" +
+ symbols.getExpressionStartWithTag() +
+ " raw " +
+ symbols.getExpressionEndWithTag() +
+ "\r\n" +
+ expression +
+ "\r\n" +
+ symbols.getExpressionStartWithTag() +
+ " endraw " +
+ symbols.getExpressionEndWithTag() +
+ "\r\ntail";
+
+ assertThat(j.render(template, ImmutableMap.of("value", "evaluated")))
+ .isEqualTo(expression + "\r\ntail");
+ }
+ }
+
@Test
public void builderRejectsEmptyDelimiter() {
assertThatThrownBy(() ->
@@ -505,6 +558,31 @@ public void itHandlesBothLinePrefixesTogether() {
// ── Helper ────────────────────────────────────────────────────────────────
+ private void assertCrLfWhitespaceOptions(
+ StringTokenScannerSymbols symbols,
+ String template
+ ) {
+ for (boolean trimBlocks : new boolean[] { false, true }) {
+ for (boolean lstripBlocks : new boolean[] { false, true }) {
+ Jinjava j = new Jinjava(
+ BaseJinjavaTest
+ .newConfigBuilder()
+ .withTokenScannerSymbols(symbols)
+ .withTrimBlocks(trimBlocks)
+ .withLstripBlocks(lstripBlocks)
+ .withKeepTrailingNewline(true)
+ .build()
+ );
+ String indent = lstripBlocks ? "" : " ";
+ String tagNewline = trimBlocks ? "" : "\r\n";
+
+ assertThat(j.render(template, ImmutableMap.of("show", true)))
+ .as("trimBlocks=%s, lstripBlocks=%s", trimBlocks, lstripBlocks)
+ .isEqualTo(indent + tagNewline + "hello\r\n" + indent + tagNewline + "tail");
+ }
+ }
+ }
+
private Jinjava jinjavaWith(StringTokenScannerSymbols symbols) {
return new Jinjava(
BaseJinjavaTest
diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java
index 09c0f643a..367cae17c 100644
--- a/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java
+++ b/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java
@@ -9,6 +9,7 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.stream.Collectors;
import org.junit.Test;
public class TokenWhitespaceTest {
@@ -32,6 +33,99 @@ public void trimBlocksTrimsAfterCommentTag() {
assertThat(tokens.get(4).getImage()).isEqualTo(" whoop\n\n");
}
+ @Test
+ public void trimBlocksTracksTheNextTokenPosition() {
+ for (String block : List.of("{% set x = 1 %}", "{# comment #}")) {
+ for (String newline : List.of("\n", "\r\n")) {
+ for (List tokens : scanBoth(block + newline + " {{ x }}", true)) {
+ assertThat(tokens)
+ .extracting(Token::getImage)
+ .containsExactly(block, " ", "{{ x }}");
+ assertThat(tokens.get(0).getLineNumber()).isEqualTo(1);
+ assertThat(tokens.get(2).getLineNumber()).isEqualTo(2);
+ assertThat(tokens.get(2).getStartPosition()).isEqualTo(3);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void trimBlocksConsumesOnlyOneNewline() {
+ for (String block : List.of("{% set x = 1 %}", "{# comment #}")) {
+ for (String first : List.of("\n", "\r\n")) {
+ for (String second : List.of("\n", "\r\n")) {
+ for (List tokens : scanBoth(block + first + second + "{{ x }}", true)) {
+ assertThat(tokens)
+ .extracting(Token::getImage)
+ .containsExactly(block, second, "{{ x }}");
+ assertThat(tokens.get(2).getLineNumber()).isEqualTo(3);
+ assertThat(tokens.get(2).getStartPosition()).isEqualTo(1);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void trimBlocksHandlesNewlinesAtEndOfInput() {
+ for (String block : List.of("{% set x = 1 %}", "{# comment #}")) {
+ for (String newline : List.of("\n", "\r\n")) {
+ for (List tokens : scanBoth(block + newline, true)) {
+ assertThat(tokens).extracting(Token::getImage).containsExactly(block);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void trimBlocksPreservesStandaloneCarriageReturnsAndInterveningWhitespace() {
+ for (String block : List.of("{% set x = 1 %}", "{# comment #}")) {
+ for (String suffix : List.of("", "\r", "\rtext", " \r\n", "\t\n", "\r\r\n")) {
+ String input = block + suffix;
+ for (List tokens : scanBoth(input, true)) {
+ assertThat(tokens.stream().map(Token::getImage).collect(Collectors.joining()))
+ .isEqualTo(input);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void trimBlocksDoesNotTrimAfterExpressions() {
+ for (String newline : List.of("\n", "\r\n")) {
+ String input = "{{ 1 }}" + newline + "text";
+ for (List tokens : scanBoth(input, true)) {
+ assertThat(tokens)
+ .extracting(Token::getImage)
+ .containsExactly("{{ 1 }}", newline + "text");
+ }
+ }
+ }
+
+ @Test
+ public void disabledTrimBlocksPreservesNewlines() {
+ for (String block : List.of("{% set x = 1 %}", "{# comment #}")) {
+ for (String newline : List.of("\n", "\r\n")) {
+ for (List tokens : scanBoth(block + newline + "text", false)) {
+ assertThat(tokens)
+ .extracting(Token::getImage)
+ .containsExactly(block, newline + "text");
+ }
+ }
+ }
+ }
+
+ private List> scanBoth(String input, boolean trimBlocks) {
+ JinjavaConfig config = BaseJinjavaTest
+ .newConfigBuilder()
+ .withTrimBlocks(trimBlocks)
+ .build();
+ return List.of(
+ Lists.newArrayList(new TokenScanner(input, config)),
+ Lists.newArrayList(new StringTokenScanner(input, config))
+ );
+ }
+
private List scanTokens(String srcPath, JinjavaConfig config) {
try {
return Lists.newArrayList(