diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java index 0e5df631c..6ec9dd259 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/StringTokenScanner.java @@ -608,11 +608,14 @@ private Token emitToken(int kind) { (t instanceof TagToken || t instanceof NoteToken) && config.isTrimBlocks() && currPost < length && - is[currPost] == '\n' + (is[currPost] == '\n' || + (is[currPost] == '\r' && currPost + 1 < length && is[currPost + 1] == '\n')) ) { - lastNewlinePos = currPost + 1; - ++currPost; - ++tokenStart; + int newlineLength = is[currPost] == '\r' ? 2 : 1; + currPost += newlineLength; + tokenStart += newlineLength; + lastNewlinePos = currPost; + ++currLine; } if (t instanceof TagToken) { diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java index de3b6e040..e35a87a21 100644 --- a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java +++ b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java @@ -287,11 +287,14 @@ private Token newToken(int kind) { (t instanceof TagToken || t instanceof NoteToken) && config.isTrimBlocks() && currPost < length && - is[currPost] == '\n' + (is[currPost] == '\n' || + (is[currPost] == '\r' && currPost + 1 < length && is[currPost + 1] == '\n')) ) { + int newlineLength = is[currPost] == '\r' ? 2 : 1; + currPost += newlineLength; + tokenStart += newlineLength; lastNewlinePos = currPost; - ++currPost; - ++tokenStart; + ++currLine; } if (t instanceof TagToken) { diff --git a/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java b/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java index 7beba0733..4e5fb32ef 100644 --- a/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/TreeParserTest.java @@ -150,6 +150,41 @@ public void trimAndLstripBlocks() { .isEqualTo("
\n" + " yay\n" + "
\n"); } + @Test + public void itTrimsCrLfAfterBlocks() { + interpreter = + new Jinjava( + BaseJinjavaTest + .newConfigBuilder() + .withLstripBlocks(true) + .withTrimBlocks(true) + .build() + ) + .newInterpreter(); + interpreter.getContext().put("bootefiDev", "sda"); + interpreter.getContext().put("bootDev", "sda"); + interpreter.getContext().put("rootDev", "sda"); + + String template = + "{% if bootefiDev %}\r\n" + + "part /boot/efi --fstype=\"efi\" --size=500 --ondisk={{ bootefiDev }}\r\n" + + "{% endif %}\r\n" + + "{% if bootDev %}\r\n" + + "part /boot --fstype ext4 --size=2048 --ondisk={{ bootDev }}\r\n" + + "{% endif %}\r\n" + + "{% if rootDev %}\r\n" + + "part / --fstype ext4 --size=1 --grow --ondisk={{ rootDev }}\r\n" + + "{% endif %}\r\n"; + + assertThat(interpreter.render(interpreter.parse(template))) + .isEqualTo( + "part /boot/efi --fstype=\"efi\" --size=500 --ondisk=sda\r\n" + + "part /boot --fstype ext4 --size=2048 --ondisk=sda\r\n" + + "part / --fstype ext4 --size=1 --grow --ondisk=sda\r\n" + ); + assertThat(interpreter.getErrors()).isEmpty(); + } + @Test public void trimAndLstripCommentBlocks() { interpreter = diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java index b7af6c074..c65851b23 100644 --- a/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/parse/StringTokenScannerSymbolsTest.java @@ -323,6 +323,59 @@ public void itRespectsLstripBlocksWithLatexSymbols() { assertThat(result).isEqualTo("hello\n"); } + @Test + public void itRespectsWhitespaceOptionsWithCrLfAndAngleSymbols() { + assertCrLfWhitespaceOptions( + ANGLE_SYMBOLS, + " <% if show %>\r\nhello\r\n <% endif %>\r\ntail" + ); + } + + @Test + public void itRespectsWhitespaceOptionsWithCrLfAndLatexSymbols() { + assertCrLfWhitespaceOptions( + LATEX_SYMBOLS, + " \\BLOCK{ if show }\r\nhello\r\n \\BLOCK{ endif }\r\ntail" + ); + } + + @Test + public void itTrimsCrLfAroundCommentsAndRawBlocksWithCustomSymbols() { + for (StringTokenScannerSymbols symbols : Lists.newArrayList( + ANGLE_SYMBOLS, + LATEX_SYMBOLS + )) { + Jinjava j = new Jinjava( + BaseJinjavaTest + .newConfigBuilder() + .withTokenScannerSymbols(symbols) + .withTrimBlocks(true) + .withKeepTrailingNewline(true) + .build() + ); + String expression = + symbols.getExpressionStart() + " value " + symbols.getExpressionEnd(); + String template = + symbols.getOpeningComment() + + " comment " + + symbols.getClosingComment() + + "\r\n" + + symbols.getExpressionStartWithTag() + + " raw " + + symbols.getExpressionEndWithTag() + + "\r\n" + + expression + + "\r\n" + + symbols.getExpressionStartWithTag() + + " endraw " + + symbols.getExpressionEndWithTag() + + "\r\ntail"; + + assertThat(j.render(template, ImmutableMap.of("value", "evaluated"))) + .isEqualTo(expression + "\r\ntail"); + } + } + @Test public void builderRejectsEmptyDelimiter() { assertThatThrownBy(() -> @@ -505,6 +558,31 @@ public void itHandlesBothLinePrefixesTogether() { // ── Helper ──────────────────────────────────────────────────────────────── + private void assertCrLfWhitespaceOptions( + StringTokenScannerSymbols symbols, + String template + ) { + for (boolean trimBlocks : new boolean[] { false, true }) { + for (boolean lstripBlocks : new boolean[] { false, true }) { + Jinjava j = new Jinjava( + BaseJinjavaTest + .newConfigBuilder() + .withTokenScannerSymbols(symbols) + .withTrimBlocks(trimBlocks) + .withLstripBlocks(lstripBlocks) + .withKeepTrailingNewline(true) + .build() + ); + String indent = lstripBlocks ? "" : " "; + String tagNewline = trimBlocks ? "" : "\r\n"; + + assertThat(j.render(template, ImmutableMap.of("show", true))) + .as("trimBlocks=%s, lstripBlocks=%s", trimBlocks, lstripBlocks) + .isEqualTo(indent + tagNewline + "hello\r\n" + indent + tagNewline + "tail"); + } + } + } + private Jinjava jinjavaWith(StringTokenScannerSymbols symbols) { return new Jinjava( BaseJinjavaTest diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java index 09c0f643a..367cae17c 100644 --- a/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java +++ b/src/test/java/com/hubspot/jinjava/tree/parse/TokenWhitespaceTest.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.stream.Collectors; import org.junit.Test; public class TokenWhitespaceTest { @@ -32,6 +33,99 @@ public void trimBlocksTrimsAfterCommentTag() { assertThat(tokens.get(4).getImage()).isEqualTo(" whoop\n\n"); } + @Test + public void trimBlocksTracksTheNextTokenPosition() { + for (String block : List.of("{% set x = 1 %}", "{# comment #}")) { + for (String newline : List.of("\n", "\r\n")) { + for (List tokens : scanBoth(block + newline + " {{ x }}", true)) { + assertThat(tokens) + .extracting(Token::getImage) + .containsExactly(block, " ", "{{ x }}"); + assertThat(tokens.get(0).getLineNumber()).isEqualTo(1); + assertThat(tokens.get(2).getLineNumber()).isEqualTo(2); + assertThat(tokens.get(2).getStartPosition()).isEqualTo(3); + } + } + } + } + + @Test + public void trimBlocksConsumesOnlyOneNewline() { + for (String block : List.of("{% set x = 1 %}", "{# comment #}")) { + for (String first : List.of("\n", "\r\n")) { + for (String second : List.of("\n", "\r\n")) { + for (List tokens : scanBoth(block + first + second + "{{ x }}", true)) { + assertThat(tokens) + .extracting(Token::getImage) + .containsExactly(block, second, "{{ x }}"); + assertThat(tokens.get(2).getLineNumber()).isEqualTo(3); + assertThat(tokens.get(2).getStartPosition()).isEqualTo(1); + } + } + } + } + } + + @Test + public void trimBlocksHandlesNewlinesAtEndOfInput() { + for (String block : List.of("{% set x = 1 %}", "{# comment #}")) { + for (String newline : List.of("\n", "\r\n")) { + for (List tokens : scanBoth(block + newline, true)) { + assertThat(tokens).extracting(Token::getImage).containsExactly(block); + } + } + } + } + + @Test + public void trimBlocksPreservesStandaloneCarriageReturnsAndInterveningWhitespace() { + for (String block : List.of("{% set x = 1 %}", "{# comment #}")) { + for (String suffix : List.of("", "\r", "\rtext", " \r\n", "\t\n", "\r\r\n")) { + String input = block + suffix; + for (List tokens : scanBoth(input, true)) { + assertThat(tokens.stream().map(Token::getImage).collect(Collectors.joining())) + .isEqualTo(input); + } + } + } + } + + @Test + public void trimBlocksDoesNotTrimAfterExpressions() { + for (String newline : List.of("\n", "\r\n")) { + String input = "{{ 1 }}" + newline + "text"; + for (List tokens : scanBoth(input, true)) { + assertThat(tokens) + .extracting(Token::getImage) + .containsExactly("{{ 1 }}", newline + "text"); + } + } + } + + @Test + public void disabledTrimBlocksPreservesNewlines() { + for (String block : List.of("{% set x = 1 %}", "{# comment #}")) { + for (String newline : List.of("\n", "\r\n")) { + for (List tokens : scanBoth(block + newline + "text", false)) { + assertThat(tokens) + .extracting(Token::getImage) + .containsExactly(block, newline + "text"); + } + } + } + } + + private List> scanBoth(String input, boolean trimBlocks) { + JinjavaConfig config = BaseJinjavaTest + .newConfigBuilder() + .withTrimBlocks(trimBlocks) + .build(); + return List.of( + Lists.newArrayList(new TokenScanner(input, config)), + Lists.newArrayList(new StringTokenScanner(input, config)) + ); + } + private List scanTokens(String srcPath, JinjavaConfig config) { try { return Lists.newArrayList(