JIT: honour offset limits when FIRSTLINE is enabled

src/pcre2_jit_compile.c

@@ -5889,6 +5889,7 @@ mainloop_entry(compiler_common *common)
   jump_list *newline = NULL;
   sljit_u32 overall_options = common->re->overall_options;
   BOOL hascrorlf = (common->re->flags & PCRE2_HASCRORLF) != 0;
+  BOOL has_offset_limit = (overall_options & PCRE2_USE_OFFSET_LIMIT) != 0;
   BOOL newlinecheck = FALSE;
   BOOL readuchar = FALSE;
 
@@ -5904,6 +5905,45 @@ mainloop_entry(compiler_common *common)
     SLJIT_ASSERT(common->match_end_ptr != 0);
     OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);
 
+    /* Temporarily reduce STR_END for the newline lookup only. */
+    if (has_offset_limit)
+    {
+      /* Allow UTF decoding to finish a character at the offset limit.
+      The CRLF scan needs STR_PTR on LF, then records STR_PTR - 1 (the CR).
+      At exhaustion it records STR_END - 1, so it also needs one extra
+      code unit to preserve the inclusive offset limit. */
+      sljit_sw extra = 32 / PCRE2_CODE_UNIT_WIDTH;
+
+      OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LOCAL0, STR_END, 0);
+      if (HAS_VIRTUAL_REGISTERS)
+      {
+        OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, offset_limit));
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
+      }
+      else
+      {
+        OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, offset_limit));
+        OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin));
+      }
+
+      /* Compare in code units before scaling, also excluding PCRE2_UNSET. */
+      OP2(SLJIT_SUB, TMP2, 0, STR_END, 0, TMP2, 0);
+#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
+      OP2(SLJIT_LSHR, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT);
+#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */
+      end = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, TMP2, 0);
+      OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, TMP1, 0);
+      end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, SLJIT_IMM, extra);
+      OP2(SLJIT_SUB, TMP2, 0, TMP2, 0, SLJIT_IMM, extra);
+#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
+      OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT);
+#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */
+      OP2(SLJIT_SUB, STR_END, 0, STR_END, 0, TMP2, 0);
+      JUMPHERE(end2);
+      JUMPHERE(end);
+    }
+
     if (common->nltype == NLTYPE_FIXED && common->newline > 255)
     {
       mainloop = LABEL();
@@ -5932,8 +5972,11 @@ mainloop_entry(compiler_common *common)
     }
 
     OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
+    if (has_offset_limit)
+      OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), LOCAL0);
   }
-  else if ((overall_options & PCRE2_USE_OFFSET_LIMIT) != 0)
+
+  if (has_offset_limit)
   {
     /* Check whether offset limit is set and valid. */
     SLJIT_ASSERT(common->match_end_ptr != 0);
@@ -5948,7 +5991,10 @@ mainloop_entry(compiler_common *common)
       OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, offset_limit));
     }
 
-    OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
+    if ((overall_options & PCRE2_FIRSTLINE) != 0)
+      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
+    else
+      OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
     end = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw)PCRE2_UNSET);
     if (HAS_VIRTUAL_REGISTERS)
       OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
@@ -5962,8 +6008,17 @@ mainloop_entry(compiler_common *common)
       OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));
 
     OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);
-    end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0);
-    OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
+    if ((overall_options & PCRE2_FIRSTLINE) != 0)
+    {
+      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
+      end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, TMP1, 0);
+      OP1(SLJIT_MOV, TMP2, 0, TMP1, 0);
+    }
+    else
+    {
+      end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0);
+      OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
+    }
     JUMPHERE(end2);
     OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE2_ERROR_NOMATCH);
     add_jump(compiler, &common->abort, CMP(SLJIT_LESS, TMP2, 0, STR_PTR, 0));

testdata/testinput10

@@ -677,6 +677,27 @@
     line1\nbreak
     line0\nline1\nbreak
 
+# FIRSTLINE offset limits with stray and truncated UTF-8 around CRLF.
+
+/\r\n\r?/firstline,use_offset_limit,newline=crlf,match_invalid_utf,no_start_optimize
+    a\x80\r\n\=offset_limit=1
+    a\x80\r\n\=offset_limit=2
+    a\x80\r\n\=offset_limit=3
+    a\xe2\x80\r\n\r\=offset_limit=2
+    a\xe2\x80\r\n\r\=offset_limit=3
+    a\xe2\x80\r\n\r\=offset_limit=4
+    a\x80\r\n\rxxxxxxxx\=offset_limit=1
+    a\x80\r\n\rxxxxxxxx\=offset_limit=2
+    a\x80\r\n\rxxxxxxxx\=offset_limit=3
+    a\r\n\r\xe2\x80\=offset_limit=0
+    a\r\n\r\xe2\x80\=offset_limit=1
+    a\r\n\r\xe2\x80\=offset_limit=2
+
+/\r\n\r?/firstline,use_offset_limit,newline=any,match_invalid_utf,no_start_optimize
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=2
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=3
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=4
+
 /A\z/utf,match_invalid_utf
     A\x80\x42\n
 

testdata/testinput12

@@ -474,6 +474,27 @@
 /aa/utf,ucp,match_invalid_utf,global
     \x{d800}aa
 
+# FIRSTLINE offset limits with unpaired surrogates around CRLF.
+
+/\r\n\r?/firstline,use_offset_limit,newline=crlf,match_invalid_utf,no_start_optimize
+    a\x{dc00}\r\n\=offset_limit=1
+    a\x{dc00}\r\n\=offset_limit=2
+    a\x{dc00}\r\n\=offset_limit=3
+    a\x{d800}\r\n\r\=offset_limit=1
+    a\x{d800}\r\n\r\=offset_limit=2
+    a\x{d800}\r\n\r\=offset_limit=3
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=1
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=2
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=3
+    a\r\n\r\x{d800}\=offset_limit=0
+    a\r\n\r\x{d800}\=offset_limit=1
+    a\r\n\r\x{d800}\=offset_limit=2
+
+/\r\n\r?/firstline,use_offset_limit,newline=any,match_invalid_utf,no_start_optimize
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=1
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=2
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=3
+
 /A\z/utf,match_invalid_utf
     A\x{df00}\n
 

testdata/testinput2

@@ -4930,6 +4930,127 @@ B)x/alt_verbnames,mark
     \n..\naa\=offset_limit=3
     \n..\naa\=offset_limit=4
 
+# FIRSTLINE and offset limits must both constrain the starting position.
+
+/b/firstline,use_offset_limit,newline=lf
+    ab\nb\=offset_limit=0
+    ab\nb\=offset_limit=1
+    ab\nb\=offset_limit=2
+    ab\nb\=offset_limit=4
+    ab\nb
+    b\nb\=offset_limit=0
+    a\nb\=offset_limit=3
+    ab\=offset_limit=0
+    ab\=offset_limit=2
+    ab\nb\=offset=1,offset_limit=0
+    ab\nb\=offset=1,offset_limit=1
+    a\nab\=offset=2,offset_limit=2
+    a\nab\=offset=2,offset_limit=3
+    a\nb\=offset=2,offset_limit=2
+
+/b/firstline,use_offset_limit,newline=lf,no_start_optimize
+    ab\nb\=offset_limit=0
+    ab\nb\=offset_limit=1
+    ab\nb\=offset_limit=2
+    ab\nb\=offset_limit=4
+    ab\nb
+    b\nb\=offset_limit=0
+    a\nb\=offset_limit=3
+    ab\=offset_limit=0
+    ab\=offset_limit=2
+    ab\nb\=offset=1,offset_limit=0
+    ab\nb\=offset=1,offset_limit=1
+    a\nab\=offset=2,offset_limit=2
+    a\nab\=offset=2,offset_limit=3
+    a\nb\=offset=2,offset_limit=2
+
+/\n/firstline,use_offset_limit,newline=lf
+    a\nb\=offset_limit=0
+    a\nb\=offset_limit=1
+    a\nb\=offset_limit=3
+
+/b\nc/firstline,use_offset_limit,newline=lf
+    ab\nc\=offset_limit=0
+    ab\nc\=offset_limit=1
+    ab\n\=offset_limit=1,ps
+    ab\n\=offset_limit=1,ph
+
+/b/firstline,use_offset_limit,newline=crlf
+    ab\r\nb\=offset_limit=0
+    ab\r\nb\=offset_limit=1
+    ab\r\nb\=offset_limit=2
+    ab\r\nb\=offset_limit=5
+    ab\r\nb\=offset_limit=50
+    ab\r\nb
+    a\r\nb\=offset_limit=4
+    b\r\nb\=offset_limit=0
+    ab\=offset_limit=1
+    ab\r\=offset_limit=1
+
+# The scan limit is inclusive, and does not truncate the match.
+
+/\r\n/firstline,use_offset_limit,newline=crlf
+    a\r\nb\=offset_limit=0
+    a\r\nb\=offset_limit=1
+    a\r\nb\=offset_limit=2
+    \r\nb\=offset_limit=0
+
+/b\r\nc/firstline,use_offset_limit,newline=crlf
+    ab\r\nc\=offset_limit=0
+    ab\r\nc\=offset_limit=1
+    ab\r\n\=offset_limit=1,ps
+    ab\r\n\=offset_limit=1,ph
+
+/(?:)/firstline,use_offset_limit,newline=lf,no_start_optimize
+    abc\=offset=2,offset_limit=1
+    abc\=offset=2,offset_limit=2
+    abc\=offset=3,offset_limit=3
+    \=offset_limit=0
+
+/(?:)/firstline,use_offset_limit,newline=crlf,no_start_optimize
+    abc\=offset=2,offset_limit=1
+    abc\=offset=2,offset_limit=2
+    abc\=offset=3,offset_limit=3
+    \=offset_limit=0
+
+# CRLF at the subject end, adjacent lone CRs, and a shortened scan.
+
+/\r\n\r?/firstline,use_offset_limit,newline=crlf
+    a\r\n\=offset_limit=0
+    a\r\n\=offset_limit=1
+    a\r\n\=offset_limit=2
+    a\r\n\=offset_limit=3
+    a\r\n\r\=offset_limit=0
+    a\r\n\r\=offset_limit=1
+    a\r\n\r\=offset_limit=2
+    a\r\n\r\=offset_limit=4
+    a\r\r\n\=offset_limit=1
+    a\r\r\n\=offset_limit=2
+    a\r\r\n\=offset_limit=3
+    a\r\r\n\=offset_limit=4
+    a\r\n\rxxxxxxxx\=offset_limit=0
+    a\r\n\rxxxxxxxx\=offset_limit=1
+    a\r\n\rxxxxxxxx\=offset_limit=2
+
+/\n/firstline,use_offset_limit,newline=crlf
+    a\r\n\=offset_limit=1
+    a\r\n\=offset_limit=2
+    a\r\n\=offset_limit=3
+    a\r\n\=offset=2,offset_limit=1
+    a\r\n\=offset=2,offset_limit=2
+    a\r\n\=offset=2,offset_limit=3
+
+/\r\z/firstline,use_offset_limit,newline=crlf,no_start_optimize
+    a\r\=offset_limit=0
+    a\r\=offset_limit=1
+    a\r\=offset_limit=2
+    a\r\n\r\=offset_limit=2
+    a\r\n\r\=offset_limit=3
+    a\r\n\r\=offset_limit=4
+    a\r\n\r\=offset=3,offset_limit=2
+    a\r\n\r\=offset=3,offset_limit=3
+    a\r\n\r\=offset=3,offset_limit=4
+
 /abcd/null_context
     abcd\=null_context
 \= Expect not allowed together

testdata/testinput5

@@ -412,6 +412,28 @@
 /X/newline=any,utf,firstline
     A\x{1ec5}ABCXYZ
 
+# FIRSTLINE scans may stop inside a multicode-unit character.
+
+/A.*Z/newline=any,utf,firstline,use_offset_limit,no_start_optimize
+    A\x{1ec5}Zxxxxxxxx\=offset_limit=0
+    A\x{10000}Zxxxxxxxx\=offset_limit=1
+    A\x{85}Zxxxxxxxx\=offset_limit=1
+    A\x{2028}Zxxxxxxxx\=offset_limit=2
+
+/Z/newline=any,utf,firstline,use_offset_limit,no_start_optimize
+    A\x{1ec5}Zxxxxxxxx\=offset_limit=1
+    A\x{10000}Zxxxxxxxx\=offset_limit=1
+    A\x{85}Zxxxxxxxx\=offset_limit=2
+    A\x{2028}Zxxxxxxxx\=offset_limit=3
+
+/A.*Z/newline=any,utf,firstline,use_offset_limit,no_start_optimize,match_invalid_utf
+    A\x{1ec5}Zxxxxxxxx\=offset_limit=1
+    A\x{10000}Zxxxxxxxx\=offset_limit=1
+
+/Z/newline=any,utf,firstline,use_offset_limit,no_start_optimize,match_invalid_utf
+    A\x{85}Zxxxxxxxx\=offset_limit=2
+    A\x{2028}Zxxxxxxxx\=offset_limit=3
+
 /Xa{2,4}b/utf
     X\=ps
     Xa\=ps

testdata/testoutput10

@@ -1981,6 +1981,42 @@ Partial match:
     line0\nline1\nbreak
 No match
 
+# FIRSTLINE offset limits with stray and truncated UTF-8 around CRLF.
+
+/\r\n\r?/firstline,use_offset_limit,newline=crlf,match_invalid_utf,no_start_optimize
+    a\x80\r\n\=offset_limit=1
+No match
+    a\x80\r\n\=offset_limit=2
+ 0: \x{0d}\x{0a}
+    a\x80\r\n\=offset_limit=3
+ 0: \x{0d}\x{0a}
+    a\xe2\x80\r\n\r\=offset_limit=2
+No match
+    a\xe2\x80\r\n\r\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+    a\xe2\x80\r\n\r\=offset_limit=4
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x80\r\n\rxxxxxxxx\=offset_limit=1
+No match
+    a\x80\r\n\rxxxxxxxx\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x80\r\n\rxxxxxxxx\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+    a\r\n\r\xe2\x80\=offset_limit=0
+No match
+    a\r\n\r\xe2\x80\=offset_limit=1
+ 0: \x{0d}\x{0a}\x{0d}
+    a\r\n\r\xe2\x80\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+
+/\r\n\r?/firstline,use_offset_limit,newline=any,match_invalid_utf,no_start_optimize
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=2
+No match
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+    a\xe2\x80\r\n\rxxxxxxxx\=offset_limit=4
+ 0: \x{0d}\x{0a}\x{0d}
+
 /A\z/utf,match_invalid_utf
     A\x80\x42\n
 No match

testdata/testoutput12-16

@@ -1625,6 +1625,42 @@ No match
     \x{d800}aa
  0: aa
 
+# FIRSTLINE offset limits with unpaired surrogates around CRLF.
+
+/\r\n\r?/firstline,use_offset_limit,newline=crlf,match_invalid_utf,no_start_optimize
+    a\x{dc00}\r\n\=offset_limit=1
+No match
+    a\x{dc00}\r\n\=offset_limit=2
+ 0: \x{0d}\x{0a}
+    a\x{dc00}\r\n\=offset_limit=3
+ 0: \x{0d}\x{0a}
+    a\x{d800}\r\n\r\=offset_limit=1
+No match
+    a\x{d800}\r\n\r\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x{d800}\r\n\r\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=1
+No match
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x{dc00}\r\n\rxxxxxxxx\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+    a\r\n\r\x{d800}\=offset_limit=0
+No match
+    a\r\n\r\x{d800}\=offset_limit=1
+ 0: \x{0d}\x{0a}\x{0d}
+    a\r\n\r\x{d800}\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+
+/\r\n\r?/firstline,use_offset_limit,newline=any,match_invalid_utf,no_start_optimize
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=1
+No match
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=2
+ 0: \x{0d}\x{0a}\x{0d}
+    a\x{d800}\r\n\rxxxxxxxx\=offset_limit=3
+ 0: \x{0d}\x{0a}\x{0d}
+
 /A\z/utf,match_invalid_utf
     A\x{df00}\n
 No match