Switch over RunGrepTest.bat to use more robust helpers

RunGrepTest.bat

@@ -81,16 +81,32 @@ if NOT "%nl%" == "LF" if NOT "%nl%" == "ANY" if NOT "%nl%" == "ANYCRLF" (
   echo Default newline setting forced to LF
 )
 
-:: Create a simple printf via cscript/JScript (an actual printf may translate
-:: LF to CRLF, which this one does not).  We only support the barebones we need:
-:: \r, \n, \0, and %s (but only once).
+:: Provide "printf" and "tr" via pcre2test. Windows has no dependable version
+:: of either: the obvious candidates translate what they write according to the
+:: current code page, which mangles any byte greater than 0x7f, and an actual
+:: printf may turn LF into CRLF. pcre2test has undocumented options that do just
+:: as much as these tests need, so that this script can produce exactly the same
+:: bytes as RunGrepTest does with the real printf and tr.
+::
+:: %printf% supports \r, \n, \\, an octal escape such as \0 or \200, and one
+:: %%s substitution; any other escape is left alone, so a literal backslash
+:: has to be written as \\. %tr% changes one byte value into another as it
+:: copies its input.
 
-echo WScript.StdOut.Write(WScript.Arguments(0).replace(/\\r/g, "\r").replace(/\\n/g, "\n").replace(/\\0/g, "\x00").replace(/%%s/g, function() { return WScript.Arguments(1) })) >printf.js
-set printf=cscript //nologo printf.js
+set printf=%pcre2test% -printf
+set tr=%pcre2test% -tr
 
-:: Create a simple 'tr' via cscript/JScript.
-echo WScript.StdOut.Write(WScript.StdIn.ReadAll().replace(/\x00/g, "@")) >trnull.js
-set trnull=cscript //nologo trnull.js
+:: pcre2grep ends the lines it writes with CRLF on Windows and with LF
+:: elsewhere, which comparison copes with, but a line whose own text ends with
+:: CR then comes out with one CR too many. %fixcrlf% removes it, for tests that
+:: match a CR at the end of a line.
+
+set fixcrlf=%pcre2test% -fixcrlf
+
+:: The script callout tests run printf as a callout, where the program name
+:: cannot be quoted, so keep an unquoted version of the path for them.
+
+set pcre2test_unquoted=%pcre2test:"=%
 
 :: ------ Normal tests ------
 
@@ -707,11 +723,11 @@ echo ---------------------------- Test 134 ----------------------------->>testtr
 echo RC=^%ERRORLEVEL%>>testtrygrep
 
 echo ---------------------------- Test 135 ----------------------------->>testtrygrep
-(pushd %srcdir% & %pcre2grep% -HZ "word" ./testdata/grepinputv & popd) | %trnull% >>testtrygrep
+(pushd %srcdir% & %pcre2grep% -HZ "word" ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
-(pushd %srcdir% & %pcre2grep% -lZ "word" ./testdata/grepinputv ./testdata/grepinputv & popd) | %trnull% >>testtrygrep
+(pushd %srcdir% & %pcre2grep% -lZ "word" ./testdata/grepinputv ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
-(pushd %srcdir% & %pcre2grep% -A 1 -B 1 -HZ "word" ./testdata/grepinputv & popd) | %trnull% >>testtrygrep
+(pushd %srcdir% & %pcre2grep% -A 1 -B 1 -HZ "word" ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
 (pushd %srcdir% & %pcre2grep% -MHZn "start[\s]+end" testdata/grepinputM & popd) >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
@@ -741,7 +757,7 @@ echo ---------------------------- Test 140 ----------------------------->>testtr
 echo RC=^%ERRORLEVEL%>>testtrygrep
 
 echo ---------------------------- Test 141 ----------------------------->>testtrygrep
-%printf% "%%s\testdata\grepinputv\n-\n" "%srcdir%" >testtemp1grep
+%printf% "%%s\\testdata\\grepinputv\n-\n" "%srcdir%" >testtemp1grep
 %printf% "This is a line from stdin." >testtemp2grep
 %pcre2grep% --file-list testtemp1grep "line from stdin" <testtemp2grep >>testtrygrep 2>&1
 echo RC=^%ERRORLEVEL%>>testtrygrep
@@ -962,7 +978,11 @@ echo RC=^%ERRORLEVEL%>>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
 
 echo ---------------------------- Test N3 ------------------------------>>testtrygrep
-for /f %%a in ('%printf% "def\rjkl"') do set pattern=%%a
+@REM The pattern contains a CR, which for /f keeps, but the command has to be
+@REM read from a file: %printf% starts with a quoted path, and cmd mangles the
+@REM quoting of a command given directly to for /f.
+%printf% "def\rjkl" >testtemp1grep
+for /f %%a in (testtemp1grep) do set pattern=%%a
 %pcre2grep% -n --newline=cr -F "!pattern!" testNinputgrep >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
 
@@ -984,9 +1004,9 @@ echo RC=^%ERRORLEVEL%>>testtrygrep
 
 echo ---------------------------- Test N7 ------------------------------>>testtrygrep
 %printf% "xyz\0abc\0def" >testNinputgrep
-%pcre2grep% -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep
+%pcre2grep% -na --newline=nul "^(abc|def)" testNinputgrep | %tr% \0 @ >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
-%pcre2grep% -B1 -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep
+%pcre2grep% -B1 -na --newline=nul "^(abc|def)" testNinputgrep | %tr% \0 @ >>testtrygrep
 echo RC=^%ERRORLEVEL%>>testtrygrep
 
 echo ---------------------------- Test N8 ------------------------------>>testtrygrep
@@ -1041,7 +1061,7 @@ if %ERRORLEVEL% equ 0 (
   %pcre2grep% "(T)(?C'|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep
   echo RC=^!ERRORLEVEL!>>testtrygrep
   echo --- Test 4 --->>testtrygrep
-  %pcre2grep% "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep
+  %pcre2grep% "(T)(?C'%pcre2test_unquoted%|-printf|%%s\r\n|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep
   echo RC=^!ERRORLEVEL!>>testtrygrep
   echo --- Test 5 --->>testtrygrep
   %pcre2grep% "(T)(?C'|$1$n')(*F)" %srcdir%\testdata\grepinputv >>testtrygrep
@@ -1069,7 +1089,7 @@ if %ERRORLEVEL% equ 0 (
     %pcre2grep% -u "(T)(?C'|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
     echo RC=^!ERRORLEVEL!>>testtrygrep
     echo --- Test 2 --->>testtrygrep
-    %pcre2grep% -u "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
+    %pcre2grep% -u "(T)(?C'%pcre2test_unquoted%|-printf|%%s\r\n|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
     echo RC=^!ERRORLEVEL!>>testtrygrep
 
     if ^!nonfork! equ 1 (
@@ -1106,7 +1126,7 @@ call :checkspecial "-e (unpaired1 -e (unpaired2 nul" 2 || exit /b 1
 
 
 :: Clean up local working files
-del testcf printf.js trnull.js testNinputgrep teststderrgrep testtrygrep testtemp1grep testtemp2grep -testtemp1grep --
+del testcf testNinputgrep teststderrgrep testtrygrep testtemp1grep testtemp2grep -testtemp1grep --
 
 exit /b 0
 

src/pcre2test.c

@@ -3646,6 +3646,218 @@ display_selected_modifiers(FALSE, "SUBJECT");
 
 
 
+/*************************************************
+*        Helpers for the grep test scripts       *
+*************************************************/
+
+/* RunGrepTest and RunGrepTest.bat have to create input files that contain
+exact byte sequences, including bytes that cannot start a UTF-8 character, and
+they also have to filter a stream, changing one byte value into another. On
+Unix-like systems printf(1) and tr(1) do this, but Windows has no dependable
+equivalent: the obvious candidates all translate what they write according to
+the current code page, which mangles any byte greater than 0x7f. As both
+scripts are checked against the same expected output, they need the same
+facilities, so these options provide just enough of printf and tr for
+RunGrepTest.bat to produce exactly the same bytes as RunGrepTest does.
+
+They are needed only by RunGrepTest.bat, so they are built only on Windows.
+They are deliberately undocumented, are not part of pcre2test's interface, and
+may change whenever the test scripts need something different. */
+
+#if defined(_WIN32) || defined(WIN32)
+
+/* Standard input and output are put into binary mode, because the Windows
+default is to turn every LF that passes through into CRLF. */
+
+static void
+helper_binary_io(void)
+{
+_setmode(_fileno(stdin), _O_BINARY);
+_setmode(_fileno(stdout), _O_BINARY);
+}
+
+
+/* Decode an argument that represents a single byte, either as itself or as a
+backslash followed by up to three octal digits. The result is the byte value,
+or -1 if the argument is not one of those things. */
+
+static int
+helper_byte(const char *s)
+{
+if (s[0] == '\\' && s[1] >= '0' && s[1] <= '7')
+  {
+  int value = 0;
+  int i = 0;
+  int n = 0;
+  while (n < 3 && s[i+1] >= '0' && s[i+1] <= '7')
+    {
+    value = value * 8 + (s[++i] - '0');
+    n++;
+    }
+  return (s[i+1] == 0 && value <= 255)? value : -1;
+  }
+
+return (s[0] != 0 && s[1] == 0)? (unsigned char)s[0] : -1;
+}
+
+
+/* Write the format argument to the standard output, interpreting the escape
+sequences that the test scripts use, and replacing every occurrence of %s with
+the second argument. The escapes are \n, \r, \\ and an octal escape of up to
+three digits. Anything else is written out unchanged, backslash and all, but
+the scripts should not rely on that: a literal backslash, as in a Windows file
+name, is written as \\ so that the format does not depend on which escapes
+happen to be implemented here. */
+
+static int
+helper_printf(int argc, char **argv)
+{
+const char *fmt;
+const char *arg;
+size_t i;
+
+if (argc < 1)
+  {
+  fprintf(stderr, "pcre2test: -printf needs a format argument\n");
+  return 1;
+  }
+
+fmt = argv[0];
+arg = (argc > 1)? argv[1] : "";
+
+helper_binary_io();
+
+for (i = 0; fmt[i] != 0; i++)
+  {
+  if (fmt[i] == '%' && fmt[i+1] == 's')
+    {
+    fputs(arg, stdout);
+    i++;
+    }
+
+  /* An octal escape of up to three digits, so that \0 gives a binary zero and
+  \200 gives a byte that cannot start a UTF-8 character. */
+
+  else if (fmt[i] == '\\' && fmt[i+1] >= '0' && fmt[i+1] <= '7')
+    {
+    int value = 0;
+    int n = 0;
+    while (n < 3 && fmt[i+1] >= '0' && fmt[i+1] <= '7')
+      {
+      value = value * 8 + (fmt[++i] - '0');
+      n++;
+      }
+    if (value > 255)
+      {
+      fprintf(stderr, "pcre2test: -printf octal escape is greater than \\377\n");
+      return 1;
+      }
+    fputc(value, stdout);
+    }
+
+  else if (fmt[i] == '\\' && fmt[i+1] == 'n') { fputc('\n', stdout); i++; }
+  else if (fmt[i] == '\\' && fmt[i+1] == 'r') { fputc('\r', stdout); i++; }
+  else if (fmt[i] == '\\' && fmt[i+1] == '\\') { fputc('\\', stdout); i++; }
+  else fputc(fmt[i], stdout);
+  }
+
+if (fflush(stdout) != 0)
+  {
+  fprintf(stderr, "pcre2test: -printf write failed: %s\n", strerror(errno));
+  return 1;
+  }
+
+return 0;
+}
+
+
+/* Copy the standard input to the standard output, changing every occurrence of
+one byte value into another. Both bytes are given in the form that helper_byte()
+accepts. */
+
+static int
+helper_tr(int argc, char **argv)
+{
+int from, to, c;
+
+if (argc < 2)
+  {
+  fprintf(stderr, "pcre2test: -tr needs two arguments\n");
+  return 1;
+  }
+
+from = helper_byte(argv[0]);
+to = helper_byte(argv[1]);
+
+if (from < 0 || to < 0)
+  {
+  fprintf(stderr, "pcre2test: -tr arguments must each be a single byte\n");
+  return 1;
+  }
+
+helper_binary_io();
+
+while ((c = getchar()) != EOF) putchar((c == from)? to : c);
+
+if (ferror(stdin) || fflush(stdout) != 0)
+  {
+  fprintf(stderr, "pcre2test: -tr copy failed: %s\n", strerror(errno));
+  return 1;
+  }
+
+return 0;
+}
+
+
+
+/* Copy the standard input to the standard output, changing every occurrence of
+the three bytes CR, CR, LF into the two bytes CR, LF.
+
+pcre2grep ends each line that it writes with STDOUT_NL, which is CRLF on
+Windows but LF elsewhere, so a line whose own text ends with CR comes out as
+CR CR LF on Windows and as CR LF everywhere else. Comparison normally copes
+with the difference between the two line endings, but not with the extra CR
+that this case produces. Both test scripts are checked against the same
+expected output, so RunGrepTest.bat passes the output of the few tests that
+match a CR at the end of a line through this, and leaves everything else alone.
+A run of CRs that is not at the end of a line is not touched, so a CR that the
+pattern matched in the middle of a line still shows up. */
+
+static int
+helper_fixcrlf(void)
+{
+unsigned long crcount = 0;
+int c;
+
+helper_binary_io();
+
+while ((c = getchar()) != EOF)
+  {
+  if (c == '\r') { crcount++; continue; }
+
+  /* A run of CRs that ends the line has one CR removed; any other run is
+  written out unchanged. */
+
+  if (c == '\n' && crcount >= 2) crcount--;
+  while (crcount > 0) { putchar('\r'); crcount--; }
+  putchar(c);
+  }
+
+while (crcount > 0) { putchar('\r'); crcount--; }
+
+if (ferror(stdin) || fflush(stdout) != 0)
+  {
+  fprintf(stderr, "pcre2test: -fixcrlf copy failed: %s\n", strerror(errno));
+  return 1;
+  }
+
+return 0;
+}
+
+#endif  /* Windows */
+
+
+
 /*************************************************
 *                Main Program                    *
 *************************************************/
@@ -3665,6 +3877,18 @@ char *arg_pattern = NULL;
 char *arg_error = NULL;
 char *env_no_color = getenv("NO_COLOR");
 
+/* The undocumented helpers for the grep test scripts do none of the work that
+follows, so deal with them before anything is set up. */
+
+#if defined(_WIN32) || defined(WIN32)
+if (argc > 2 && strcmp(argv[1], "-printf") == 0)
+  return helper_printf(argc - 2, argv + 2);
+if (argc > 2 && strcmp(argv[1], "-tr") == 0)
+  return helper_tr(argc - 2, argv + 2);
+if (argc == 2 && strcmp(argv[1], "-fixcrlf") == 0)
+  return helper_fixcrlf();
+#endif
+
 /* Get buffers from malloc() so that valgrind will check their misuse when
 debugging. They grow automatically when very long lines are read. The 16-
 and 32-bit buffers (pbuffer16, pbuffer32) are obtained only if needed. */