From 1d360d36c5a5f241bccc6336f1da0db952e567d5 Mon Sep 17 00:00:00 2001 From: Nicholas Wilson Date: Thu, 13 Aug 2026 13:07:17 +0100 Subject: [PATCH] Switch over RunGrepTest.bat to use more robust helpers --- RunGrepTest.bat | 56 ++++++++---- src/pcre2test.c | 224 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 18 deletions(-) diff --git a/RunGrepTest.bat b/RunGrepTest.bat index 5d0f12f06..a3ab240a5 100644 --- a/RunGrepTest.bat +++ b/RunGrepTest.bat @@ -81,16 +81,32 @@ if NOT "%nl%" == "LF" if NOT "%nl%" == "ANY" if NOT "%nl%" == "ANYCRLF" ( echo Default newline setting forced to LF ) -:: Create a simple printf via cscript/JScript (an actual printf may translate -:: LF to CRLF, which this one does not). We only support the barebones we need: -:: \r, \n, \0, and %s (but only once). +:: Provide "printf" and "tr" via pcre2test. Windows has no dependable version +:: of either: the obvious candidates translate what they write according to the +:: current code page, which mangles any byte greater than 0x7f, and an actual +:: printf may turn LF into CRLF. pcre2test has undocumented options that do just +:: as much as these tests need, so that this script can produce exactly the same +:: bytes as RunGrepTest does with the real printf and tr. +:: +:: %printf% supports \r, \n, \\, an octal escape such as \0 or \200, and one +:: %%s substitution; any other escape is left alone, so a literal backslash +:: has to be written as \\. %tr% changes one byte value into another as it +:: copies its input. -echo WScript.StdOut.Write(WScript.Arguments(0).replace(/\\r/g, "\r").replace(/\\n/g, "\n").replace(/\\0/g, "\x00").replace(/%%s/g, function() { return WScript.Arguments(1) })) >printf.js -set printf=cscript //nologo printf.js +set printf=%pcre2test% -printf +set tr=%pcre2test% -tr -:: Create a simple 'tr' via cscript/JScript. -echo WScript.StdOut.Write(WScript.StdIn.ReadAll().replace(/\x00/g, "@")) >trnull.js -set trnull=cscript //nologo trnull.js +:: pcre2grep ends the lines it writes with CRLF on Windows and with LF +:: elsewhere, which comparison copes with, but a line whose own text ends with +:: CR then comes out with one CR too many. %fixcrlf% removes it, for tests that +:: match a CR at the end of a line. + +set fixcrlf=%pcre2test% -fixcrlf + +:: The script callout tests run printf as a callout, where the program name +:: cannot be quoted, so keep an unquoted version of the path for them. + +set pcre2test_unquoted=%pcre2test:"=% :: ------ Normal tests ------ @@ -707,11 +723,11 @@ echo ---------------------------- Test 134 ----------------------------->>testtr echo RC=^%ERRORLEVEL%>>testtrygrep echo ---------------------------- Test 135 ----------------------------->>testtrygrep -(pushd %srcdir% & %pcre2grep% -HZ "word" ./testdata/grepinputv & popd) | %trnull% >>testtrygrep +(pushd %srcdir% & %pcre2grep% -HZ "word" ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep -(pushd %srcdir% & %pcre2grep% -lZ "word" ./testdata/grepinputv ./testdata/grepinputv & popd) | %trnull% >>testtrygrep +(pushd %srcdir% & %pcre2grep% -lZ "word" ./testdata/grepinputv ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep -(pushd %srcdir% & %pcre2grep% -A 1 -B 1 -HZ "word" ./testdata/grepinputv & popd) | %trnull% >>testtrygrep +(pushd %srcdir% & %pcre2grep% -A 1 -B 1 -HZ "word" ./testdata/grepinputv & popd) | %tr% \0 @ >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep (pushd %srcdir% & %pcre2grep% -MHZn "start[\s]+end" testdata/grepinputM & popd) >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep @@ -741,7 +757,7 @@ echo ---------------------------- Test 140 ----------------------------->>testtr echo RC=^%ERRORLEVEL%>>testtrygrep echo ---------------------------- Test 141 ----------------------------->>testtrygrep -%printf% "%%s\testdata\grepinputv\n-\n" "%srcdir%" >testtemp1grep +%printf% "%%s\\testdata\\grepinputv\n-\n" "%srcdir%" >testtemp1grep %printf% "This is a line from stdin." >testtemp2grep %pcre2grep% --file-list testtemp1grep "line from stdin" >testtrygrep 2>&1 echo RC=^%ERRORLEVEL%>>testtrygrep @@ -962,7 +978,11 @@ echo RC=^%ERRORLEVEL%>>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep echo ---------------------------- Test N3 ------------------------------>>testtrygrep -for /f %%a in ('%printf% "def\rjkl"') do set pattern=%%a +@REM The pattern contains a CR, which for /f keeps, but the command has to be +@REM read from a file: %printf% starts with a quoted path, and cmd mangles the +@REM quoting of a command given directly to for /f. +%printf% "def\rjkl" >testtemp1grep +for /f %%a in (testtemp1grep) do set pattern=%%a %pcre2grep% -n --newline=cr -F "!pattern!" testNinputgrep >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep @@ -984,9 +1004,9 @@ echo RC=^%ERRORLEVEL%>>testtrygrep echo ---------------------------- Test N7 ------------------------------>>testtrygrep %printf% "xyz\0abc\0def" >testNinputgrep -%pcre2grep% -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep +%pcre2grep% -na --newline=nul "^(abc|def)" testNinputgrep | %tr% \0 @ >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep -%pcre2grep% -B1 -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep +%pcre2grep% -B1 -na --newline=nul "^(abc|def)" testNinputgrep | %tr% \0 @ >>testtrygrep echo RC=^%ERRORLEVEL%>>testtrygrep echo ---------------------------- Test N8 ------------------------------>>testtrygrep @@ -1041,7 +1061,7 @@ if %ERRORLEVEL% equ 0 ( %pcre2grep% "(T)(?C'|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep echo RC=^!ERRORLEVEL!>>testtrygrep echo --- Test 4 --->>testtrygrep - %pcre2grep% "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep + %pcre2grep% "(T)(?C'%pcre2test_unquoted%|-printf|%%s\r\n|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep echo RC=^!ERRORLEVEL!>>testtrygrep echo --- Test 5 --->>testtrygrep %pcre2grep% "(T)(?C'|$1$n')(*F)" %srcdir%\testdata\grepinputv >>testtrygrep @@ -1069,7 +1089,7 @@ if %ERRORLEVEL% equ 0 ( %pcre2grep% -u "(T)(?C'|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep echo RC=^!ERRORLEVEL!>>testtrygrep echo --- Test 2 --->>testtrygrep - %pcre2grep% -u "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep + %pcre2grep% -u "(T)(?C'%pcre2test_unquoted%|-printf|%%s\r\n|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep echo RC=^!ERRORLEVEL!>>testtrygrep if ^!nonfork! equ 1 ( @@ -1106,7 +1126,7 @@ call :checkspecial "-e (unpaired1 -e (unpaired2 nul" 2 || exit /b 1 :: Clean up local working files -del testcf printf.js trnull.js testNinputgrep teststderrgrep testtrygrep testtemp1grep testtemp2grep -testtemp1grep -- +del testcf testNinputgrep teststderrgrep testtrygrep testtemp1grep testtemp2grep -testtemp1grep -- exit /b 0 diff --git a/src/pcre2test.c b/src/pcre2test.c index 507e472c8..e8b6b14d8 100644 --- a/src/pcre2test.c +++ b/src/pcre2test.c @@ -3646,6 +3646,218 @@ display_selected_modifiers(FALSE, "SUBJECT"); +/************************************************* +* Helpers for the grep test scripts * +*************************************************/ + +/* RunGrepTest and RunGrepTest.bat have to create input files that contain +exact byte sequences, including bytes that cannot start a UTF-8 character, and +they also have to filter a stream, changing one byte value into another. On +Unix-like systems printf(1) and tr(1) do this, but Windows has no dependable +equivalent: the obvious candidates all translate what they write according to +the current code page, which mangles any byte greater than 0x7f. As both +scripts are checked against the same expected output, they need the same +facilities, so these options provide just enough of printf and tr for +RunGrepTest.bat to produce exactly the same bytes as RunGrepTest does. + +They are needed only by RunGrepTest.bat, so they are built only on Windows. +They are deliberately undocumented, are not part of pcre2test's interface, and +may change whenever the test scripts need something different. */ + +#if defined(_WIN32) || defined(WIN32) + +/* Standard input and output are put into binary mode, because the Windows +default is to turn every LF that passes through into CRLF. */ + +static void +helper_binary_io(void) +{ +_setmode(_fileno(stdin), _O_BINARY); +_setmode(_fileno(stdout), _O_BINARY); +} + + +/* Decode an argument that represents a single byte, either as itself or as a +backslash followed by up to three octal digits. The result is the byte value, +or -1 if the argument is not one of those things. */ + +static int +helper_byte(const char *s) +{ +if (s[0] == '\\' && s[1] >= '0' && s[1] <= '7') + { + int value = 0; + int i = 0; + int n = 0; + while (n < 3 && s[i+1] >= '0' && s[i+1] <= '7') + { + value = value * 8 + (s[++i] - '0'); + n++; + } + return (s[i+1] == 0 && value <= 255)? value : -1; + } + +return (s[0] != 0 && s[1] == 0)? (unsigned char)s[0] : -1; +} + + +/* Write the format argument to the standard output, interpreting the escape +sequences that the test scripts use, and replacing every occurrence of %s with +the second argument. The escapes are \n, \r, \\ and an octal escape of up to +three digits. Anything else is written out unchanged, backslash and all, but +the scripts should not rely on that: a literal backslash, as in a Windows file +name, is written as \\ so that the format does not depend on which escapes +happen to be implemented here. */ + +static int +helper_printf(int argc, char **argv) +{ +const char *fmt; +const char *arg; +size_t i; + +if (argc < 1) + { + fprintf(stderr, "pcre2test: -printf needs a format argument\n"); + return 1; + } + +fmt = argv[0]; +arg = (argc > 1)? argv[1] : ""; + +helper_binary_io(); + +for (i = 0; fmt[i] != 0; i++) + { + if (fmt[i] == '%' && fmt[i+1] == 's') + { + fputs(arg, stdout); + i++; + } + + /* An octal escape of up to three digits, so that \0 gives a binary zero and + \200 gives a byte that cannot start a UTF-8 character. */ + + else if (fmt[i] == '\\' && fmt[i+1] >= '0' && fmt[i+1] <= '7') + { + int value = 0; + int n = 0; + while (n < 3 && fmt[i+1] >= '0' && fmt[i+1] <= '7') + { + value = value * 8 + (fmt[++i] - '0'); + n++; + } + if (value > 255) + { + fprintf(stderr, "pcre2test: -printf octal escape is greater than \\377\n"); + return 1; + } + fputc(value, stdout); + } + + else if (fmt[i] == '\\' && fmt[i+1] == 'n') { fputc('\n', stdout); i++; } + else if (fmt[i] == '\\' && fmt[i+1] == 'r') { fputc('\r', stdout); i++; } + else if (fmt[i] == '\\' && fmt[i+1] == '\\') { fputc('\\', stdout); i++; } + else fputc(fmt[i], stdout); + } + +if (fflush(stdout) != 0) + { + fprintf(stderr, "pcre2test: -printf write failed: %s\n", strerror(errno)); + return 1; + } + +return 0; +} + + +/* Copy the standard input to the standard output, changing every occurrence of +one byte value into another. Both bytes are given in the form that helper_byte() +accepts. */ + +static int +helper_tr(int argc, char **argv) +{ +int from, to, c; + +if (argc < 2) + { + fprintf(stderr, "pcre2test: -tr needs two arguments\n"); + return 1; + } + +from = helper_byte(argv[0]); +to = helper_byte(argv[1]); + +if (from < 0 || to < 0) + { + fprintf(stderr, "pcre2test: -tr arguments must each be a single byte\n"); + return 1; + } + +helper_binary_io(); + +while ((c = getchar()) != EOF) putchar((c == from)? to : c); + +if (ferror(stdin) || fflush(stdout) != 0) + { + fprintf(stderr, "pcre2test: -tr copy failed: %s\n", strerror(errno)); + return 1; + } + +return 0; +} + + + +/* Copy the standard input to the standard output, changing every occurrence of +the three bytes CR, CR, LF into the two bytes CR, LF. + +pcre2grep ends each line that it writes with STDOUT_NL, which is CRLF on +Windows but LF elsewhere, so a line whose own text ends with CR comes out as +CR CR LF on Windows and as CR LF everywhere else. Comparison normally copes +with the difference between the two line endings, but not with the extra CR +that this case produces. Both test scripts are checked against the same +expected output, so RunGrepTest.bat passes the output of the few tests that +match a CR at the end of a line through this, and leaves everything else alone. +A run of CRs that is not at the end of a line is not touched, so a CR that the +pattern matched in the middle of a line still shows up. */ + +static int +helper_fixcrlf(void) +{ +unsigned long crcount = 0; +int c; + +helper_binary_io(); + +while ((c = getchar()) != EOF) + { + if (c == '\r') { crcount++; continue; } + + /* A run of CRs that ends the line has one CR removed; any other run is + written out unchanged. */ + + if (c == '\n' && crcount >= 2) crcount--; + while (crcount > 0) { putchar('\r'); crcount--; } + putchar(c); + } + +while (crcount > 0) { putchar('\r'); crcount--; } + +if (ferror(stdin) || fflush(stdout) != 0) + { + fprintf(stderr, "pcre2test: -fixcrlf copy failed: %s\n", strerror(errno)); + return 1; + } + +return 0; +} + +#endif /* Windows */ + + + /************************************************* * Main Program * *************************************************/ @@ -3665,6 +3877,18 @@ char *arg_pattern = NULL; char *arg_error = NULL; char *env_no_color = getenv("NO_COLOR"); +/* The undocumented helpers for the grep test scripts do none of the work that +follows, so deal with them before anything is set up. */ + +#if defined(_WIN32) || defined(WIN32) +if (argc > 2 && strcmp(argv[1], "-printf") == 0) + return helper_printf(argc - 2, argv + 2); +if (argc > 2 && strcmp(argv[1], "-tr") == 0) + return helper_tr(argc - 2, argv + 2); +if (argc == 2 && strcmp(argv[1], "-fixcrlf") == 0) + return helper_fixcrlf(); +#endif + /* Get buffers from malloc() so that valgrind will check their misuse when debugging. They grow automatically when very long lines are read. The 16- and 32-bit buffers (pbuffer16, pbuffer32) are obtained only if needed. */