From 697ebee8098b07e90dd4a496fe3db4e680f83c5d Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 16 Jul 2026 00:46:32 +0200 Subject: [PATCH 01/11] feat: add support for BEGINFILE and ENDFILE rules (#457) Implement gawk's BEGINFILE/ENDFILE special patterns, the nextfile statement, and the ERRNO and ARGIND special variables: - Parser: BEGINFILE/ENDFILE/nextfile keywords, per-file input-loop tuple layout (NEXT_FILE / CONSUME_FILE_INPUT scaffolding emitted only when these features are used, so existing programs keep the flat layout), and gawk-compatible compile-time restrictions (next rejected in BEGINFILE/ENDFILE, nextfile rejected in BEGIN/END/ENDFILE). - Runtime: per-file stepping in StreamInputSource with FILENAME, FNR=0, cleared $0, ARGIND, and ERRNO on open failure; open failures become non-fatal only when a BEGINFILE rule skips the file with nextfile; nextfile unwinds user-defined function calls like exit does; non-redirected getline is fatal inside BEGINFILE/ENDFILE. - New opcodes appended at the end of the Opcode enum to preserve precompiled tuple (-L) compatibility. - BEGINFILE/ENDFILE are not special in POSIX mode (--posix), as in gawk. - Lexer: a slash inside a regexp bracket expression (/[/]/) no longer terminates the literal, which beginfile1.awk requires; this also makes the vendored gawk regexpbrack test pass. - FNR now counts records read from standard input (POSIX: FNR is the record number in the current input file); custom InputSource implementations keep the documented isFromFilenameList() contract. - Tests: BeginFileEndFileTest (25 cases), re-enabled the POSIX nextfile conformance test, new stdin-FNR conformance tests, and wired the vendored beginfile1 fixture plus six beginfile2 sub-tests into GawkExtensionIT (beginfile1 matches beginfile1.ok byte for byte). - Docs: README, cli.md, cli-reference.md, extensions.md, java.md, java-input.md. Co-Authored-By: Claude Fable 5 --- README.md | 2 + src/it/java/io/jawk/gawk/GawkExtensionIT.java | 167 ++++++++- .../io/jawk/gawk/GawkOptionalFeatureIT.java | 5 +- src/main/java/io/jawk/Cli.java | 5 +- src/main/java/io/jawk/backend/AVM.java | 167 +++++++++ src/main/java/io/jawk/frontend/AwkParser.java | 344 +++++++++++++++-- .../java/io/jawk/intermediate/AwkTuples.java | 78 ++++ .../java/io/jawk/intermediate/Opcode.java | 93 ++++- src/main/java/io/jawk/jrt/InputSource.java | 6 + src/main/java/io/jawk/jrt/JRT.java | 158 +++++++- .../java/io/jawk/jrt/StreamInputSource.java | 181 +++++++++ src/main/java/io/jawk/util/AwkSettings.java | 7 +- src/site/markdown/cli-reference.md | 4 +- src/site/markdown/cli.md | 18 + src/site/markdown/extensions.md | 2 + src/site/markdown/java-input.md | 2 +- src/site/markdown/java.md | 4 +- .../java/io/jawk/BeginFileEndFileTest.java | 347 ++++++++++++++++++ src/test/java/io/jawk/InputSourceTest.java | 17 + .../java/io/jawk/PosixConformanceTest.java | 21 +- 20 files changed, 1586 insertions(+), 42 deletions(-) create mode 100644 src/test/java/io/jawk/BeginFileEndFileTest.java diff --git a/README.md b/README.md index d8101178..4f868d38 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Jawk is a pure Java implementation of [AWK](https://en.wikipedia.org/wiki/AWK). Gawk-specific builtins — `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, and `PROCINFO["sorted_in"]`-controlled array traversal — are available by default through the built-in GNU Awk compatibility extension. +Jawk also supports gawk's `BEGINFILE` / `ENDFILE` special patterns (with the `ERRNO` and `ARGIND` special variables) and the `nextfile` statement, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error. As in gawk, `BEGINFILE` and `ENDFILE` are not special in `--posix` mode. + ## CLI Example ```shell diff --git a/src/it/java/io/jawk/gawk/GawkExtensionIT.java b/src/it/java/io/jawk/gawk/GawkExtensionIT.java index 650d6ff4..aa12f7a2 100644 --- a/src/it/java/io/jawk/gawk/GawkExtensionIT.java +++ b/src/it/java/io/jawk/gawk/GawkExtensionIT.java @@ -22,6 +22,12 @@ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ */ +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import io.jawk.AwkTestSupport; import org.junit.Test; @@ -1553,12 +1559,169 @@ public void test_mixed1() throws Exception { @Test public void test_beginfile1() throws Exception { - skip("BEGINFILE and ENDFILE are not implemented by Jawk yet."); + // Upstream: $AWK -f beginfile1.awk "$(srcdir)"/beginfile1.awk . ./no/such/file Makefile + // The script strips directories from FILENAME with gsub(/.*[/]/, ""), + // so operands are passed with forward slashes. "Makefile" is not part + // of the vendored fixtures (it is generated by gawk's configure), so + // an equivalent two-record file is materialized on the fly. + Path makefile = AwkTestSupport.sharedTempDirectory().resolve("Makefile"); + Files.write(makefile, "# first record\n# second record\n".getBytes(StandardCharsets.UTF_8)); + AwkTestSupport + .cliTest("GAWK beginfile1") + .argument("-f", gawkFile("beginfile1.awk")) + .operand( + slashed(gawkPath("beginfile1.awk")), + ".", + "./no/such/file", + slashed(makefile)) + .expectLines(gawkPath("beginfile1.ok")) + .expectExit(0) + .runAndAssert(); } @Test public void test_beginfile2() throws Exception { - skip("BEGINFILE and ENDFILE are not implemented by Jawk yet."); + skip( + "Covered by the test_beginfile2_test* cases below; the remaining beginfile2.sh sub-tests rely on" + + " non-redirected getline in BEGIN/END driving the BEGINFILE/ENDFILE hooks, or compare" + + " non-zero gawk CLI transcripts, which Jawk does not reproduce."); + } + + @Test + public void test_beginfile2_test2() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 2--") + .argument(beginfile2Program(2)) + .operand(slashed(gawkPath("beginfile2.in")), "file/does/not/exist") + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("2")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test5() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 5--") + .argument(beginfile2Program(5)) + .operand(slashed(gawkPath("beginfile2.in"))) + .expectLines(beginfile2ExpectedLines("5")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test9a() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 9a--") + .argument(beginfile2Program(9)) + .operand("file/does/not/exist", slashed(gawkPath("beginfile2.in"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("9a")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test9b() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 9b--") + .preassign("skip", 1) + .argument(beginfile2Program(9)) + .operand("file/does/not/exist", slashed(gawkPath("beginfile2.in"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("9b")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test12() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 12--") + .argument(beginfile2Program(12)) + .operand(slashed(gawkPath("beginfile2.in")), slashed(gawkPath("beginfile2.sh"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("12")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test15() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 15--") + .argument(beginfile2Program(15)) + .operand(slashed(gawkPath("beginfile2.in")), slashed(gawkPath("beginfile2.sh"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("15")) + .expectExit(0) + .runAndAssert(); + } + + /** + * Extracts one sub-test program from the vendored {@code beginfile2.in}, + * exactly like {@code beginfile2.sh} does with its awk range patterns. + * The {@code #TESTn#} marker lines are comments and stay harmless. + * + * @param testNumber sub-test number + * @return the AWK program of that sub-test + */ + private static String beginfile2Program(int testNumber) throws IOException { + String text = gawkText("beginfile2.in"); + String beginMarker = "#TEST" + testNumber + "#"; + String endMarker = "#TEST" + (testNumber + 1) + "#"; + return text.substring(text.indexOf(beginMarker), text.indexOf(endMarker) + endMarker.length()); + } + + /** + * Extracts the expected transcript of one beginfile2 sub-test from the + * vendored {@code beginfile2.ok}, i.e. the lines between the + * {@code --Test N--} banner and the next banner. + * + * @param testLabel sub-test label as it appears in the banner (e.g. "9a") + * @return the expected output lines of that sub-test + */ + private static List beginfile2ExpectedLines(String testLabel) throws IOException { + List lines = Files.readAllLines(gawkPath("beginfile2.ok"), StandardCharsets.UTF_8); + List section = new ArrayList<>(); + boolean inSection = false; + for (String line : lines) { + if (line.startsWith("--Test ")) { + if (inSection) { + break; + } + inSection = line.equals("--Test " + testLabel + "--"); + continue; + } + if (inSection) { + section.add(line); + } + } + return section; + } + + /** + * Removes the vendored gawk directory prefix from FILENAME occurrences in + * the transcript, so it can be compared with the {@code .ok} files that + * gawk produces while running inside that directory. + * + * @param output raw transcript + * @return transcript with bare fixture file names + */ + private static String stripGawkDirectory(String output) { + return output.replace(slashed(GAWK_DIRECTORY) + "/", ""); + } + + /** + * Returns the path as a string with forward slashes, so FILENAME-based + * basename extraction with {@code gsub(/.*[/]/, "")} works on Windows too. + * + * @param path path to convert + * @return the path with forward slashes + */ + private static String slashed(Path path) { + return path.toString().replace('\\', '/'); } @Test diff --git a/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java b/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java index 076b4482..2afb5737 100644 --- a/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java +++ b/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java @@ -729,7 +729,10 @@ public void test_testext() throws Exception { @Test public void test_getfile() throws Exception { - skip(MANUAL_SKIP_REASON); + skip( + "Requires gawk's dynamic extension API (the get_file() input hand-off used by gawkextlib)," + + " which Jawk does not provide. The BEGINFILE/ENDFILE rules the fixture also exercises" + + " are covered by GawkExtensionIT.test_beginfile1 and test_beginfile2_test*."); } @Test diff --git a/src/main/java/io/jawk/Cli.java b/src/main/java/io/jawk/Cli.java index da0891a7..e56ba8ee 100644 --- a/src/main/java/io/jawk/Cli.java +++ b/src/main/java/io/jawk/Cli.java @@ -645,7 +645,10 @@ private static void usage(PrintStream dest) { .println( " -S, --sandbox = (extension) Enable sandbox mode (no system(), redirection, pipelines, or" + " dynamic extensions)."); - dest.println(" --posix = Enforce POSIX-compatible behavior such as disabling nested arrays."); + dest + .println( + " --posix = Enforce POSIX-compatible behavior such as disabling nested arrays and" + + " BEGINFILE/ENDFILE rules."); dest.println(" --dump-syntax = Print the syntax tree."); dest.println(" --dump-intermediate = Print the intermediate code."); dest.println(" -s, --no-optimize = (extension) Disable optimizations during compilation."); diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 27314790..791b4d1e 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -590,6 +590,37 @@ private void initExtensions() { private Address exitAddress = null; + /** + * Address of the ENDFILE section, registered by SET_ENDFILE_ADDRESS when + * the program requires per-file input stepping (BEGINFILE/ENDFILE rules + * or a {@code nextfile} statement); {@code null} otherwise. + */ + private Address endFileAddress = null; + + /** + * Address of the NEXT_FILE tuple, registered by SET_NEXTFILE_ADDRESS when + * the program requires per-file input stepping; {@code null} otherwise. + */ + private Address nextFileAddress = null; + + /** + * true if execution position is within a BEGINFILE rule; + * false otherwise. + */ + private boolean withinBeginFileBlocks = false; + + /** + * true if execution position is within an ENDFILE rule; + * false otherwise. + */ + private boolean withinEndFileBlocks = false; + + /** + * true once the per-file main input loop has advanced to its + * first input file; false while still in the BEGIN blocks. + */ + private boolean inputFileLoopStarted = false; + /** * true if execution position is within an END block; * false otherwise. @@ -678,6 +709,11 @@ private void resetTransientRuntimeState(List runtimeArguments, Map getFunctionNames() { "RSTART", "RLENGTH", "IGNORECASE", + "ERRNO", + "ARGIND", "ARGC", "ARGV"))); @@ -3294,6 +3402,10 @@ public final Object getVariable(String name) { return jrt.getRLENGTH(); case "IGNORECASE": return jrt.getIGNORECASEVar(); + case "ERRNO": + return jrt.getERRNO(); + case "ARGIND": + return jrt.getARGIND(); // lazily-materialized globals answered through their synthetic accessors case "ARGC": return getARGC(); @@ -3395,6 +3507,61 @@ public final void assignVariable(String name, Object obj) { updateSymtabEntry(name, normalized); } + /** + * Executes the {@code nextfile} statement: abandons the current input file + * and resumes the per-file input loop at the appropriate point. The + * runtime and operand stacks are cleared, so {@code nextfile} unwinds + * user-defined function calls, mirroring {@code exit}. + * + * @param position the tuple position tracker to redirect + */ + private void executeNextfile(PositionTracker position) { + if (endFileAddress == null || !inputFileLoopStarted) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from a BEGIN rule"); + } + if (withinEndBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from an END rule"); + } + if (withinEndFileBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from an ENDFILE rule"); + } + // nextfile can be invoked from user-defined functions: unwind them. + runtimeStack.popAllFrames(); + operandStack.clear(); + if (withinBeginFileBlocks && jrt.hasPendingInputFileError(resolvedInputSource)) { + // The file could not be opened: skip its ENDFILE rules (gawk + // BEGINFILE error handling) and go straight to the next file. + withinBeginFileBlocks = false; + position.jump(nextFileAddress); + } else { + withinBeginFileBlocks = false; + withinEndFileBlocks = true; + position.jump(endFileAddress); + } + } + + /** + * Raises the gawk-compatible fatal error when a non-redirected + * {@code getline} executes inside a BEGINFILE or ENDFILE rule. + * + * @param position the tuple position tracker, for error reporting + */ + private void checkGetlineAllowed(PositionTracker position) { + if (withinBeginFileBlocks || withinEndFileBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "non-redirected `getline' invalid inside `" + + (withinBeginFileBlocks ? "BEGINFILE" : "ENDFILE") + + "' rule"); + } + } + private void applyInputSourceFilelistAssignmentsIfNeeded() { if (inputSourceFilelistAssignmentsApplied || resolvedInputSource instanceof StreamInputSource) { return; diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 0bad1b36..5b31fe23 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -131,6 +131,8 @@ enum Token { KW_FUNCTION, KW_BEGIN, KW_END, + KW_BEGINFILE, + KW_ENDFILE, KW_IN, KW_IF, KW_ELSE, @@ -140,6 +142,7 @@ enum Token { KW_RETURN, KW_EXIT, KW_NEXT, + KW_NEXTFILE, KW_CONTINUE, KW_DELETE, KW_BREAK, @@ -169,6 +172,8 @@ enum Token { KEYWORDS.put("function", Token.KW_FUNCTION); KEYWORDS.put("BEGIN", Token.KW_BEGIN); KEYWORDS.put("END", Token.KW_END); + KEYWORDS.put("BEGINFILE", Token.KW_BEGINFILE); + KEYWORDS.put("ENDFILE", Token.KW_ENDFILE); KEYWORDS.put("in", Token.KW_IN); // statements @@ -180,6 +185,7 @@ enum Token { KEYWORDS.put("return", Token.KW_RETURN); KEYWORDS.put("exit", Token.KW_EXIT); KEYWORDS.put("next", Token.KW_NEXT); + KEYWORDS.put("nextfile", Token.KW_NEXTFILE); KEYWORDS.put("continue", Token.KW_CONTINUE); KEYWORDS.put("delete", Token.KW_DELETE); KEYWORDS.put("break", Token.KW_BREAK); @@ -283,6 +289,8 @@ private static BuiltinFunction of(String name) { SPECIAL_VAR_NAMES.put("ARGC", SP_IDX); SPECIAL_VAR_NAMES.put("ARGV", SP_IDX); SPECIAL_VAR_NAMES.put("IGNORECASE", SP_IDX); + SPECIAL_VAR_NAMES.put("ERRNO", SP_IDX); + SPECIAL_VAR_NAMES.put("ARGIND", SP_IDX); } /** @@ -310,6 +318,19 @@ public AwkParser(Map extensions, boolean posix) { this.posix = posix; } + /** + * Returns whether the keyword is disabled in the current compile-time + * mode. BEGINFILE and ENDFILE are gawk extensions: in POSIX mode they are + * not special and lex as plain identifiers, exactly like + * {@code gawk --posix}. + * + * @param keywordToken the keyword token to inspect + * @return {@code true} when the keyword must be treated as an identifier + */ + private boolean isDisabledKeyword(Token keywordToken) { + return posix && (keywordToken == Token.KW_BEGINFILE || keywordToken == Token.KW_ENDFILE); + } + private List scriptSources; private int scriptSourcesCurrentIndex; private LineNumberReader reader; @@ -515,18 +536,42 @@ private void readString() throws IOException { /** * Reads the regular expression (between slashes '/') and handle '\/'. + * A slash within a bracket expression (e.g. {@code /[/]/}) does not + * terminate the regular expression, per POSIX ERE bracket semantics. * * @throws IOException */ private void readRegexp() throws IOException { regexp.setLength(0); - while (token != Token.EOF && c > 0 && c != '/' && c != '\n') { + boolean inBracket = false; + while (token != Token.EOF && c > 0 && (c != '/' || inBracket) && c != '\n') { if (c == '\\') { read(); if (c != '/') { regexp.append('\\'); } + regexp.append((char) c); + read(); + continue; + } + if (!inBracket && c == '[') { + inBracket = true; + regexp.append((char) c); + read(); + // a ']' right after '[' (or after '[^') is a literal ']' + if (c == '^') { + regexp.append((char) c); + read(); + } + if (c == ']') { + regexp.append((char) c); + read(); + } + continue; + } + if (inBracket && c == ']') { + inBracket = false; } regexp.append((char) c); read(); @@ -839,7 +884,7 @@ private Token lexer() throws IOException { return token; } Token kwToken = KEYWORDS.get(text.toString()); - if (kwToken != null) { + if (kwToken != null && !isDisabledKeyword(kwToken)) { token = kwToken; return token; } @@ -1043,6 +1088,12 @@ AST RULE() throws IOException { } else if (token == Token.KW_END) { lexer(); optExpr = symbolTable.addEND(); + } else if (token == Token.KW_BEGINFILE) { + lexer(); + optExpr = symbolTable.addBEGINFILE(); + } else if (token == Token.KW_ENDFILE) { + lexer(); + optExpr = symbolTable.addENDFILE(); } else if (token != Token.OPEN_BRACE && token != Token.SEMICOLON && token != Token.NEWLINE && token != Token.EOF) { // true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions optExpr = ASSIGNMENT_EXPRESSION(null, true, true, false); @@ -1676,6 +1727,8 @@ AST STATEMENT() throws IOException { stmt = PRINTF_STATEMENT(); } else if (token == Token.KW_NEXT) { stmt = NEXT_STATEMENT(); + } else if (token == Token.KW_NEXTFILE) { + stmt = NEXTFILE_STATEMENT(); } else if (token == Token.KW_CONTINUE) { stmt = CONTINUE_STATEMENT(); } else if (token == Token.KW_BREAK) { @@ -2096,6 +2149,12 @@ AST NEXT_STATEMENT() throws IOException { return new NextStatementAst(); } + AST NEXTFILE_STATEMENT() throws IOException { + expectKeyword("nextfile"); + nextfileEncountered = true; + return new NextfileStatementAst(); + } + AST CONTINUE_STATEMENT() throws IOException { expectKeyword("continue"); return new ContinueStatementAst(); @@ -2614,6 +2673,52 @@ private boolean isEnd() { return result; } + private boolean isBeginFile = isBeginFile(); + + protected final void setBeginFileFlag(boolean flag) { + isBeginFile = flag; + } + + private boolean isBeginFile() { + boolean result = isBeginFile; + if (!result && ast1 != null) { + result = ast1.isBeginFile(); + } + if (!result && ast2 != null) { + result = ast2.isBeginFile(); + } + if (!result && ast3 != null) { + result = ast3.isBeginFile(); + } + if (!result && ast4 != null) { + result = ast4.isBeginFile(); + } + return result; + } + + private boolean isEndFile = isEndFile(); + + protected final void setEndFileFlag(boolean flag) { + isEndFile = flag; + } + + private boolean isEndFile() { + boolean result = isEndFile; + if (!result && ast1 != null) { + result = ast1.isEndFile(); + } + if (!result && ast2 != null) { + result = ast2.isEndFile(); + } + if (!result && ast3 != null) { + result = ast3.isEndFile(); + } + if (!result && ast4 != null) { + result = ast4.isEndFile(); + } + return result; + } + private boolean isFunction = isFunction(); @SuppressWarnings("unused") @@ -2719,7 +2824,12 @@ public boolean isScalar() { } private static boolean isRule(AST ast) { - return ast != null && !ast.isBegin() && !ast.isEnd() && !ast.isFunction(); + return ast != null + && !ast.isBegin() + && !ast.isEnd() + && !ast.isBeginFile() + && !ast.isEndFile() + && !ast.isFunction(); } /** @@ -2775,6 +2885,14 @@ private static boolean containsASTType(AST ast, Class[] clsArray) { private Address nextAddress; + /** + * Whether the program contains at least one {@code nextfile} statement, + * anywhere (rules or user-defined functions). When set, the main input + * loop is compiled with per-file stepping so the runtime can jump to the + * ENDFILE section and advance to the next input file. + */ + private boolean nextfileEncountered; + private final class RuleListAst extends AST { private RuleListAst(AST rule, AST rest) { @@ -2857,20 +2975,26 @@ public int populateTuples(AwkTuples tuples) { Address exitAddr = tuples.createAddress("end blocks start address"); tuples.setExitAddress(exitAddr); - // grab all BEGINs + // Does the program use BEGINFILE/ENDFILE rules or nextfile? If so, + // the main input loop must step through the input one file at a + // time (per-file scaffolding) instead of streaming across files. + boolean hasBeginFileRules = false; + boolean hasEndFileRules = false; ptr = this; - // ptr.getAst1() == blank rule condition (i.e.: { print }) while (ptr != null) { - if (ptr.getAst1() != null && ptr.getAst1().isBegin()) { - ptr.getAst1().populateTuples(tuples); + if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) { + hasBeginFileRules = true; + } + if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) { + hasEndFileRules = true; } - ptr = ptr.getAst2(); } // Do we have rules? (apart from BEGIN) - // If we have rules or END, we need to parse the input - boolean reqInput = false; + // If we have rules, END, BEGINFILE, or ENDFILE, we need to parse + // the input + boolean reqInput = hasBeginFileRules || hasEndFileRules; // Check for "normal" rules ptr = this; @@ -2890,19 +3014,64 @@ public int populateTuples(AwkTuples tuples) { ptr = ptr.getAst2(); } - if (reqInput) { - Address inputLoopAddress = null; - Address noMoreInput = null; + boolean perFileScaffolding = reqInput + && (hasBeginFileRules || hasEndFileRules || nextfileEncountered); - inputLoopAddress = tuples.createAddress("input_loop_address"); - tuples.address(inputLoopAddress); + // The per-file addresses must be registered before the BEGIN rules + // run, so a nextfile executed from a user-defined function can + // resolve them at runtime. + Address beginFileAddress = null; + Address endFileAddress = null; + Address nextFileAddress = null; + if (perFileScaffolding) { + beginFileAddress = tuples.createAddress("begin_file"); + endFileAddress = tuples.createAddress("end_file"); + nextFileAddress = tuples.createAddress("next_file"); + tuples.setEndFileAddress(endFileAddress); + tuples.setNextFileAddress(nextFileAddress); + } - ptr = this; + // grab all BEGINs + ptr = this; + // ptr.getAst1() == blank rule condition (i.e.: { print }) + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isBegin()) { + ptr.getAst1().populateTuples(tuples); + } - noMoreInput = tuples.createAddress("no_more_input"); - tuples.consumeInput(noMoreInput); + ptr = ptr.getAst2(); + } + + if (reqInput) { + Address inputLoopAddress = tuples.createAddress("input_loop_address"); + Address noMoreInput = tuples.createAddress("no_more_input"); + + if (perFileScaffolding) { + // Advance to the first input file so the BEGINFILE rules + // observe its FILENAME, FNR, ARGIND, and ERRNO. + tuples.nextFile(noMoreInput); + + // BEGINFILE rules, in the order they were read + tuples.address(beginFileAddress); + ptr = this; + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) { + ptr.getAst1().populateTuples(tuples); + } + ptr = ptr.getAst2(); + } + + // per-file input loop: at end of the current file, run the + // ENDFILE rules instead of silently opening the next file + tuples.address(inputLoopAddress); + tuples.consumeFileInput(endFileAddress); + } else { + tuples.address(inputLoopAddress); + tuples.consumeInput(noMoreInput); + } // grab all INPUT RULES + ptr = this; while (ptr != null) { // the first one of these is an input rule if (isRule(ptr.getAst1())) { @@ -2914,11 +3083,27 @@ public int populateTuples(AwkTuples tuples) { tuples.gotoAddress(inputLoopAddress); - if (reqInput) { - tuples.address(noMoreInput); - // compiler has issue with missing nop here - tuples.nop(); + if (perFileScaffolding) { + // ENDFILE rules, in the order they were read + tuples.address(endFileAddress); + ptr = this; + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) { + ptr.getAst1().populateTuples(tuples); + } + ptr = ptr.getAst2(); + } + + // then move on to the next input file, or fall through to + // the END rules once the input is exhausted + tuples.address(nextFileAddress); + tuples.nextFile(noMoreInput); + tuples.gotoAddress(beginFileAddress); } + + tuples.address(noMoreInput); + // compiler has issue with missing nop here + tuples.nop(); } // indicate where the first end block resides @@ -3008,7 +3193,9 @@ public int populateTuples(AwkTuples tuples) { pushSourceLineNumber(tuples); boolean unconditionalRule = getAst1() == null || getAst1().isBegin() - || getAst1().isEnd(); + || getAst1().isEnd() + || getAst1().isBeginFile() + || getAst1().isEndFile(); if (!unconditionalRule) { getAst1().populateTuples(tuples); // result of whether to execute or not is on the stack @@ -3026,12 +3213,12 @@ public int populateTuples(AwkTuples tuples) { private void populateRuleBody(AwkTuples tuples) { // execute the optRule here! if (getAst2() == null) { - if (getAst1() == null || (!getAst1().isBegin() && !getAst1().isEnd())) { + if (isRule(this)) { // display $0 tuples.print(0); } // else, don't populate it with anything - // (i.e., blank BEGIN/END rule) + // (i.e., blank BEGIN/END/BEGINFILE/ENDFILE rule) } else { // execute it, and leave nothing on the stack getAst2().populateTuples(tuples); @@ -3041,13 +3228,35 @@ private void populateRuleBody(AwkTuples tuples) { @Override public Address nextAddress() { if (!isRule(this)) { - throw new SemanticException("Must call next within an input rule."); + throw new SemanticException( + "`next' cannot be called from a `" + specialRuleName() + "' rule."); } if (nextAddress == null) { throw new SemanticException("Cannot call next here."); } return nextAddress; } + + /** + * Names the special (non-input) rule this AST represents, for + * gawk-compatible diagnostics. + */ + private String specialRuleName() { + AST pattern = getAst1(); + if (pattern != null && pattern.isBegin()) { + return "BEGIN"; + } + if (pattern != null && pattern.isEnd()) { + return "END"; + } + if (pattern != null && pattern.isBeginFile()) { + return "BEGINFILE"; + } + if (pattern != null && pattern.isEndFile()) { + return "ENDFILE"; + } + return "special"; + } } private final class IfStatementAst extends AST { @@ -4770,6 +4979,38 @@ public int populateTuples(AwkTuples tuples) { } } + private final class BeginFileAst extends AST { + + private BeginFileAst() { + super(); + setBeginFileFlag(true); + } + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + tuples.push(1); + popSourceLineNumber(tuples); + return 1; + } + } + + private final class EndFileAst extends AST { + + private EndFileAst() { + super(); + setEndFileFlag(true); + } + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + tuples.push(1); + popSourceLineNumber(tuples); + return 1; + } + } + private final class PreIncAst extends ScalarExpressionAst { private PreIncAst(AST symbolAst) { @@ -5074,6 +5315,12 @@ private void pushSpecialVariable(AwkTuples tuples, String id) { case "ARGC": tuples.pushARGC(); break; + case "ERRNO": + tuples.pushERRNO(); + break; + case "ARGIND": + tuples.pushARGIND(); + break; default: throw new Error("Unhandled special var: " + id); } @@ -5127,6 +5374,12 @@ private void assignSpecialVariable(AwkTuples tuples, String id) { case "ARGC": tuples.assignARGC(); break; + case "ERRNO": + tuples.assignERRNO(); + break; + case "ARGIND": + tuples.assignARGIND(); + break; default: throw new Error("Unhandled special var: " + id); } @@ -5494,6 +5747,29 @@ public int populateTuples(AwkTuples tuples) { } } + private class NextfileStatementAst extends AST { + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + AST nextable = searchFor(AstFlag.NEXTABLE); + if (nextable != null) { + // Direct use inside a rule: BEGIN, END, and ENDFILE reject + // nextfile at compile time, mirroring gawk's fatal errors. + // (Uses within user-defined functions are checked at runtime.) + AST pattern = nextable.getAst1(); + if (pattern != null && (pattern.isBegin() || pattern.isEnd() || pattern.isEndFile())) { + String ruleName = pattern.isBegin() ? "BEGIN" : pattern.isEnd() ? "END" : "ENDFILE"; + throw new SemanticException( + "`nextfile' cannot be called from a `" + ruleName + "' rule."); + } + } + tuples.execNextfile(); + popSourceLineNumber(tuples); + return 0; + } + } + private final class ContinueStatementAst extends AST { private ContinueStatementAst() { @@ -5589,6 +5865,8 @@ int numGlobals() { // "constants" private BeginAst beginAst = null; private EndAst endAst = null; + private BeginFileAst beginFileAst = null; + private EndFileAst endFileAst = null; // functions (proxies) private Map functionProxies = new HashMap(); @@ -5625,6 +5903,20 @@ AST addEND() { return endAst; } + AST addBEGINFILE() { + if (beginFileAst == null) { + beginFileAst = new BeginFileAst(); + } + return beginFileAst; + } + + AST addENDFILE() { + if (endFileAst == null) { + endFileAst = new EndFileAst(); + } + return endFileAst; + } + /** * Returns whether the script references the named global variable, * without creating a symbol for it. diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 396c2891..03c0aa59 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -1492,6 +1492,36 @@ public void assignIGNORECASE() { queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_IGNORECASE)); } + /** + * Emits the tuple pushing the value of ERRNO, managed by the JRT. + */ + public void pushERRNO() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_ERRNO)); + } + + /** + * Emits the tuple assigning the top of the stack to ERRNO, managed by the + * JRT. + */ + public void assignERRNO() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_ERRNO)); + } + + /** + * Emits the tuple pushing the value of ARGIND, managed by the JRT. + */ + public void pushARGIND() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_ARGIND)); + } + + /** + * Emits the tuple assigning the top of the stack to ARGIND, managed by + * the JRT. + */ + public void assignARGIND() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_ARGIND)); + } + /** Pushes the current value of {@code RS} onto the operand stack. */ public void pushRS() { queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_RS)); @@ -1744,6 +1774,54 @@ public void setWithinEndBlocks(boolean b) { queue.add(new Tuple.BooleanTuple(Opcode.SET_WITHIN_END_BLOCKS, b)); } + /** + * Emits the tuple registering the address of the ENDFILE section, so that + * a runtime {@code nextfile} statement can jump to it. + * + * @param addr address of the ENDFILE section + */ + public void setEndFileAddress(Address addr) { + queue.add(new Tuple.AddressTuple(Opcode.SET_ENDFILE_ADDRESS, addr)); + } + + /** + * Emits the tuple registering the address of the {@code NEXT_FILE} tuple, + * so that a runtime {@code nextfile} statement can bypass the ENDFILE + * rules for input files that could not be opened. + * + * @param addr address of the NEXT_FILE tuple + */ + public void setNextFileAddress(Address addr) { + queue.add(new Tuple.AddressTuple(Opcode.SET_NEXTFILE_ADDRESS, addr)); + } + + /** + * Emits the tuple advancing the main input to the next input file, or + * jumping to the given address when no input file remains. + * + * @param address address to jump to when no more input files remain + */ + public void nextFile(Address address) { + queue.add(new Tuple.AddressTuple(Opcode.NEXT_FILE, address)); + } + + /** + * Emits the tuple consuming one record of the current input file only, + * jumping to the given address at end of the current file. + * + * @param address address to jump to at end of the current input file + */ + public void consumeFileInput(Address address) { + queue.add(new Tuple.AddressTuple(Opcode.CONSUME_FILE_INPUT, address)); + } + + /** + * Emits the tuple executing the {@code nextfile} statement at runtime. + */ + public void execNextfile() { + queue.add(new Tuple.NoOperandTuple(Opcode.EXEC_NEXTFILE)); + } + /** *

* exitWithCode. diff --git a/src/main/java/io/jawk/intermediate/Opcode.java b/src/main/java/io/jawk/intermediate/Opcode.java index 8057f103..c95510ad 100644 --- a/src/main/java/io/jawk/intermediate/Opcode.java +++ b/src/main/java/io/jawk/intermediate/Opcode.java @@ -1540,7 +1540,98 @@ public enum Opcode { * Argument: offset of the FUNCTAB global
* Stack unchanged. */ - UPDATE_FUNCTAB; + UPDATE_FUNCTAB, + + /** + * Advances the main input to the next input file, applying pending + * {@code name=value} command-line assignments along the way. On success, + * FILENAME, FNR, ARGIND, and ERRNO are updated and execution falls through + * to the BEGINFILE rules. When no input file remains, it jumps to the + * specified address. Emitted only when BEGINFILE/ENDFILE rules or a + * {@code nextfile} statement require per-file input stepping. + *

+ * Argument: address to jump to when no more input files remain + *

+ * Stack unchanged. + */ + NEXT_FILE, + + /** + * Consume the next record of the current input file only; assigning $0 and + * recalculating $1, $2, etc. Unlike {@link #CONSUME_INPUT}, it never + * advances to the next input file: at end of the current file it jumps to + * the specified address so the ENDFILE rules can run. + *

+ * Argument: address to jump to at end of the current input file + *

+ * Stack unchanged. + */ + CONSUME_FILE_INPUT, + + /** + * Executes the {@code nextfile} statement: abandons the current input + * file and resumes the per-file input loop. The runtime jumps to the + * ENDFILE rules when the current file was opened successfully, or + * directly past them when the file could not be opened (BEGINFILE error + * handling). The runtime stack and operand stack are cleared, allowing + * {@code nextfile} to be invoked from user-defined functions. + *

+ * Stack after: (empty) + */ + EXEC_NEXTFILE, + + /** + * Internal. Registers the address of the ENDFILE section so that a + * runtime {@code nextfile} can jump to it. + *

+ * Argument: address of the ENDFILE section + *

+ * Stack remains unchanged. + */ + SET_ENDFILE_ADDRESS, + + /** + * Internal. Registers the address of the {@link #NEXT_FILE} tuple so that + * a runtime {@code nextfile} can bypass the ENDFILE rules for input files + * that could not be opened. + *

+ * Argument: address of the NEXT_FILE tuple + *

+ * Stack remains unchanged. + */ + SET_NEXTFILE_ADDRESS, + + /** + * Assigns the top of the stack to ERRNO, managed by the JRT. + *

+ * Stack before: value ...
+ * Stack after: value ... + */ + ASSIGN_ERRNO, + + /** + * Pushes the value of ERRNO, managed by the JRT. + *

+ * Stack before: ...
+ * Stack after: errno-value ... + */ + PUSH_ERRNO, + + /** + * Assigns the top of the stack to ARGIND, managed by the JRT. + *

+ * Stack before: value ...
+ * Stack after: value ... + */ + ASSIGN_ARGIND, + + /** + * Pushes the value of ARGIND, managed by the JRT. + *

+ * Stack before: ...
+ * Stack after: argind-value ... + */ + PUSH_ARGIND; private static final Opcode[] VALUES = values(); diff --git a/src/main/java/io/jawk/jrt/InputSource.java b/src/main/java/io/jawk/jrt/InputSource.java index a9932cc1..d850473a 100644 --- a/src/main/java/io/jawk/jrt/InputSource.java +++ b/src/main/java/io/jawk/jrt/InputSource.java @@ -98,6 +98,12 @@ default String getRecord() { /** * Indicates whether the current record originates from a named file in the * argument list. + *

+ * For custom implementations, this flag also controls whether consuming a + * record advances the per-file record counter {@code FNR}. Records read by + * the built-in stream input always advance {@code FNR} — including + * standard input — per POSIX. + *

* * @return {@code true} when sourced from a filename argument */ diff --git a/src/main/java/io/jawk/jrt/JRT.java b/src/main/java/io/jawk/jrt/JRT.java index 92ad9962..928a91e5 100644 --- a/src/main/java/io/jawk/jrt/JRT.java +++ b/src/main/java/io/jawk/jrt/JRT.java @@ -136,6 +136,9 @@ public class JRT { private int rstart; // last match start (1-based) private int rlength; // last match length private Object filename; // current input filename scalar (or empty for stdin/pipe) + private Object errno; // last input I/O error description (gawk ERRNO) + private Object argind; // ARGV index of the current input file (gawk ARGIND) + private boolean syntheticFilePresented; // custom InputSource already presented as a single "file" private String fs; // field separator private String rs; // record separator (regexp) private String ofs; // output field separator @@ -320,6 +323,8 @@ public static boolean isJrtManagedSpecialVariable(String name) { case "FNR": case "ARGC": case "IGNORECASE": + case "ERRNO": + case "ARGIND": return true; default: return false; @@ -375,6 +380,9 @@ public void prepareForExecution(String defaultFs, String defaultRs) { rstart = 0; rlength = 0; filename = ""; + errno = ""; + argind = ZERO; + syntheticFilePresented = false; // Apply default runtime special variables. setFS(defaultFs == null ? Awk.DEFAULT_FS : defaultFs); @@ -457,6 +465,12 @@ public boolean applySpecialVariable(String name, Object value) { case "IGNORECASE": setIGNORECASE(value); return true; + case "ERRNO": + setERRNO(value); + return true; + case "ARGIND": + setARGIND(value); + return true; default: return false; } @@ -1585,6 +1599,42 @@ public void setFILENAMEViaJrt(Object name) { this.filename = normalizeRecordValue(name); } + /** + * Get ERRNO as tracked by JRT. + * + * @return current ERRNO (empty string when no input error is pending) + */ + public Object getERRNO() { + return errno == null ? "" : errno; + } + + /** + * Set ERRNO tracked by JRT. + * + * @param value new ERRNO value + */ + public void setERRNO(Object value) { + this.errno = normalizeRecordValue(value); + } + + /** + * Get ARGIND as tracked by JRT. + * + * @return ARGV index of the current input file (0 before any file is open) + */ + public Object getARGIND() { + return argind == null ? ZERO : argind; + } + + /** + * Set ARGIND tracked by JRT. + * + * @param value new ARGIND value + */ + public void setARGIND(Object value) { + this.argind = normalizeRecordValue(value); + } + /** * Get SUBSEP from the VariableManager. * @@ -1731,14 +1781,116 @@ public boolean consumeInput(final InputSource source) throws IOException { return false; } + bindConsumedRecord(source); + return true; + } + + /** + * Attempt to consume one record from the current input file only, without + * ever advancing to the next input file. Used by the per-file main input + * loop when BEGINFILE/ENDFILE rules or {@code nextfile} are present, so + * that the ENDFILE rules can run at each file boundary. + *

+ * When the current input file could not be opened (a pending ERRNO set by + * {@link #advanceToNextFile(InputSource)} that no {@code nextfile} + * consumed), the usual fatal error is raised, mirroring gawk. + *

+ * + * @param source source strategy that provides records and optional + * pre-split fields + * @return {@code true} if a record was consumed; {@code false} at the end + * of the current input file + * @throws IOException if the source raises an I/O error + */ + public boolean consumeCurrentFileInput(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (!(source instanceof StreamInputSource)) { + // Custom input sources behave as a single unnamed input file. + return consumeInput(source); + } + StreamInputSource streamSource = (StreamInputSource) source; + String openError = streamSource.getCurrentFileOpenError(); + if (openError != null) { + throw new AwkRuntimeException( + "cannot open file `" + toAwkString(getFILENAME()) + "' for reading: " + openError); + } + activeSource = source; + if (!streamSource.nextRecordInCurrentFile()) { + return false; + } + bindConsumedRecord(source); + return true; + } + + /** + * Advance the main input to the next input file, applying pending + * {@code name=value} command-line assignments along the way. On success, + * FILENAME, FNR, ARGIND, and ERRNO are updated and {@code $0} is cleared, + * so the BEGINFILE rules observe the new file. A file that cannot be + * opened is still reported as available, with ERRNO carrying the error + * description (gawk BEGINFILE error handling). + * + * @param source source strategy that provides records and optional + * pre-split fields + * @return {@code true} when a new input file (or the initial stdin + * stream) is available; {@code false} when input is exhausted + * @throws IOException if an I/O error occurs while traversing ARGV + */ + public boolean advanceToNextFile(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (source instanceof StreamInputSource) { + return ((StreamInputSource) source).advanceToNextFile(); + } + // Custom input sources behave as a single unnamed input file. + if (syntheticFilePresented) { + return false; + } + syntheticFilePresented = true; + return true; + } + + /** + * Returns whether the current input file of the given source failed to + * open, leaving a pending error that only a {@code nextfile} statement in + * a BEGINFILE rule may bypass. + * + * @param source source strategy that provides records + * @return {@code true} when the current input file could not be opened + */ + public boolean hasPendingInputFileError(InputSource source) { + return source instanceof StreamInputSource + && ((StreamInputSource) source).getCurrentFileOpenError() != null; + } + + /** + * Binds the record just consumed from the given source as the current + * input record and updates the NR/FNR counters. + * + * @param source the source a record was just consumed from + */ + private void bindConsumedRecord(InputSource source) { inputLine = null; recordState = new RecordState(source); this.nr++; - if (source.isFromFilenameList()) { + if (countsTowardFNR(source)) { this.fnr++; } - return true; + } + + /** + * Returns whether consuming a record from the given source advances FNR, + * the per-file record counter. All records of the main command-line input + * flow count, including standard input (POSIX defines FNR as the record + * number in the current input file, which stdin is). For custom + * {@link InputSource} implementations, {@link InputSource#isFromFilenameList()} + * keeps controlling FNR, as documented. + * + * @param source the source a record was just consumed from + * @return {@code true} when the record advances FNR + */ + private static boolean countsTowardFNR(InputSource source) { + return source instanceof StreamInputSource || source.isFromFilenameList(); } /** @@ -1762,7 +1914,7 @@ public Object consumeInputToTarget(final InputSource source) throws IOException RecordState inputState = new RecordState(source); this.nr++; - if (source.isFromFilenameList()) { + if (countsTowardFNR(source)) { this.fnr++; } return new StrNum(inputState.getRecordText(), decimalSeparator); diff --git a/src/main/java/io/jawk/jrt/StreamInputSource.java b/src/main/java/io/jawk/jrt/StreamInputSource.java index f3cfa72e..2659cdaf 100644 --- a/src/main/java/io/jawk/jrt/StreamInputSource.java +++ b/src/main/java/io/jawk/jrt/StreamInputSource.java @@ -23,6 +23,7 @@ */ import java.io.Closeable; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -73,6 +74,11 @@ public class StreamInputSource implements InputSource, Closeable { private String currentRecord; private boolean currentReaderExhausted; + // Per-file stepping state (BEGINFILE/ENDFILE and nextfile support) + private String currentFileOpenError; + private boolean currentPresentedToLoop; + private int lastArgumentIndex; + /** * Creates a stream-backed input source. * @@ -274,6 +280,7 @@ private String nextArgument() { } String arg = jrt.toAwkString(argValue); if (!arg.isEmpty()) { + lastArgumentIndex = idx; return arg; } } @@ -359,12 +366,186 @@ private boolean prepareNextReader() throws IOException { true); jrt.setFILENAMEViaJrt(jrt.toInputScalar(arg)); jrt.setFNR(0L); + jrt.setARGIND(Long.valueOf(lastArgumentIndex)); ready = true; } } return true; } + /** + * Advance to the next input file for the per-file main input loop used + * when BEGINFILE/ENDFILE rules or {@code nextfile} are present. Variable + * assignment arguments are applied along the way, exactly like + * {@link #nextRecord()} does when it crosses a file boundary. + *

+ * On success, FILENAME, FNR, ARGIND, and ERRNO are updated and {@code $0} + * is cleared, so the BEGINFILE rules observe the new file before any + * record is read. A file that cannot be opened is still reported as + * available: ERRNO carries the error description and + * {@link #getCurrentFileOpenError()} returns it until the next advance, + * enabling gawk's non-fatal BEGINFILE error handling. + *

+ * + * @return {@code true} when a new input file (or the initial stdin + * stream) is current; {@code false} when input is exhausted + * @throws IOException if an I/O error occurs while traversing ARGV + */ + public boolean advanceToNextFile() throws IOException { + initializeArgList(); + + // Adopt a reader already opened by a non-redirected getline that ran + // before the per-file loop (e.g. in a BEGIN rule): it is the current + // input file, already positioned after the records getline consumed. + if (!currentPresentedToLoop + && partitioningReader != null + && !currentReaderExhausted + && currentFileOpenError == null) { + currentPresentedToLoop = true; + return true; + } + + currentFileOpenError = null; + arglistMaxKey = computeMaxArgvKey(); + hasFilenames = detectFilenames(); + while (true) { + String arg = nextArgument(); + if (arg == null) { + // ARGC/ARGV may have changed while evaluating assignments. + hasFilenames = detectFilenames(); + if (partitioningReader == null && !hasFilenames) { + return presentDefaultInput(); + } + closeCurrentReaderIfFileStream(); + return false; + } + if (arg.indexOf('=') > 0) { + setFilelistVariable(arg); + // Recompute bounds so ARGC changes are reflected immediately. + arglistMaxKey = computeMaxArgvKey(); + hasFilenames = detectFilenames(); + if (partitioningReader == null && !hasFilenames) { + return presentDefaultInput(); + } + if (partitioningReader != null) { + jrt.setNR(jrt.getNR() + 1); + } + } else { + closeCurrentReaderIfFileStream(); + partitioningReader = null; + currentReaderExhausted = false; + currentPresentedToLoop = true; + jrt.setFILENAMEViaJrt(jrt.toInputScalar(arg)); + beginFileState(lastArgumentIndex); + currentFileOpenError = openCurrentFile(arg); + if (currentFileOpenError != null) { + jrt.setERRNO(currentFileOpenError); + } + return true; + } + } + } + + /** + * Reads the next record of the current input file only, never advancing + * to the next input file. Used by the per-file main input loop so that + * ENDFILE rules can run at each file boundary. + * + * @return {@code true} when a record is available; {@code false} at the + * end of the current input file + * @throws IOException if an I/O error occurs + */ + public boolean nextRecordInCurrentFile() throws IOException { + if (partitioningReader == null || currentReaderExhausted || currentFileOpenError != null) { + return false; + } + String nextRecord = partitioningReader.readRecord(); + if (nextRecord == null) { + currentReaderExhausted = true; + return false; + } + currentRecord = nextRecord; + currentFromFilenameList = partitioningReader.fromFilenameList(); + return true; + } + + /** + * Returns the error description recorded when the current input file + * could not be opened by {@link #advanceToNextFile()}, or {@code null} + * when the current input is readable. + * + * @return the pending open error, or {@code null} + */ + public String getCurrentFileOpenError() { + return currentFileOpenError; + } + + /** + * Presents the default input stream (usually stdin) as the current and + * only input "file" for the per-file main input loop. + * + * @return always {@code true} + */ + private boolean presentDefaultInput() { + partitioningReader = new PartitioningReader( + new InputStreamReader(defaultInput, StandardCharsets.UTF_8), + jrt.getRSString()); + currentPresentedToLoop = true; + jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + beginFileState(0); + return true; + } + + /** + * Resets the per-file special variables observed by BEGINFILE rules: FNR + * and {@code $0} are cleared, ERRNO is emptied, and ARGIND designates the + * ARGV entry being processed. + * + * @param argvIndex the ARGV index of the new current file, or {@code 0} + * for the default input stream + */ + private void beginFileState(int argvIndex) { + jrt.setFNR(0L); + jrt.setERRNO(""); + jrt.setARGIND(Long.valueOf(argvIndex)); + jrt.setInputLine(""); + } + + /** + * Attempts to open the given filename as the current input file. + * + * @param arg the filename to open + * @return {@code null} on success, or a gawk-style error description when + * the file cannot be opened for reading + */ + private String openCurrentFile(String arg) { + File file = new File(arg); + if (file.isDirectory()) { + return "Is a directory"; + } + if (!file.exists()) { + return "No such file or directory"; + } + try { + partitioningReader = new PartitioningReader( + new InputStreamReader(new FileInputStream(arg), StandardCharsets.UTF_8), + jrt.getRSString(), + true); + return null; + } catch (IOException e) { + String message = e.getMessage(); + if (message == null || message.isEmpty()) { + return "Permission denied"; + } + // Java prefixes the failing path: "path (reason)". Keep the reason. + int open = message.lastIndexOf('('); + if (open >= 0 && message.endsWith(")")) { + return message.substring(open + 1, message.length() - 1); + } + return message; + } + } + /** * Closes the current {@link PartitioningReader} if it wraps a file stream * (not {@code defaultInput}). This prevents file-descriptor leaks when diff --git a/src/main/java/io/jawk/util/AwkSettings.java b/src/main/java/io/jawk/util/AwkSettings.java index 909843da..f30df0cd 100644 --- a/src/main/java/io/jawk/util/AwkSettings.java +++ b/src/main/java/io/jawk/util/AwkSettings.java @@ -277,7 +277,9 @@ public void setUseSortedArrayKeys(boolean useSortedArrayKeys) { /** * Whether POSIX compile-time behavior is enforced, rejecting gawk syntax * such as arrays of arrays ({@code a[i][j]}, subarray operands like - * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}). + * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}), + * and treating the gawk-specific {@code BEGINFILE} / {@code ENDFILE} + * patterns as ordinary identifiers. * * @return {@code true} when POSIX compile-time behavior is enforced */ @@ -289,7 +291,8 @@ public boolean isPosix() { * Enables or disables POSIX compile-time behavior. When enabled, gawk * syntax such as arrays of arrays ({@code a[i][j]}, subarray operands like * {@code split(..., a[i])} or {@code for (k in a[i])}) and typed regexp - * literals ({@code @/re/}) is rejected. + * literals ({@code @/re/}) is rejected, and the gawk-specific + * {@code BEGINFILE} / {@code ENDFILE} patterns are not special. * * @param posix {@code true} to enforce POSIX compile-time behavior */ diff --git a/src/site/markdown/cli-reference.md b/src/site/markdown/cli-reference.md index fd514a4b..b3d9338a 100644 --- a/src/site/markdown/cli-reference.md +++ b/src/site/markdown/cli-reference.md @@ -49,7 +49,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext > - `-r` disables Jawk's default trapping of `IllegalFormatException` for `printf` and `sprintf`. > - `--locale ` sets the locale through `Locale.forLanguageTag(...)`. > - `-t` keeps associative array keys sorted. -> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays and typed regexp literals (`@/re/`). +> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns. > - `JAWK_PERSISTENT_MEMORY` can also point at the persistent-memory file when you do not want to pass `--persist` explicitly. `--persist` wins when both are present. > > - Extensions and sandbox @@ -78,7 +78,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext - `--profile` is an executing mode. It keeps normal AWK output on stdout and writes the profiling report to stderr after execution finishes. - `--profile=` keeps normal AWK output on stdout and writes only the profiling report to the file. - `-S` affects compilation and execution, not just runtime behavior. -- `--posix` currently disables arrays-of-arrays syntax and related subarray-only operands in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. +- `--posix` currently disables arrays-of-arrays syntax and related subarray-only operands, and stops treating gawk's `BEGINFILE` / `ENDFILE` patterns as special, in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. - `--posix` is rejected together with `-L`, because loading precompiled tuples bypasses source compilation entirely. - `-L` lets you skip source compilation, but the loaded tuples must still be compatible with the current runtime. - `-f` and `-L` are distinct paths: source files compile now, tuple files load now. diff --git a/src/site/markdown/cli.md b/src/site/markdown/cli.md index 9a577444..798cd099 100644 --- a/src/site/markdown/cli.md +++ b/src/site/markdown/cli.md @@ -75,6 +75,24 @@ $ java -jar jawk-${project.version}-standalone.jar -F : '{ print FILENAME ":" FN Jawk follows the usual AWK distinction between the script itself and the remaining operands. Files and `name=value` operands after the script are visible through `ARGV` and `ARGC`. +### BEGINFILE and ENDFILE Rules + +Jawk supports gawk's `BEGINFILE` and `ENDFILE` special patterns, which hook into the command-line file processing loop. `BEGINFILE` rules run just before the first record of each input file is read, with `FILENAME` set, `FNR` at 0, `$0` cleared, and `ARGIND` designating the `ARGV` entry being processed. `ENDFILE` rules run when the last record of a file has been consumed — even for empty files — and before the `END` rules for the last file. + +When a `BEGINFILE` rule is present, a file that cannot be opened is no longer an immediate fatal error: `ERRNO` carries the error description (for example `No such file or directory` or `Is a directory`), and the script can skip the file with `nextfile`. If the `BEGINFILE` rule does not skip it, the usual fatal error is raised: + +```shell-session +$ java -jar jawk-${project.version}-standalone.jar ' + BEGINFILE { if (ERRNO != "") { printf "skipping %s (%s)\n", FILENAME, ERRNO; nextfile } } + { print FILENAME ":" FNR ":" $0 }' good.txt missing.txt +good.txt:1:hello +skipping missing.txt (No such file or directory) +``` + +The `nextfile` statement is also available in ordinary rules — including from user-defined functions — and abandons the rest of the current input file after running the `ENDFILE` rules. `next` is rejected inside `BEGINFILE`/`ENDFILE` rules, `nextfile` is rejected inside `ENDFILE`, `BEGIN`, and `END` rules, and only redirected forms of `getline` (such as `getline line < "file"`) may be used inside `BEGINFILE`/`ENDFILE`, all matching gawk's restrictions. + +As in gawk, `BEGINFILE` and `ENDFILE` are gawk extensions: with `--posix` they are not special and parse as ordinary identifiers. + ## Pass Variables Use `-v` for variables that must exist before `BEGIN` runs: diff --git a/src/site/markdown/extensions.md b/src/site/markdown/extensions.md index bacc7f2f..86bcf1d1 100644 --- a/src/site/markdown/extensions.md +++ b/src/site/markdown/extensions.md @@ -76,6 +76,8 @@ That keeps extension availability explicit and local to the embedding code. `asort()`, `asorti()`, and the `for (index in array)` statement honor `PROCINFO["sorted_in"]` with gawk's predefined comparison modes: `@unsorted`, `@ind_str_asc`, `@ind_num_asc`, `@val_str_asc`, `@val_num_asc`, `@val_type_asc`, and their `_desc` counterparts. String comparisons ignore case when `IGNORECASE` is non-zero. +Beyond the extension functions, the interpreter itself implements gawk's `BEGINFILE` / `ENDFILE` special patterns, the `nextfile` statement, and the `ERRNO` and `ARGIND` special variables (see the [CLI guide](cli.html#BEGINFILE_and_ENDFILE_Rules)). Like the other gawk-specific syntax, `BEGINFILE` and `ENDFILE` are not special in POSIX mode. + Scripts that reference `SYMTAB` or `FUNCTAB` get honest, Jawk-shaped content, populated by the runtime itself (outside POSIX mode): `SYMTAB` holds the names of the program's globals, Jawk's special variables, and `-v`/host-supplied variables; `FUNCTAB` holds the names of the program's user-defined functions plus the loaded extensions' function keywords. Command-line `name=value` operand assignments update `SYMTAB` live, as in gawk; ordinary in-script assignments are not reflected (the array is a startup snapshot, not gawk's live view). As in gawk, assigning a scalar to `SYMTAB` or `FUNCTAB` is a runtime error. > [!NOTE] diff --git a/src/site/markdown/java-input.md b/src/site/markdown/java-input.md index b3f7bdae..69577e1f 100644 --- a/src/site/markdown/java-input.md +++ b/src/site/markdown/java-input.md @@ -21,7 +21,7 @@ An `InputSource` implementation controls four things: - `nextRecord()` advances to the next record and returns `true` while input remains - `getRecordText()` provides the current `$0`, or `null` if you only expose fields - `getFields()` provides `$1..$NF`, or `null` if Jawk should split `$0` using `FS` -- `isFromFilenameList()` tells Jawk whether the current record should behave like file-list input for counters such as `FNR` +- `isFromFilenameList()` tells Jawk whether the current record should behave like file-list input for counters such as `FNR` (records from Jawk's built-in stream input always advance `FNR`, including standard input, per POSIX) When both record text and fields are available, Jawk uses: diff --git a/src/site/markdown/java.md b/src/site/markdown/java.md index e38ba7bf..a4000abc 100644 --- a/src/site/markdown/java.md +++ b/src/site/markdown/java.md @@ -36,14 +36,14 @@ Awk awk = new Awk(settings); | `setLocale(Locale)` | `Locale.US` | Locale for numeric output formatting | | `setDefaultRS(String)` | Platform line separator | Default value for `RS`, the record separator | | `setUseSortedArrayKeys(boolean)` | `false` | Whether to keep associative array keys in sorted order | -| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, and typed regexp literals (`@/re/`) | +| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns | | `putVariable(String, Object)` | Empty map | Pre-set variables available before `BEGIN` | Output destination is specified per-call on the builder (`execute()`, `execute(PrintStream)`, `execute(OutputStream)`, `execute(Appendable)`, or `execute(AwkSink)`). See the [Custom Output](java-output.html) guide for details. For more on passing variables to scripts, see [Variables and Arguments](java-variables.html). -By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`). Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, and typed regexp literals (`@/re/`): +By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`), and treats gawk's `BEGINFILE` / `ENDFILE` patterns as special. Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, and typed regexp literals (`@/re/`), and it stops treating `BEGINFILE` / `ENDFILE` as special patterns: ```java AwkSettings settings = new AwkSettings(); diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java new file mode 100644 index 00000000..afc41b29 --- /dev/null +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -0,0 +1,347 @@ +package io.jawk; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * Jawk + * ჻჻჻჻჻჻ + * Copyright (C) 2006 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import org.junit.Test; +import io.jawk.jrt.AwkRuntimeException; + +/** + * Tests for the gawk-specific BEGINFILE and ENDFILE special patterns, the + * {@code nextfile} statement, and the ERRNO / ARGIND special variables that + * support them. BEGINFILE and ENDFILE are gawk extensions and are therefore + * not special in POSIX mode. + */ +public class BeginFileEndFileTest { + + @Test + public void beginFileAndEndFileFireOncePerFile() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE/ENDFILE fire once per input file, in order") + .script( + "BEGIN { print \"begin\" }" + + " BEGINFILE { print \"bf\", FNR }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\", FNR }" + + " END { print \"end\", NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines( + "begin", + "bf 0", + "r:a1", + "r:a2", + "ef 2", + "bf 0", + "r:b1", + "ef 1", + "end 3") + .runAndAssert(); + } + + @Test + public void endFileRunsForEmptyFiles() throws Exception { + AwkTestSupport + .awkTest("ENDFILE runs even for empty input files") + .script("BEGINFILE { print \"bf\" } ENDFILE { print \"ef\", FNR }") + .file("empty", "") + .operand("{{empty}}") + .expectLines("bf", "ef 0") + .runAndAssert(); + } + + @Test + public void beginFileSeesFilenameAndClearedRecord() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE sees FILENAME, FNR=0, and a cleared $0") + .script( + "BEGINFILE { ok = (FILENAME == \"{{f1}}\"); print ok, FNR, \"[\" $0 \"]\", NF }" + + " { x = $0 }") + .file("f1", "one two\n") + .operand("{{f1}}") + .expectLines("1 0 [] 0") + .runAndAssert(); + } + + @Test + public void beginFileAndEndFileFireForStandardInput() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE/ENDFILE fire for standard input") + .script( + "BEGINFILE { print \"bf[\" FILENAME \"]\", FNR }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef[\" FILENAME \"]\", FNR }") + .stdin("s1\ns2\n") + .expectLines("bf[] 0", "r:s1", "r:s2", "ef[] 2") + .runAndAssert(); + } + + @Test + public void errnoAllowsSkippingUnreadableFile() throws Exception { + AwkTestSupport + .cliTest("ERRNO + nextfile skips a file that cannot be opened") + .script( + "BEGINFILE { if (ERRNO != \"\") { print \"skipping\"; nextfile } print \"bf\" }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\n") + .operand("{{f1}}", "no/such/file", "{{f1}}") + .expectLines("bf", "r:a1", "ef", "skipping", "bf", "r:a1", "ef") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void unreadableFileIsFatalWhenBeginFileDoesNotSkipIt() throws Exception { + AwkTestSupport + .awkTest("open failure stays fatal when BEGINFILE does not call nextfile") + .script("BEGINFILE { print \"bf\" } { print }") + .operand("no/such/file") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileInBeginFileRunsEndFileForReadableFile() throws Exception { + AwkTestSupport + .awkTest("nextfile in BEGINFILE still runs ENDFILE for a readable file") + .script( + "BEGINFILE { print \"bf\"; nextfile }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("bf", "ef") + .runAndAssert(); + } + + @Test + public void nextfileSkipsRestOfCurrentFile() throws Exception { + AwkTestSupport + .awkTest("nextfile skips the rest of the current file") + .script("{ print FILENAME \":\" $0; if (FNR == 1) nextfile }") + .file("f1", "A1\nA2\n") + .file("f2", "B1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("{{f1}}:A1", "{{f2}}:B1") + .runAndAssert(); + } + + @Test + public void nextfileWorksFromUserDefinedFunction() throws Exception { + AwkTestSupport + .awkTest("nextfile unwinds user-defined function calls") + .script( + "function skip() { nextfile; print \"unreached\" }" + + " BEGINFILE { print \"bf\" }" + + " { print \"r:\" $0; skip() }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("bf", "r:a1", "ef", "bf", "r:b1", "ef") + .runAndAssert(); + } + + @Test + public void nextfileWithoutBeginFileOrEndFileRules() throws Exception { + AwkTestSupport + .awkTest("nextfile works without any BEGINFILE/ENDFILE rules") + .script("function f() { nextfile } { print $0; f() } END { print \"nr=\" NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("a1", "b1", "nr=2") + .runAndAssert(); + } + + @Test + public void nextIsRejectedInsideBeginFile() throws Exception { + AwkTestSupport + .awkTest("next cannot be called from a BEGINFILE rule") + .script("BEGINFILE { next } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextIsRejectedInsideEndFile() throws Exception { + AwkTestSupport + .awkTest("next cannot be called from an ENDFILE rule") + .script("ENDFILE { next } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileIsRejectedInsideEndFile() throws Exception { + AwkTestSupport + .awkTest("nextfile cannot be called from an ENDFILE rule") + .script("ENDFILE { nextfile } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileIsRejectedInsideBeginAndEnd() throws Exception { + AwkTestSupport + .awkTest("nextfile cannot be called from a BEGIN rule") + .script("BEGIN { nextfile } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("nextfile cannot be called from an END rule") + .script("{ print } END { nextfile }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileInEndViaFunctionIsARuntimeError() throws Exception { + AwkTestSupport + .awkTest("nextfile reached from an END rule through a function is fatal") + .script("function f() { nextfile } { } END { f() }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsFatalInsideBeginFile() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline is invalid inside BEGINFILE") + .script("BEGINFILE { getline } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsFatalInsideEndFile() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline is invalid inside ENDFILE") + .script("ENDFILE { getline } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsFatalInsideBeginFileViaFunction() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline through a function is invalid inside BEGINFILE") + .script("function g() { getline } BEGINFILE { g() } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void redirectedGetlineIsAllowedInsideBeginFile() throws Exception { + AwkTestSupport + .awkTest("redirected getline is allowed inside BEGINFILE") + .script( + "BEGINFILE { if ((getline line < \"{{f2}}\") > 0) print \"got:\" line;" + + " close(\"{{f2}}\") }" + + " { print \"r:\" $0 }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}") + .expectLines("got:b1", "r:a1") + .runAndAssert(); + } + + @Test + public void argindTracksTheArgvIndexOfTheCurrentFile() throws Exception { + AwkTestSupport + .awkTest("ARGIND designates the ARGV entry of the current file") + .script("BEGINFILE { print \"bf\", ARGIND } END { print \"end\", ARGIND }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "x=1", "{{f2}}") + .expectLines("bf 1", "bf 3", "end 3") + .runAndAssert(); + } + + @Test + public void errnoIsAnAssignableSpecialVariable() throws Exception { + AwkTestSupport + .awkTest("ERRNO starts empty and is assignable") + .script("BEGIN { print \"[\" ERRNO \"]\"; ERRNO = \"boom\"; print ERRNO }") + .expectLines("[]", "boom") + .runAndAssert(); + } + + @Test + public void exitInsideBeginFileRunsEndRules() throws Exception { + AwkTestSupport + .cliTest("exit in BEGINFILE skips ENDFILE and runs END") + .script( + "BEGINFILE { print \"bf\"; exit(3) }" + + " { print \"r\" }" + + " ENDFILE { print \"ef\" }" + + " END { print \"end\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("bf", "end") + .expectExit(3) + .runAndAssert(); + } + + @Test + public void beginFileIsNotSpecialInPosixMode() throws Exception { + AwkTestSupport + .cliTest("--posix treats BEGINFILE/ENDFILE as plain identifiers") + .argument("--posix") + .script("BEGINFILE { print \"bf\" } ENDFILE { print \"ef\" } { print \"r:\" $0 }") + .stdin("x\n") + .expectLines("r:x") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void beginFileIsUsableAsVariableInPosixMode() throws Exception { + AwkTestSupport + .cliTest("--posix allows BEGINFILE as an ordinary variable name") + .argument("--posix") + .script("BEGIN { BEGINFILE = 42; ENDFILE = 8; print BEGINFILE + ENDFILE }") + .expectLines("50") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void regexpBracketExpressionMayContainSlash() throws Exception { + AwkTestSupport + .awkTest("a slash inside a bracket expression does not end the regexp") + .script("BEGIN { s = \"a/b\"; gsub(/[/]/, \"-\", s); print s }") + .expectLines("a-b") + .runAndAssert(); + } +} diff --git a/src/test/java/io/jawk/InputSourceTest.java b/src/test/java/io/jawk/InputSourceTest.java index f73ff62b..e51afc48 100644 --- a/src/test/java/io/jawk/InputSourceTest.java +++ b/src/test/java/io/jawk/InputSourceTest.java @@ -83,6 +83,23 @@ public void testNrCountsRecordsFromInputSource() throws Exception { .runAndAssert(); } + @Test + public void testFnrFollowsIsFromFilenameListContract() throws Exception { + // For custom InputSource implementations, isFromFilenameList() keeps + // controlling whether records advance FNR (unlike the built-in + // stream input, where stdin also counts). + awkTest("input source FNR gated by isFromFilenameList") + .script("{ print FNR, NR }") + .withInputSource( + new TableInputSource( + Arrays + .asList( + Collections.singletonList("r1"), + Collections.singletonList("r2")))) + .expectLines("0 1", "0 2") + .runAndAssert(); + } + @Test public void testDollarZeroUsesInputSourceRecord() throws Exception { awkTest("input source controls dollar zero text") diff --git a/src/test/java/io/jawk/PosixConformanceTest.java b/src/test/java/io/jawk/PosixConformanceTest.java index 88a1def1..d8ba064a 100644 --- a/src/test/java/io/jawk/PosixConformanceTest.java +++ b/src/test/java/io/jawk/PosixConformanceTest.java @@ -387,6 +387,26 @@ public void posix51NrAndFnrCounts() throws Exception { .runAndAssert(); } + @Test + public void posix51FnrCountsRecordsOnStandardInput() throws Exception { + AwkTestSupport + .awkTest("POSIX 5.1 FNR counts records read from standard input") + .script("{ print FNR, NR }") + .stdin("a\nb\nc\n") + .expectLines("1 1", "2 2", "3 3") + .runAndAssert(); + } + + @Test + public void posix51FnrCountsGetlineRecordsOnStandardInput() throws Exception { + AwkTestSupport + .awkTest("POSIX 5.1 plain getline from standard input advances FNR and NR") + .script("BEGIN { getline; getline line; print FNR, NR }") + .stdin("a\nb\nc\n") + .expectLines("2 2") + .runAndAssert(); + } + @Test public void posix52FilenameUnsetInBegin() throws Exception { AwkTestSupport @@ -584,7 +604,6 @@ public void posix87NextSkipsCurrentRecord() throws Exception { @Test public void posix88NextfileSkipsRestOfCurrentFile() throws Exception { - Assume.assumeTrue("nextfile is not supported by Jawk", false); AwkTestSupport .awkTest("POSIX 8.8 nextfile skips rest of current file") .script("{print FILENAME \":\" $0; if (FNR==1) nextfile}") From 4622583816dae465bf60a32ea9f22a47a03f1d89 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 16 Jul 2026 00:56:15 +0200 Subject: [PATCH 02/11] fix: clear ERRNO when the main input advances successfully gawk resets ERRNO to the empty string whenever the next main input (file operand or stdin) is selected successfully, not only on the BEGINFILE per-file path. Mirror that in the ordinary input flow and pin the behavior with unit tests. Addresses review feedback on #515. Co-Authored-By: Claude Fable 5 --- .../java/io/jawk/jrt/StreamInputSource.java | 9 ++++++++ .../java/io/jawk/BeginFileEndFileTest.java | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/java/io/jawk/jrt/StreamInputSource.java b/src/main/java/io/jawk/jrt/StreamInputSource.java index 2659cdaf..e5b031e3 100644 --- a/src/main/java/io/jawk/jrt/StreamInputSource.java +++ b/src/main/java/io/jawk/jrt/StreamInputSource.java @@ -338,6 +338,9 @@ private boolean prepareNextReader() throws IOException { new InputStreamReader(defaultInput, StandardCharsets.UTF_8), jrt.getRSString()); jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); return true; } closeCurrentReaderIfFileStream(); @@ -353,6 +356,9 @@ private boolean prepareNextReader() throws IOException { new InputStreamReader(defaultInput, StandardCharsets.UTF_8), jrt.getRSString()); jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); return true; } if (partitioningReader != null) { @@ -367,6 +373,9 @@ private boolean prepareNextReader() throws IOException { jrt.setFILENAMEViaJrt(jrt.toInputScalar(arg)); jrt.setFNR(0L); jrt.setARGIND(Long.valueOf(lastArgumentIndex)); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); ready = true; } } diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java index afc41b29..88a14d9a 100644 --- a/src/test/java/io/jawk/BeginFileEndFileTest.java +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -297,6 +297,29 @@ public void errnoIsAnAssignableSpecialVariable() throws Exception { .runAndAssert(); } + @Test + public void errnoIsClearedWhenMainInputOpensAFile() throws Exception { + // gawk clears ERRNO whenever the main input advances successfully, + // even without BEGINFILE/ENDFILE rules. + AwkTestSupport + .awkTest("ERRNO is cleared when an input file opens successfully") + .script("BEGIN { ERRNO = \"boom\" } { print \"[\" ERRNO \"]\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("[]") + .runAndAssert(); + } + + @Test + public void errnoIsClearedWhenMainInputReadsStandardInput() throws Exception { + AwkTestSupport + .awkTest("ERRNO is cleared when standard input is selected as main input") + .script("BEGIN { ERRNO = \"boom\" } { print \"[\" ERRNO \"]\" }") + .stdin("s1\n") + .expectLines("[]") + .runAndAssert(); + } + @Test public void exitInsideBeginFileRunsEndRules() throws Exception { AwkTestSupport From df930a419409048e6281655e632ebffa961a20fd Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 16 Jul 2026 14:33:01 +0200 Subject: [PATCH 03/11] fix: confine non-redirected getline to the current file when BEGINFILE/ENDFILE rules exist A non-redirected getline (or getline var) in an ordinary action could open the next ARGV file directly through the auto-advancing input path, skipping the ENDFILE rules of the old file and the BEGINFILE rules of the new one, and later running ENDFILE with the wrong FILENAME. While the per-file main input loop is active, only the loop itself may now cross file boundaries: getline in an action reports end-of-input at the end of the current file, and the loop then fires ENDFILE/BEGINFILE in order before processing the next file's records. In BEGIN and END rules getline keeps streaming across the input, and programs without BEGINFILE/ENDFILE rules (including nextfile-only programs) keep the classic AWK file-crossing behavior. Addresses review feedback on #515. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 33 +++++++++-- src/main/java/io/jawk/frontend/AwkParser.java | 8 ++- src/main/java/io/jawk/jrt/JRT.java | 57 +++++++++++++++++-- src/site/markdown/cli.md | 2 + .../java/io/jawk/BeginFileEndFileTest.java | 44 ++++++++++++++ 5 files changed, 132 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 791b4d1e..76107e0e 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -2105,14 +2105,17 @@ private void executeTuples(PositionTracker position) case GETLINE_INPUT: { checkGetlineAllowed(position); applyInputSourceFilelistAssignmentsIfNeeded(); - push(jrt.consumeInput(resolvedInputSource) ? 1 : 0); + boolean consumed = isMainInputFileBounded() ? + jrt.consumeCurrentFileInput(resolvedInputSource) : jrt.consumeInput(resolvedInputSource); + push(consumed ? 1 : 0); position.next(); break; } case GETLINE_INPUT_TO_TARGET: { checkGetlineAllowed(position); applyInputSourceFilelistAssignmentsIfNeeded(); - Object input = jrt.consumeInputToTarget(resolvedInputSource); + Object input = isMainInputFileBounded() ? + jrt.consumeCurrentFileInputToTarget(resolvedInputSource) : jrt.consumeInputToTarget(resolvedInputSource); if (input != null) { push(1); push(input); @@ -3516,7 +3519,7 @@ public final void assignVariable(String name, Object obj) { * @param position the tuple position tracker to redirect */ private void executeNextfile(PositionTracker position) { - if (endFileAddress == null || !inputFileLoopStarted) { + if (nextFileAddress == null || !inputFileLoopStarted) { throw new AwkRuntimeException( position.lineNumber(), "`nextfile' cannot be called from a BEGIN rule"); @@ -3534,9 +3537,11 @@ private void executeNextfile(PositionTracker position) { // nextfile can be invoked from user-defined functions: unwind them. runtimeStack.popAllFrames(); operandStack.clear(); - if (withinBeginFileBlocks && jrt.hasPendingInputFileError(resolvedInputSource)) { - // The file could not be opened: skip its ENDFILE rules (gawk - // BEGINFILE error handling) and go straight to the next file. + if (endFileAddress == null + || withinBeginFileBlocks && jrt.hasPendingInputFileError(resolvedInputSource)) { + // No ENDFILE rules to run, or the file could not be opened: skip + // the ENDFILE section (gawk BEGINFILE error handling) and go + // straight to the next file. withinBeginFileBlocks = false; position.jump(nextFileAddress); } else { @@ -3546,6 +3551,22 @@ private void executeNextfile(PositionTracker position) { } } + /** + * Returns whether a non-redirected {@code getline} must be confined to + * the current input file. While the per-file main input loop of a program + * with BEGINFILE/ENDFILE rules is running, only the loop itself may cross + * file boundaries, so that no file's hooks are ever skipped; a + * {@code getline} in an action therefore reports end-of-input at the end + * of the current file. In BEGIN and END rules — before the loop starts or + * after it ends — {@code getline} keeps streaming across the remaining + * input. + * + * @return {@code true} when getline must not advance to the next file + */ + private boolean isMainInputFileBounded() { + return endFileAddress != null && inputFileLoopStarted && !withinEndBlocks; + } + /** * Raises the gawk-compatible fatal error when a non-redirected * {@code getline} executes inside a BEGINFILE or ENDFILE rule. diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 5b31fe23..19c86260 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -3027,7 +3027,13 @@ public int populateTuples(AwkTuples tuples) { beginFileAddress = tuples.createAddress("begin_file"); endFileAddress = tuples.createAddress("end_file"); nextFileAddress = tuples.createAddress("next_file"); - tuples.setEndFileAddress(endFileAddress); + // The ENDFILE address is registered only when BEGINFILE or + // ENDFILE rules exist: its presence also confines a + // non-redirected getline to the current input file, which + // only matters when there are per-file hooks to protect. + if (hasBeginFileRules || hasEndFileRules) { + tuples.setEndFileAddress(endFileAddress); + } tuples.setNextFileAddress(nextFileAddress); } diff --git a/src/main/java/io/jawk/jrt/JRT.java b/src/main/java/io/jawk/jrt/JRT.java index 928a91e5..95bdf1da 100644 --- a/src/main/java/io/jawk/jrt/JRT.java +++ b/src/main/java/io/jawk/jrt/JRT.java @@ -1809,11 +1809,7 @@ public boolean consumeCurrentFileInput(final InputSource source) throws IOExcept return consumeInput(source); } StreamInputSource streamSource = (StreamInputSource) source; - String openError = streamSource.getCurrentFileOpenError(); - if (openError != null) { - throw new AwkRuntimeException( - "cannot open file `" + toAwkString(getFILENAME()) + "' for reading: " + openError); - } + throwIfCurrentFileUnopened(streamSource); activeSource = source; if (!streamSource.nextRecordInCurrentFile()) { return false; @@ -1822,6 +1818,57 @@ public boolean consumeCurrentFileInput(final InputSource source) throws IOExcept return true; } + /** + * Attempt to consume one record of the current input file only for + * {@code getline target}, returning the input value and leaving the + * current input record state untouched. Used instead of + * {@link #consumeInputToTarget(InputSource)} while the per-file main + * input loop is active, so a {@code getline} in an action never crosses a + * file boundary behind the BEGINFILE/ENDFILE rules' back. + * + * @param source source strategy that provides records and optional + * pre-split fields + * @return the consumed input value, or {@code null} at the end of the + * current input file + * @throws IOException if the source raises an I/O error + */ + public Object consumeCurrentFileInputToTarget(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (!(source instanceof StreamInputSource)) { + // Custom input sources behave as a single unnamed input file. + return consumeInputToTarget(source); + } + StreamInputSource streamSource = (StreamInputSource) source; + throwIfCurrentFileUnopened(streamSource); + activeSource = source; + materializeCurrentRecord(); + if (!streamSource.nextRecordInCurrentFile()) { + return null; + } + + RecordState inputState = new RecordState(source); + this.nr++; + if (countsTowardFNR(source)) { + this.fnr++; + } + return new StrNum(inputState.getRecordText(), decimalSeparator); + } + + /** + * Raises the gawk-compatible fatal error when the current input file + * could not be opened and no BEGINFILE rule bypassed it with + * {@code nextfile}. + * + * @param streamSource the main input source to check + */ + private void throwIfCurrentFileUnopened(StreamInputSource streamSource) { + String openError = streamSource.getCurrentFileOpenError(); + if (openError != null) { + throw new AwkRuntimeException( + "cannot open file `" + toAwkString(getFILENAME()) + "' for reading: " + openError); + } + } + /** * Advance the main input to the next input file, applying pending * {@code name=value} command-line assignments along the way. On success, diff --git a/src/site/markdown/cli.md b/src/site/markdown/cli.md index 798cd099..197d1903 100644 --- a/src/site/markdown/cli.md +++ b/src/site/markdown/cli.md @@ -91,6 +91,8 @@ skipping missing.txt (No such file or directory) The `nextfile` statement is also available in ordinary rules — including from user-defined functions — and abandons the rest of the current input file after running the `ENDFILE` rules. `next` is rejected inside `BEGINFILE`/`ENDFILE` rules, `nextfile` is rejected inside `ENDFILE`, `BEGIN`, and `END` rules, and only redirected forms of `getline` (such as `getline line < "file"`) may be used inside `BEGINFILE`/`ENDFILE`, all matching gawk's restrictions. +When `BEGINFILE`/`ENDFILE` rules are present, a non-redirected `getline` in an ordinary rule never crosses a file boundary: it reports end-of-input at the end of the current file, and the main loop then runs the `ENDFILE` and `BEGINFILE` rules before the next file's records are processed. (gawk instead lets such a `getline` pull in the next file's first record, firing the hooks mid-statement.) Without `BEGINFILE`/`ENDFILE` rules, `getline` keeps the classic AWK behavior of streaming across input files. + As in gawk, `BEGINFILE` and `ENDFILE` are gawk extensions: with `--posix` they are not special and parse as ordinary identifiers. ## Pass Variables diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java index 88a14d9a..a30450d0 100644 --- a/src/test/java/io/jawk/BeginFileEndFileTest.java +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -261,6 +261,50 @@ public void nonRedirectedGetlineIsFatalInsideBeginFileViaFunction() throws Excep .runAndAssert(); } + @Test + public void getlineInActionStopsAtFileBoundaryWhenHooksArePresent() throws Exception { + // The per-file loop is the only place allowed to cross file + // boundaries when BEGINFILE/ENDFILE rules exist, so no file's hooks + // are ever skipped: getline reports end-of-input at the end of the + // current file instead of silently opening the next one. + AwkTestSupport + .awkTest("getline in an action does not cross file boundaries behind the hooks") + .script( + "BEGINFILE { print \"bf:\" FILENAME }" + + " { print \"r:\" $0; if ((getline x) > 0) print \"g:\" x; else print \"g:eof\" }" + + " ENDFILE { print \"ef:\" FILENAME }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines( + "bf:{{f1}}", + "r:a1", + "g:a2", + "ef:{{f1}}", + "bf:{{f2}}", + "r:b1", + "g:eof", + "ef:{{f2}}") + .runAndAssert(); + } + + @Test + public void getlineInActionStillCrossesFilesWithoutHooks() throws Exception { + // Without BEGINFILE/ENDFILE rules there are no per-file hooks to + // protect, so getline keeps the classic AWK behavior of streaming + // across input files — even when nextfile forces the per-file loop. + AwkTestSupport + .awkTest("getline in an action crosses file boundaries without hooks") + .script( + "function unused() { nextfile }" + + " { print \"r:\" $0; if ((getline x) > 0) print \"g:\" x }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("r:a1", "g:b1") + .runAndAssert(); + } + @Test public void redirectedGetlineIsAllowedInsideBeginFile() throws Exception { AwkTestSupport From 48e6017f742a4dd6e20700f14897d4bde4e3cf76 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 16 Jul 2026 16:49:38 +0200 Subject: [PATCH 04/11] fix: NR accounting for assignment operands and POSIX classes in regexp brackets - A name=value operand between two input files no longer increments NR: per POSIX (and gawk), NR only counts records actually read. The bump existed in both the streaming and the per-file input paths and made NR off by one per assignment operand; the unit test that pinned the old behavior now asserts the gawk-compatible count. - The regexp lexer keeps bracket mode through POSIX character class, collating element, and equivalence subexpressions ([:digit:], [.x.], [=e=]), so a slash after such a class (e.g. /[[:digit:]/]/) no longer terminates the regexp literal early. Addresses review feedback on #515. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/frontend/AwkParser.java | 22 ++++++++++++++ .../java/io/jawk/jrt/StreamInputSource.java | 6 ---- .../java/io/jawk/BeginFileEndFileTest.java | 29 +++++++++++++++++++ .../java/io/jawk/JRTConsumeInputTest.java | 9 +++--- .../java/io/jawk/StreamInputSourceTest.java | 13 +++++++++ 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 19c86260..c8b3e3ec 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -570,6 +570,28 @@ private void readRegexp() throws IOException { } continue; } + if (inBracket && c == '[') { + regexp.append((char) c); + read(); + // POSIX character class, collating element, or equivalence + // class ([:alpha:], [.x.], [=e=]): its closing ']' does not + // end the outer bracket expression. + if (c == ':' || c == '.' || c == '=') { + int delimiter = c; + boolean closed = false; + while (token != Token.EOF && c > 0 && c != '\n' && !closed) { + int previous = c; + regexp.append((char) c); + read(); + if (previous == delimiter && c == ']') { + regexp.append((char) c); + read(); + closed = true; + } + } + } + continue; + } if (inBracket && c == ']') { inBracket = false; } diff --git a/src/main/java/io/jawk/jrt/StreamInputSource.java b/src/main/java/io/jawk/jrt/StreamInputSource.java index e5b031e3..610297ea 100644 --- a/src/main/java/io/jawk/jrt/StreamInputSource.java +++ b/src/main/java/io/jawk/jrt/StreamInputSource.java @@ -361,9 +361,6 @@ private boolean prepareNextReader() throws IOException { jrt.setERRNO(""); return true; } - if (partitioningReader != null) { - jrt.setNR(jrt.getNR() + 1); - } } else { closeCurrentReaderIfFileStream(); partitioningReader = new PartitioningReader( @@ -436,9 +433,6 @@ public boolean advanceToNextFile() throws IOException { if (partitioningReader == null && !hasFilenames) { return presentDefaultInput(); } - if (partitioningReader != null) { - jrt.setNR(jrt.getNR() + 1); - } } else { closeCurrentReaderIfFileStream(); partitioningReader = null; diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java index a30450d0..1edfca64 100644 --- a/src/test/java/io/jawk/BeginFileEndFileTest.java +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -411,4 +411,33 @@ public void regexpBracketExpressionMayContainSlash() throws Exception { .expectLines("a-b") .runAndAssert(); } + + @Test + public void regexpBracketExpressionMayContainSlashAfterCharacterClass() throws Exception { + // The ']' closing a POSIX character class such as [:digit:] must not + // be taken as the end of the enclosing bracket expression, so the + // slash that follows still belongs to the regexp literal. + AwkTestSupport + .awkTest("a slash after a POSIX character class does not end the regexp") + .script("BEGIN { s = \"a/b\"; gsub(/[[:digit:]/]/, \"-\", s); print s }") + .expectLines("a-b") + .runAndAssert(); + } + + @Test + public void assignmentOperandsDoNotConsumeRecordsInPerFileLoop() throws Exception { + // A name=value operand between input files applies its assignment + // without reading a record, so NR stays contiguous across files. + AwkTestSupport + .awkTest("assignment operands leave NR untouched in the per-file loop") + .script( + "BEGINFILE { print \"bf\", NR }" + + " { print NR \":\" $0 }" + + " END { print \"end\", NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "x=1", "{{f2}}") + .expectLines("bf 0", "1:a1", "2:a2", "bf 2", "3:b1", "end 3") + .runAndAssert(); + } } diff --git a/src/test/java/io/jawk/JRTConsumeInputTest.java b/src/test/java/io/jawk/JRTConsumeInputTest.java index 3ff035c6..74808449 100644 --- a/src/test/java/io/jawk/JRTConsumeInputTest.java +++ b/src/test/java/io/jawk/JRTConsumeInputTest.java @@ -32,19 +32,20 @@ public class JRTConsumeInputTest { /** * Ensures that variable assignments interleaved with filenames in - * {@code ARGV} correctly advance {@code NR}. + * {@code ARGV} apply without consuming a record: per POSIX (and gawk), + * {@code NR} only counts records actually read. * * @throws Exception if the AWK invocation fails */ @Test - public void testVariableAssignmentBetweenFilesIncrementsNR() throws Exception { + public void testVariableAssignmentBetweenFilesDoesNotAdvanceNR() throws Exception { AwkTestSupport - .awkTest("variable assignments interleaved with filenames advance NR") + .awkTest("variable assignments interleaved with filenames leave NR untouched") .file("file1", "a\n") .file("file2", "b\n") .script("{ next } \nEND { print NR }") .operand("{{file1}}", "X=1", "{{file2}}") - .expectLines("3") + .expectLines("2") .runAndAssert(); } diff --git a/src/test/java/io/jawk/StreamInputSourceTest.java b/src/test/java/io/jawk/StreamInputSourceTest.java index 7200cafa..55ee55ce 100644 --- a/src/test/java/io/jawk/StreamInputSourceTest.java +++ b/src/test/java/io/jawk/StreamInputSourceTest.java @@ -107,6 +107,19 @@ public void testVariableAssignmentInArgv() throws Exception { .runAndAssert(); } + @Test + public void testVariableAssignmentBetweenFilesDoesNotConsumeARecord() throws Exception { + // A name=value operand between two input files applies its assignment + // without reading a record: NR stays contiguous across the boundary. + awkTest("StreamInputSource name=value operands leave NR untouched") + .script("{ print NR \":\" x \":\" $0 } END { print \"end:\" NR }") + .file("a.txt", "A1\nA2\n") + .file("b.txt", "B1\n") + .operand("{{a.txt}}", "x=1", "{{b.txt}}") + .expectLines("1::A1", "2::A2", "3:1:B1", "end:3") + .runAndAssert(); + } + @Test public void testGetlineFromStdin() throws Exception { awkTest("StreamInputSource getline from stdin") From 1a53dc5a42ca65f437853453bc1a135c1c556637 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 16 Jul 2026 18:18:51 +0200 Subject: [PATCH 05/11] fix: treat ERRNO and ARGIND as ordinary variables in POSIX mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Like gawk --posix: ERRNO and ARGIND are gawk extensions, so POSIX mode compiles them as plain global variables — usable as function parameters, seeded normally by -v and name=value assignments, and untouched by the input machinery at file boundaries. Outside POSIX mode they remain JRT-managed special variables. Addresses review feedback on #515. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 43 +++++++++++++++--- src/main/java/io/jawk/frontend/AwkParser.java | 20 ++++++++- .../java/io/jawk/BeginFileEndFileTest.java | 44 +++++++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 76107e0e..7d5989cb 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1017,10 +1017,33 @@ private void applyGlobalsToStack(Map globals) { */ private boolean isPersistentEligibleGlobal(String name) { return name != null - && !JRT.isJrtManagedSpecialVariable(name) + && !isManagedSpecialVariable(name) && !NON_PERSISTENT_GLOBALS.contains(name); } + /** + * Returns whether the name is a JRT-managed special variable for this + * execution. ERRNO and ARGIND are gawk extensions: in POSIX mode they are + * ordinary global variables, as in {@code gawk --posix}. + * + * @param name variable name to inspect + * @return {@code true} when the variable is managed by the JRT + */ + private boolean isManagedSpecialVariable(String name) { + return JRT.isJrtManagedSpecialVariable(name) && !isPosixOrdinaryVariable(name); + } + + /** + * Returns whether the name is a gawk-only special variable that POSIX + * mode treats as an ordinary global. + * + * @param name variable name to inspect + * @return {@code true} when the name is ordinary in the current mode + */ + private boolean isPosixOrdinaryVariable(String name) { + return settings.isPosix() && ("ERRNO".equals(name) || "ARGIND".equals(name)); + } + /** * Validates that a seeded global name is compatible with the compiled * metadata of the current program before any value normalization can mutate a @@ -3406,9 +3429,17 @@ public final Object getVariable(String name) { case "IGNORECASE": return jrt.getIGNORECASEVar(); case "ERRNO": - return jrt.getERRNO(); + if (!settings.isPosix()) { + return jrt.getERRNO(); + } + // POSIX mode: ordinary global, answered by the slot lookup below + break; case "ARGIND": - return jrt.getARGIND(); + if (!settings.isPosix()) { + return jrt.getARGIND(); + } + // POSIX mode: ordinary global, answered by the slot lookup below + break; // lazily-materialized globals answered through their synthetic accessors case "ARGC": return getARGC(); @@ -3473,7 +3504,7 @@ public final void assignVariable(String name, Object obj) { if (globalVariableOffsets == null || globalVariableArrays == null) { Object normalized = normalizeExternalVariableValue(obj); baseInitialVariables.put(name, normalized); - if (JRT.isJrtManagedSpecialVariable(name)) { + if (isManagedSpecialVariable(name)) { baseSpecialVariables.put(name, normalized); } return; @@ -3489,7 +3520,9 @@ public final void assignVariable(String name, Object obj) { // FS=: between input files) must reach the JRT, not a global slot. // ARGC is excluded: JRT.setARGC delegates back to this method, and its // authoritative storage is the compiled slot below. - if (!"ARGC".equals(name) && jrt.applySpecialVariable(name, normalized)) { + if (!"ARGC".equals(name) + && !isPosixOrdinaryVariable(name) + && jrt.applySpecialVariable(name, normalized)) { updateSymtabEntry(name, normalized); return; } diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index c8b3e3ec..f336a332 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -4698,7 +4698,7 @@ public void semanticAnalysis() throws SemanticException { FunctionDefParamListAst ptr = this; while (ptr != null) { - if (SPECIAL_VAR_NAMES.get(ptr.id) != null) { + if (SPECIAL_VAR_NAMES.get(ptr.id) != null && !isPosixOrdinarySpecialName(ptr.id)) { throw new SemanticException("Special variable " + ptr.id + " cannot be used as a formal parameter"); } ptr = (FunctionDefParamListAst) ptr.getAst1(); @@ -5292,7 +5292,23 @@ public int populateTuples(AwkTuples tuples) { * them. */ private boolean isJrtManagedSpecialName(String id) { - return SPECIAL_VAR_NAMES.containsKey(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id); + return SPECIAL_VAR_NAMES.containsKey(id) + && !"ENVIRON".equals(id) + && !"ARGV".equals(id) + && !isPosixOrdinarySpecialName(id); + } + + /** + * Returns whether the name is a gawk-only special variable that POSIX + * mode treats as an ordinary identifier. Like {@code gawk --posix}, POSIX + * mode compiles ERRNO and ARGIND as plain global variables, usable as + * function parameters and untouched by the input machinery. + * + * @param id the identifier to inspect + * @return {@code true} when the name is ordinary in the current mode + */ + private boolean isPosixOrdinarySpecialName(String id) { + return posix && ("ERRNO".equals(id) || "ARGIND".equals(id)); } /** Emits the tuple pushing the value of a JRT-managed special variable. */ diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java index 1edfca64..db9dd3dd 100644 --- a/src/test/java/io/jawk/BeginFileEndFileTest.java +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -392,6 +392,50 @@ public void beginFileIsNotSpecialInPosixMode() throws Exception { .runAndAssert(); } + @Test + public void errnoAndArgindAreOrdinaryVariablesInPosixMode() throws Exception { + // Like gawk --posix: ERRNO and ARGIND are plain identifiers, usable + // as function parameters and as ordinary globals. + AwkTestSupport + .cliTest("--posix treats ERRNO and ARGIND as ordinary identifiers") + .argument("--posix") + .script( + "function f(ARGIND) { return ARGIND * 2 }" + + " function g(ERRNO) { return ERRNO \"x\" }" + + " BEGIN { ERRNO = \"a\"; print f(3), g(ERRNO) }") + .expectLines("6 ax") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void argindIsNotUpdatedAtFileBoundariesInPosixMode() throws Exception { + // Like gawk --posix: reading input files does not touch a + // user-assigned ARGIND. + AwkTestSupport + .cliTest("--posix keeps a user-assigned ARGIND across input files") + .argument("--posix") + .script("BEGIN { ARGIND = 99 } { print ARGIND \":\" $0 }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("99:a1", "99:b1") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void errnoIsAssignableThroughDashVInPosixMode() throws Exception { + // -v ERRNO=... must reach the ordinary global slot in POSIX mode. + AwkTestSupport + .cliTest("--posix -v ERRNO seeds the ordinary global") + .argument("--posix", "-v", "ERRNO=foo") + .script("BEGIN { print ERRNO }") + .expectLines("foo") + .expectExit(0) + .runAndAssert(); + } + @Test public void beginFileIsUsableAsVariableInPosixMode() throws Exception { AwkTestSupport From f744d8c7bd8b966725aa22ae0b36829a927f150c Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 17:44:17 +0200 Subject: [PATCH 06/11] docs: group POSIX and gawk feature support in README Introduce a "Support for POSIX AWK and Gawk" section listing the gawk-specific features in one place, and drop the redundant skipped test_beginfile2 umbrella case in favor of a comment on the transcribed test_beginfile2_test* group. Co-Authored-By: Claude Fable 5 --- README.md | 12 ++++++++++-- src/it/java/io/jawk/gawk/GawkExtensionIT.java | 12 +++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4f868d38..12f3aafa 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,17 @@ Jawk is a pure Java implementation of [AWK](https://en.wikipedia.org/wiki/AWK). You can run it as a CLI, embed it directly in Java applications, compile scripts to reusable tuples, evaluate AWK expressions, feed it structured input, load extensions explicitly, and enable a sandboxed runtime when you need tighter execution constraints. -Gawk-specific builtins — `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, and `PROCINFO["sorted_in"]`-controlled array traversal — are available by default through the built-in GNU Awk compatibility extension. +## Support for POSIX AWK and Gawk -Jawk also supports gawk's `BEGINFILE` / `ENDFILE` special patterns (with the `ERRNO` and `ARGIND` special variables) and the `nextfile` statement, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error. As in gawk, `BEGINFILE` and `ENDFILE` are not special in `--posix` mode. +Jawk fully implements POSIX AWK, and adds support for the most commonly used gawk-specific features: + +- Builtins, available by default through the built-in GNU Awk compatibility extension: `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, and `PROCINFO["sorted_in"]`-controlled array traversal +- Arrays of arrays (`a[i][j]`) and typed regexp literals (`@/re/`) +- `BEGINFILE` / `ENDFILE` special patterns, with the `ERRNO` and `ARGIND` special variables, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error +- The `nextfile` statement +- The `IGNORECASE`, `SYMTAB`, and `FUNCTAB` special variables + +As in gawk, the gawk-specific syntax is not special in `--posix` mode. ## CLI Example diff --git a/src/it/java/io/jawk/gawk/GawkExtensionIT.java b/src/it/java/io/jawk/gawk/GawkExtensionIT.java index aa12f7a2..61836a34 100644 --- a/src/it/java/io/jawk/gawk/GawkExtensionIT.java +++ b/src/it/java/io/jawk/gawk/GawkExtensionIT.java @@ -1579,13 +1579,11 @@ public void test_beginfile1() throws Exception { .runAndAssert(); } - @Test - public void test_beginfile2() throws Exception { - skip( - "Covered by the test_beginfile2_test* cases below; the remaining beginfile2.sh sub-tests rely on" - + " non-redirected getline in BEGIN/END driving the BEGINFILE/ENDFILE hooks, or compare" - + " non-zero gawk CLI transcripts, which Jawk does not reproduce."); - } + // The gawk beginfile2 target is a shell script running 16 sub-tests; the + // reproducible ones are transcribed below as test_beginfile2_test*. The + // remaining sub-tests rely on non-redirected getline in BEGIN/END driving + // the BEGINFILE/ENDFILE hooks mid-statement, or compare non-zero gawk CLI + // transcripts, which Jawk does not reproduce. @Test public void test_beginfile2_test2() throws Exception { From c22b9ed767c7df1ccfcc32283f47a0219203b837 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 19:22:00 +0200 Subject: [PATCH 07/11] refactor: address review feedback on the BEGINFILE/ENDFILE internals - Share the gawk-only special-variable predicate (ERRNO/ARGIND) between AwkParser and AVM through JRT.isGawkOnlySpecialVariable, so the two posix gates cannot drift apart. - Reject direct non-redirected getline inside BEGINFILE/ENDFILE at compile time; the runtime check remains for uses reached through user-defined functions, where the calling rule is unknowable statically (documented on checkGetlineAllowed and executeNextfile, which follow the same compile-time/runtime split as gawk). - Drop the redundant name=value application from CONSUME_FILE_INPUT (NEXT_FILE always runs first) and document that the hook exists only for custom InputSources, which have no ARGV traversal. - Deduplicate the ARGV entry enumeration shared by populateArgv and the synthetic getARGV accessor, preserving populateArgv's Integer keys on purpose: host-injected plain ARGV maps use Long keys and must keep precedence during traversal. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 70 +++++++++++++++---- src/main/java/io/jawk/frontend/AwkParser.java | 18 ++++- src/main/java/io/jawk/jrt/JRT.java | 12 ++++ .../java/io/jawk/BeginFileEndFileTest.java | 23 ++++-- 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 7d5989cb..08892544 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -1041,7 +1041,7 @@ private boolean isManagedSpecialVariable(String name) { * @return {@code true} when the name is ordinary in the current mode */ private boolean isPosixOrdinaryVariable(String name) { - return settings.isPosix() && ("ERRNO".equals(name) || "ARGIND".equals(name)); + return settings.isPosix() && JRT.isGawkOnlySpecialVariable(name); } /** @@ -2097,7 +2097,8 @@ private void executeTuples(PositionTracker position) case CONSUME_FILE_INPUT: { // arg[0] = address of the ENDFILE section // store the next record of the current file into $0, $1, ... - applyInputSourceFilelistAssignmentsIfNeeded(); + // (name=value assignments were already applied by the + // NEXT_FILE tuple that necessarily executed before this one) withinBeginFileBlocks = false; if (jrt.consumeCurrentFileInput(resolvedInputSource)) { position.next(); @@ -2984,14 +2985,35 @@ private void populateArgc(long offset) { private void populateArgv(long offset) { argvOffset = offset; - // ARGV[0] is the program name, ARGV[1..n] the command-line arguments. - // The count comes straight from the argument list because ARGC may not - // be materialized when the script does not reference it. - assignArray(argvOffset, 0, "jawk", true); - pop(); - for (int i = 1; i <= arguments.size(); i++) { - assignArray(argvOffset, i, jrt.toInputScalar(arguments.get(i - 1)), true); + // Integer keys, deliberately: a host-injected plain ARGV map uses + // Long keys, and the ARGV traversal gives those precedence, so the + // injected entries must not be overwritten here. + forEachArgvEntry((index, value) -> { + assignArray(argvOffset, Integer.valueOf(index), value, true); pop(); // clean up the stack after the assignment + }); + } + + /** + * Receives one ARGV entry from {@link #forEachArgvEntry(ArgvEntryConsumer)}. + */ + @FunctionalInterface + private interface ArgvEntryConsumer { + void accept(int index, Object value); + } + + /** + * Supplies the ARGV entries to the given consumer, in index order: + * ARGV[0] is the program name, ARGV[1..n] the command-line arguments. + * The count comes straight from the argument list because ARGC may not + * be materialized when the script does not reference it. + * + * @param consumer receives each (index, value) ARGV entry + */ + private void forEachArgvEntry(ArgvEntryConsumer consumer) { + consumer.accept(0, "jawk"); + for (int i = 1; i <= arguments.size(); i++) { + consumer.accept(i, jrt.toInputScalar(arguments.get(i - 1))); } } @@ -3548,6 +3570,14 @@ public final void assignVariable(String name, Object obj) { * and resumes the per-file input loop at the appropriate point. The * runtime and operand stacks are cleared, so {@code nextfile} unwinds * user-defined function calls, mirroring {@code exit}. + *

+ * A {@code nextfile} written directly inside a BEGIN, END, or ENDFILE + * rule is already rejected at compile time by the parser. The checks + * below cover the uses reached through user-defined functions, where the + * calling rule cannot be known statically (the same function may be + * called from both an ordinary rule and an END rule); gawk performs the + * same checks at runtime. + *

* * @param position the tuple position tracker to redirect */ @@ -3603,6 +3633,14 @@ private boolean isMainInputFileBounded() { /** * Raises the gawk-compatible fatal error when a non-redirected * {@code getline} executes inside a BEGINFILE or ENDFILE rule. + *

+ * A non-redirected {@code getline} written directly inside a + * BEGINFILE/ENDFILE rule is already rejected at compile time by the + * parser. This runtime check covers the uses reached through + * user-defined functions, where the calling rule cannot be known + * statically; it lives here rather than in {@link JRT} because the + * current-rule flags are interpreter execution state. + *

* * @param position the tuple position tracker, for error reporting */ @@ -3616,6 +3654,15 @@ private void checkGetlineAllowed(PositionTracker position) { } } + /** + * Applies the command-line {@code name=value} operands once, before the + * first record of a custom {@link InputSource} is consumed. The built-in + * {@link StreamInputSource} needs none of this: it applies each + * assignment itself while traversing ARGV, at the file boundary where it + * appears. Custom sources have no ARGV traversal, so every input opcode + * that can be the first one executed (CONSUME_INPUT, NEXT_FILE, and the + * getline forms) calls this cheap, idempotent hook. + */ private void applyInputSourceFilelistAssignmentsIfNeeded() { if (inputSourceFilelistAssignmentsApplied || resolvedInputSource instanceof StreamInputSource) { return; @@ -3677,10 +3724,7 @@ public void setFILENAME(String filename) { public Object getARGV() { if (argvOffset == NULL_OFFSET) { Map argv = newAwkArray(); - argv.put(0L, "jawk"); - for (int i = 0; i < arguments.size(); i++) { - argv.put(Long.valueOf(i + 1L), jrt.toInputScalar(arguments.get(i))); - } + forEachArgvEntry((index, value) -> argv.put(Long.valueOf(index), value)); return argv; } return runtimeStack.getVariable(argvOffset, true); diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index f336a332..8ceb7fc1 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -40,6 +40,7 @@ import io.jawk.ext.ExtensionFunction; import io.jawk.intermediate.Address; import io.jawk.intermediate.AwkTuples; +import io.jawk.jrt.JRT; import io.jawk.util.ScriptSource; import io.jawk.frontend.ast.LexerException; import io.jawk.frontend.ast.ParserException; @@ -5308,7 +5309,7 @@ private boolean isJrtManagedSpecialName(String id) { * @return {@code true} when the name is ordinary in the current mode */ private boolean isPosixOrdinarySpecialName(String id) { - return posix && ("ERRNO".equals(id) || "ARGIND".equals(id)); + return posix && JRT.isGawkOnlySpecialVariable(id); } /** Emits the tuple pushing the value of a JRT-managed special variable. */ @@ -5624,6 +5625,21 @@ private GetlineAst(AST pipeExpr, AST lvalueAst, AST inRedirect) { @Override public int populateTuples(AwkTuples tuples) { pushSourceLineNumber(tuples); + // gawk restriction: only redirected forms of getline may be used + // inside BEGINFILE/ENDFILE rules. Direct uses are rejected here at + // compile time; uses reached through user-defined functions are + // caught at runtime by the interpreter, which knows the current + // rule. + if (getAst1() == null && getAst3() == null) { + AST enclosingRule = searchFor(AstFlag.NEXTABLE); + AST pattern = enclosingRule == null ? null : enclosingRule.getAst1(); + if (pattern != null && (pattern.isBeginFile() || pattern.isEndFile())) { + throw new SemanticException( + "non-redirected `getline' invalid inside `" + + (pattern.isBeginFile() ? "BEGINFILE" : "ENDFILE") + + "' rule"); + } + } if (getAst1() == null && getAst3() == null && getAst2() == null) { tuples.getlineInput(); popSourceLineNumber(tuples); diff --git a/src/main/java/io/jawk/jrt/JRT.java b/src/main/java/io/jawk/jrt/JRT.java index 95bdf1da..11ddfc08 100644 --- a/src/main/java/io/jawk/jrt/JRT.java +++ b/src/main/java/io/jawk/jrt/JRT.java @@ -331,6 +331,18 @@ public static boolean isJrtManagedSpecialVariable(String name) { } } + /** + * Returns whether the name is a gawk-only special variable that POSIX + * mode treats as an ordinary identifier, like {@code gawk --posix} does. + * Shared by the parser and the interpreter so both stay in sync. + * + * @param name variable name to inspect + * @return {@code true} when POSIX mode must treat the name as ordinary + */ + public static boolean isGawkOnlySpecialVariable(String name) { + return "ERRNO".equals(name) || "ARGIND".equals(name); + } + /** * Copies only the JRT-managed special variables from the supplied map. * diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java index db9dd3dd..b4a5ecef 100644 --- a/src/test/java/io/jawk/BeginFileEndFileTest.java +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -232,22 +232,25 @@ public void nextfileInEndViaFunctionIsARuntimeError() throws Exception { } @Test - public void nonRedirectedGetlineIsFatalInsideBeginFile() throws Exception { + public void nonRedirectedGetlineIsRejectedInsideBeginFile() throws Exception { + // Direct use is a compile-time error; see the ViaFunction cases for + // the runtime detection. AwkTestSupport .awkTest("non-redirected getline is invalid inside BEGINFILE") .script("BEGINFILE { getline } { print }") .stdin("x\n") - .expectThrow(AwkRuntimeException.class) + .expectThrow(RuntimeException.class) .runAndAssert(); } @Test - public void nonRedirectedGetlineIsFatalInsideEndFile() throws Exception { + public void nonRedirectedGetlineIsRejectedInsideEndFile() throws Exception { + // Direct use of the "getline var" form is a compile-time error too. AwkTestSupport .awkTest("non-redirected getline is invalid inside ENDFILE") - .script("ENDFILE { getline } { print }") + .script("ENDFILE { getline line } { print }") .stdin("x\n") - .expectThrow(AwkRuntimeException.class) + .expectThrow(RuntimeException.class) .runAndAssert(); } @@ -261,6 +264,16 @@ public void nonRedirectedGetlineIsFatalInsideBeginFileViaFunction() throws Excep .runAndAssert(); } + @Test + public void nonRedirectedGetlineIsFatalInsideEndFileViaFunction() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline through a function is invalid inside ENDFILE") + .script("function g() { getline } ENDFILE { g() } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + @Test public void getlineInActionStopsAtFileBoundaryWhenHooksArePresent() throws Exception { // The per-file loop is the only place allowed to cross file From 475ab9cfcd51b6a380d311abf63ec79731f5ca6d Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 20:14:36 +0200 Subject: [PATCH 08/11] refactor: drop name=value operand handling for custom InputSources name=value runtime arguments belong to the ARGV file-list traversal, which a custom InputSource replaces entirely: stop applying them before the first record. They remain visible as plain ARGV entries; -v-style variables or putVariable(...) are the supported ways to pass values. Removes the applyInputSourceFilelistAssignmentsIfNeeded hook, its one-shot flag, and the setFilelistVariable/NameValueAssignment helper chain; the two tests that pinned the old behavior now pin the new contract, which is also documented in java-input.md. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 73 ---------------------- src/site/markdown/java-input.md | 2 + src/test/java/io/jawk/AwkTest.java | 9 ++- src/test/java/io/jawk/InputSourceTest.java | 11 ++-- 4 files changed, 15 insertions(+), 80 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 08892544..777706c3 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -78,7 +78,6 @@ import io.jawk.jrt.BlockObject; import io.jawk.jrt.ConditionPair; import io.jawk.jrt.InputSource; -import io.jawk.jrt.StreamInputSource; import io.jawk.jrt.JRT; import io.jawk.jrt.VariableManager; import io.jawk.util.AwkSettings; @@ -150,7 +149,6 @@ private void push(Object o) { private final Map tupleProfilingStats; private final Map functionProfilingStats; private final Deque activeProfilingFunctions; - private boolean inputSourceFilelistAssignmentsApplied; private InputSource resolvedInputSource; private AwkExpression installedEvalExpression; private boolean mergedGlobalLayoutActive; @@ -717,7 +715,6 @@ private void resetTransientRuntimeState(List runtimeArguments, Mapname=value form. - */ - @SuppressWarnings("unused") - private void setFilelistVariable(String nameValue) { - // route through assignVariable so JRT-managed specials, global slots, - // and SYMTAB updates are handled in exactly one place - NameValueAssignment assignment = parseNameValueAssignment(nameValue); - assignVariable(assignment.name, assignment.value); - } - /** {@inheritDoc} */ @Override public final void assignVariable(String name, Object obj) { @@ -3654,27 +3612,6 @@ private void checkGetlineAllowed(PositionTracker position) { } } - /** - * Applies the command-line {@code name=value} operands once, before the - * first record of a custom {@link InputSource} is consumed. The built-in - * {@link StreamInputSource} needs none of this: it applies each - * assignment itself while traversing ARGV, at the file boundary where it - * appears. Custom sources have no ARGV traversal, so every input opcode - * that can be the first one executed (CONSUME_INPUT, NEXT_FILE, and the - * getline forms) calls this cheap, idempotent hook. - */ - private void applyInputSourceFilelistAssignmentsIfNeeded() { - if (inputSourceFilelistAssignmentsApplied || resolvedInputSource instanceof StreamInputSource) { - return; - } - for (String argument : arguments) { - if (argument.indexOf('=') > 0) { - setFilelistVariable(argument); - } - } - inputSourceFilelistAssignmentsApplied = true; - } - /** {@inheritDoc} */ @Override public Object getFS() { @@ -3840,16 +3777,6 @@ private Object normalizeExternalVariableValue(Object value) { private static final Set NON_PERSISTENT_GLOBALS = new HashSet<>( Arrays.asList("ARGV", "ARGC", "ENVIRON", "RSTART", "RLENGTH", "IGNORECASE")); - private static final class NameValueAssignment { - private final String name; - private final Object value; - - private NameValueAssignment(String name, Object value) { - this.name = name; - this.value = value; - } - } - private static final class SingleRecordInputSource implements InputSource { private final String record; diff --git a/src/site/markdown/java-input.md b/src/site/markdown/java-input.md index 69577e1f..641d6fbd 100644 --- a/src/site/markdown/java-input.md +++ b/src/site/markdown/java-input.md @@ -23,6 +23,8 @@ An `InputSource` implementation controls four things: - `getFields()` provides `$1..$NF`, or `null` if Jawk should split `$0` using `FS` - `isFromFilenameList()` tells Jawk whether the current record should behave like file-list input for counters such as `FNR` (records from Jawk's built-in stream input always advance `FNR`, including standard input, per POSIX) +A custom `InputSource` replaces the ARGV file-list traversal entirely: `name=value` runtime arguments are not applied and remain visible only as plain `ARGV` entries. Use `-v`-style variables or `putVariable(...)` to pass values to the script. + When both record text and fields are available, Jawk uses: - the field list for `$1..$NF` diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 55ab6b62..88686f90 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1508,20 +1508,23 @@ public void executePersistingGlobalsPersistsSeededCompiledAndUncompiledUserGloba } @Test - public void executePersistingGlobalsDefersRuntimeArgumentAssignmentsUntilInputTraversal() throws Exception { + public void executePersistingGlobalsIgnoresRuntimeArgumentAssignmentsWithCustomInput() throws Exception { + // name=value runtime arguments belong to the ARGV file-list traversal, + // which a custom InputSource replaces entirely: they are never applied, + // so nothing is persisted for them either. AwkProgram seedFromRuntimeArguments = AWK.compile("BEGIN { print x } { x = x } END { print x }"); AwkProgram readValue = AWK.compile("BEGIN { print x }"); try (AVM avm = AWK.createAvm()) { assertEquals( - "\n1\n", + "\n\n", executePersistent( avm, seedFromRuntimeArguments, Collections.singletonList("x=1"), Collections.emptyMap(), new SingleRecordInputSource("record"))); - assertEquals("1\n", executePersistent(avm, readValue)); + assertEquals("\n", executePersistent(avm, readValue)); } } diff --git a/src/test/java/io/jawk/InputSourceTest.java b/src/test/java/io/jawk/InputSourceTest.java index e51afc48..d733fb41 100644 --- a/src/test/java/io/jawk/InputSourceTest.java +++ b/src/test/java/io/jawk/InputSourceTest.java @@ -233,12 +233,15 @@ public void testGetlineIntoVariableMaterializesCurrentRecordBeforeAdvance() thro } @Test - public void testFileListVariableAssignmentsAreAppliedWithInputSource() throws Exception { - awkTest("input source still applies name=value operands") - .script("{ print x }") + public void testFileListVariableAssignmentsAreIgnoredWithInputSource() throws Exception { + // name=value operands belong to the ARGV file-list traversal, which a + // custom InputSource replaces entirely: they are never applied and + // remain visible only as plain ARGV entries. + awkTest("input source ignores name=value operands") + .script("{ print \"[\" x \"]\" ARGV[1] }") .operand("x=42") .withInputSource(new TableInputSource(Collections.singletonList(Collections.singletonList("row")))) - .expectLines("42") + .expectLines("[]x=42") .runAndAssert(); } From b1b4ae912d3dcca78ddf1d4d977a9b0740e8bdfd Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 22:55:38 +0200 Subject: [PATCH 09/11] refactor: carry per-file addresses as tuple-stream properties and tighten the loop Address review feedback on the BEGINFILE/ENDFILE internals: - Drop the SET_ENDFILE_ADDRESS / SET_NEXTFILE_ADDRESS opcodes: the ENDFILE and NEXT_FILE addresses are now properties of AwkTuples, read once by the AVM when it installs the program. Because no tuple may reference them anymore, they are remapped explicitly by the optimizer and seeded as reachability roots so dead-code elimination never removes the sections a runtime nextfile can jump to. - Tighten the per-file loop: the begin_file label now sits on the single NEXT_FILE tuple and the ENDFILE section loops back with one GOTO, removing the second NEXT_FILE tuple (and the two SET tuples) from every per-file program. - Clarify the POSIX gating of gawk-only specials: the parser's isSpecialVariableName and the AVM's isManagedSpecialVariable now express "special in the current mode" positively, replacing the confusing isPosixOrdinary* helpers. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 64 +++++------ src/main/java/io/jawk/frontend/AwkParser.java | 54 ++++----- .../java/io/jawk/intermediate/AwkTuples.java | 104 +++++++++++++++--- .../java/io/jawk/intermediate/Opcode.java | 27 +---- 4 files changed, 145 insertions(+), 104 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 777706c3..943d06a7 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -589,15 +589,16 @@ private void initExtensions() { private Address exitAddress = null; /** - * Address of the ENDFILE section, registered by SET_ENDFILE_ADDRESS when - * the program requires per-file input stepping (BEGINFILE/ENDFILE rules - * or a {@code nextfile} statement); {@code null} otherwise. + * Address of the ENDFILE section, read from the compiled program when it + * has BEGINFILE/ENDFILE rules; {@code null} otherwise. */ private Address endFileAddress = null; /** - * Address of the NEXT_FILE tuple, registered by SET_NEXTFILE_ADDRESS when - * the program requires per-file input stepping; {@code null} otherwise. + * Address of the NEXT_FILE tuple that opens each input file, read from + * the compiled program when it requires per-file input stepping + * (BEGINFILE/ENDFILE rules or a {@code nextfile} statement); + * {@code null} otherwise. */ private Address nextFileAddress = null; @@ -744,6 +745,8 @@ private void installProgramMetadata(AwkProgram compiledProgram) { globalVariableArrays = compiledProgram.getGlobalVariableAarrayMap(); functionNames = compiledProgram.getFunctionNameSet(); sourceDescription = compiledProgram.getSourceDescription(); + endFileAddress = compiledProgram.getEndFileAddress(); + nextFileAddress = compiledProgram.getNextFileAddress(); } private void rebindResolvedInputSource(InputSource resolvedSource) { @@ -1019,26 +1022,24 @@ private boolean isPersistentEligibleGlobal(String name) { } /** - * Returns whether the name is a JRT-managed special variable for this - * execution. ERRNO and ARGIND are gawk extensions: in POSIX mode they are - * ordinary global variables, as in {@code gawk --posix}. + * Returns whether the named variable is a JRT-managed special variable in + * the current execution mode. Most special variables (NR, FS, FILENAME, + * ...) always are. The gawk-only ERRNO and ARGIND are special outside + * POSIX mode only: with {@code --posix} they are ordinary global + * variables, exactly like {@code gawk --posix} treats them. * * @param name variable name to inspect - * @return {@code true} when the variable is managed by the JRT + * @return {@code true} when reads and writes of the variable go through + * the JRT instead of a global slot */ private boolean isManagedSpecialVariable(String name) { - return JRT.isJrtManagedSpecialVariable(name) && !isPosixOrdinaryVariable(name); - } - - /** - * Returns whether the name is a gawk-only special variable that POSIX - * mode treats as an ordinary global. - * - * @param name variable name to inspect - * @return {@code true} when the name is ordinary in the current mode - */ - private boolean isPosixOrdinaryVariable(String name) { - return settings.isPosix() && JRT.isGawkOnlySpecialVariable(name); + if (!JRT.isJrtManagedSpecialVariable(name)) { + return false; + } + if (JRT.isGawkOnlySpecialVariable(name)) { + return !settings.isPosix(); + } + return true; } /** @@ -2314,18 +2315,6 @@ private void executeTuples(PositionTracker position) position.next(); break; } - case SET_ENDFILE_ADDRESS: { - // arg[0] = address of the ENDFILE section - endFileAddress = tuple.getAddress(); - position.next(); - break; - } - case SET_NEXTFILE_ADDRESS: { - // arg[0] = address of the NEXT_FILE tuple - nextFileAddress = tuple.getAddress(); - position.next(); - break; - } case SET_WITHIN_END_BLOCKS: { // arg[0] = whether within the END blocks section BooleanTuple endBlocksTuple = (BooleanTuple) tuple; @@ -3424,13 +3413,13 @@ public final Object getVariable(String name) { case "IGNORECASE": return jrt.getIGNORECASEVar(); case "ERRNO": - if (!settings.isPosix()) { + if (isManagedSpecialVariable(name)) { return jrt.getERRNO(); } // POSIX mode: ordinary global, answered by the slot lookup below break; case "ARGIND": - if (!settings.isPosix()) { + if (isManagedSpecialVariable(name)) { return jrt.getARGIND(); } // POSIX mode: ordinary global, answered by the slot lookup below @@ -3500,9 +3489,8 @@ public final void assignVariable(String name, Object obj) { // FS=: between input files) must reach the JRT, not a global slot. // ARGC is excluded: JRT.setARGC delegates back to this method, and its // authoritative storage is the compiled slot below. - if (!"ARGC".equals(name) - && !isPosixOrdinaryVariable(name) - && jrt.applySpecialVariable(name, normalized)) { + if (!"ARGC".equals(name) && isManagedSpecialVariable(name)) { + jrt.applySpecialVariable(name, normalized); updateSymtabEntry(name, normalized); return; } diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 8ceb7fc1..134085fd 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -3040,16 +3040,15 @@ public int populateTuples(AwkTuples tuples) { boolean perFileScaffolding = reqInput && (hasBeginFileRules || hasEndFileRules || nextfileEncountered); - // The per-file addresses must be registered before the BEGIN rules - // run, so a nextfile executed from a user-defined function can - // resolve them at runtime. + // The per-file addresses are carried as properties of the tuple + // stream, so a nextfile executed from a user-defined function can + // resolve them at runtime. The begin_file address labels the + // NEXT_FILE tuple that opens each input file. Address beginFileAddress = null; Address endFileAddress = null; - Address nextFileAddress = null; if (perFileScaffolding) { beginFileAddress = tuples.createAddress("begin_file"); endFileAddress = tuples.createAddress("end_file"); - nextFileAddress = tuples.createAddress("next_file"); // The ENDFILE address is registered only when BEGINFILE or // ENDFILE rules exist: its presence also confines a // non-redirected getline to the current input file, which @@ -3057,7 +3056,7 @@ public int populateTuples(AwkTuples tuples) { if (hasBeginFileRules || hasEndFileRules) { tuples.setEndFileAddress(endFileAddress); } - tuples.setNextFileAddress(nextFileAddress); + tuples.setNextFileAddress(beginFileAddress); } // grab all BEGINs @@ -3076,12 +3075,13 @@ public int populateTuples(AwkTuples tuples) { Address noMoreInput = tuples.createAddress("no_more_input"); if (perFileScaffolding) { - // Advance to the first input file so the BEGINFILE rules - // observe its FILENAME, FNR, ARGIND, and ERRNO. + // Advance to the next input file so the BEGINFILE rules + // observe its FILENAME, FNR, ARGIND, and ERRNO. The + // ENDFILE section loops back here for the following files. + tuples.address(beginFileAddress); tuples.nextFile(noMoreInput); // BEGINFILE rules, in the order they were read - tuples.address(beginFileAddress); ptr = this; while (ptr != null) { if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) { @@ -3123,10 +3123,9 @@ public int populateTuples(AwkTuples tuples) { ptr = ptr.getAst2(); } - // then move on to the next input file, or fall through to - // the END rules once the input is exhausted - tuples.address(nextFileAddress); - tuples.nextFile(noMoreInput); + // then loop back to the NEXT_FILE tuple at begin_file, + // which falls through to the END rules once the input is + // exhausted tuples.gotoAddress(beginFileAddress); } @@ -4699,7 +4698,7 @@ public void semanticAnalysis() throws SemanticException { FunctionDefParamListAst ptr = this; while (ptr != null) { - if (SPECIAL_VAR_NAMES.get(ptr.id) != null && !isPosixOrdinarySpecialName(ptr.id)) { + if (isSpecialVariableName(ptr.id)) { throw new SemanticException("Special variable " + ptr.id + " cannot be used as a formal parameter"); } ptr = (FunctionDefParamListAst) ptr.getAst1(); @@ -5293,23 +5292,28 @@ public int populateTuples(AwkTuples tuples) { * them. */ private boolean isJrtManagedSpecialName(String id) { - return SPECIAL_VAR_NAMES.containsKey(id) - && !"ENVIRON".equals(id) - && !"ARGV".equals(id) - && !isPosixOrdinarySpecialName(id); + return isSpecialVariableName(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id); } /** - * Returns whether the name is a gawk-only special variable that POSIX - * mode treats as an ordinary identifier. Like {@code gawk --posix}, POSIX - * mode compiles ERRNO and ARGIND as plain global variables, usable as - * function parameters and untouched by the input machinery. + * Returns whether the identifier names a special variable in the current + * compile-time mode. Most special names (NR, FS, FILENAME, ...) always + * are. The gawk-only ERRNO and ARGIND are special outside POSIX mode + * only: with {@code --posix} they are plain identifiers — usable as + * function parameters and compiled as ordinary globals — exactly like + * {@code gawk --posix} treats them. * * @param id the identifier to inspect - * @return {@code true} when the name is ordinary in the current mode + * @return {@code true} when the name is special in the current mode */ - private boolean isPosixOrdinarySpecialName(String id) { - return posix && JRT.isGawkOnlySpecialVariable(id); + private boolean isSpecialVariableName(String id) { + if (!SPECIAL_VAR_NAMES.containsKey(id)) { + return false; + } + if (JRT.isGawkOnlySpecialVariable(id)) { + return !posix; + } + return true; } /** Emits the tuple pushing the value of a JRT-managed special variable. */ diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 03c0aa59..5fb5e882 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -111,6 +111,19 @@ public boolean add(Tuple t) { /** Whether this tuple stream was produced by {@code compileExpression()}. */ private boolean evalTupleStream; + /** + * Address of the ENDFILE section, or {@code null} when the program has + * no BEGINFILE/ENDFILE rules. Created addresses are remapped by the + * optimizer through the address manager, so this stays valid. + */ + private Address endFileAddress; + + /** + * Address of the {@code NEXT_FILE} tuple that opens each input file, or + * {@code null} when the program does not use per-file input stepping. + */ + private Address nextFileAddress; + /** *

* toOpcodeString. @@ -1775,24 +1788,49 @@ public void setWithinEndBlocks(boolean b) { } /** - * Emits the tuple registering the address of the ENDFILE section, so that - * a runtime {@code nextfile} statement can jump to it. + * Registers the address of the ENDFILE section, so that a runtime + * {@code nextfile} statement can jump to it. A property of the tuple + * stream rather than a tuple: the interpreter reads it once when it + * installs the program. * * @param addr address of the ENDFILE section */ public void setEndFileAddress(Address addr) { - queue.add(new Tuple.AddressTuple(Opcode.SET_ENDFILE_ADDRESS, addr)); + endFileAddress = addr; + } + + /** + * Returns the address of the ENDFILE section, or {@code null} when the + * program has no BEGINFILE/ENDFILE rules. + * + * @return address of the ENDFILE section, or {@code null} + */ + public Address getEndFileAddress() { + return endFileAddress; } /** - * Emits the tuple registering the address of the {@code NEXT_FILE} tuple, - * so that a runtime {@code nextfile} statement can bypass the ENDFILE - * rules for input files that could not be opened. + * Registers the address of the {@code NEXT_FILE} tuple that opens each + * input file, so that a runtime {@code nextfile} statement can bypass + * the ENDFILE rules for input files that could not be opened. A property + * of the tuple stream rather than a tuple: the interpreter reads it once + * when it installs the program. * * @param addr address of the NEXT_FILE tuple */ public void setNextFileAddress(Address addr) { - queue.add(new Tuple.AddressTuple(Opcode.SET_NEXTFILE_ADDRESS, addr)); + nextFileAddress = addr; + } + + /** + * Returns the address of the {@code NEXT_FILE} tuple that opens each + * input file, or {@code null} when the program does not use per-file + * input stepping. + * + * @return address of the NEXT_FILE tuple, or {@code null} + */ + public Address getNextFileAddress() { + return nextFileAddress; } /** @@ -2398,21 +2436,45 @@ private void remapAddresses(int[] indexMapping) { } Set

processedAddresses = Collections.newSetFromMap(new IdentityHashMap()); for (Tuple tuple : queue) { - Address address = tuple.getAddress(); - if (address != null && processedAddresses.add(address)) { - int oldIndex = address.index(); - if (oldIndex >= 0 && oldIndex < indexMapping.length) { - int mappedIndex = indexMapping[oldIndex]; - if (mappedIndex < 0) { - throw new Error("Address " + address + " references removed tuple " + oldIndex); - } - address.assignIndex(mappedIndex); - } - } + remapAddress(tuple.getAddress(), indexMapping, processedAddresses); } + // The per-file property addresses may not be referenced by any tuple + // (e.g. after jump threading rewired the loop-back GOTO), so they + // must be remapped explicitly to stay valid. + remapAddress(endFileAddress, indexMapping, processedAddresses); + remapAddress(nextFileAddress, indexMapping, processedAddresses); addressManager.remapIndexes(indexMapping); } + private static void seedPropertyAddress( + Address address, + int size, + boolean[] reachable, + Deque worklist) { + if (address == null) { + return; + } + int targetIndex = address.index(); + if (targetIndex >= 0 && targetIndex < size && !reachable[targetIndex]) { + reachable[targetIndex] = true; + worklist.addLast(targetIndex); + } + } + + private static void remapAddress(Address address, int[] indexMapping, Set
processedAddresses) { + if (address == null || !processedAddresses.add(address)) { + return; + } + int oldIndex = address.index(); + if (oldIndex >= 0 && oldIndex < indexMapping.length) { + int mappedIndex = indexMapping[oldIndex]; + if (mappedIndex < 0) { + throw new Error("Address " + address + " references removed tuple " + oldIndex); + } + address.assignIndex(mappedIndex); + } + } + private void reprocessQueue() { assignSequentialNextPointers(); for (Tuple tuple : queue) { @@ -2592,6 +2654,12 @@ private void optimizeQueue() { reachable[0] = true; worklist.add(0); } + // The per-file property addresses are runtime jump targets (nextfile, + // and the ENDFILE section it resumes at) that no tuple may reference: + // treat them as reachability roots so their sections are never + // eliminated as dead code. + seedPropertyAddress(endFileAddress, size, reachable, worklist); + seedPropertyAddress(nextFileAddress, size, reachable, worklist); while (!worklist.isEmpty()) { int index = worklist.removeFirst(); diff --git a/src/main/java/io/jawk/intermediate/Opcode.java b/src/main/java/io/jawk/intermediate/Opcode.java index c95510ad..5af45493 100644 --- a/src/main/java/io/jawk/intermediate/Opcode.java +++ b/src/main/java/io/jawk/intermediate/Opcode.java @@ -1573,34 +1573,15 @@ public enum Opcode { * file and resumes the per-file input loop. The runtime jumps to the * ENDFILE rules when the current file was opened successfully, or * directly past them when the file could not be opened (BEGINFILE error - * handling). The runtime stack and operand stack are cleared, allowing - * {@code nextfile} to be invoked from user-defined functions. + * handling). The jump targets are carried as properties of the tuple + * stream, not as tuples. The runtime stack and operand stack are + * cleared, allowing {@code nextfile} to be invoked from user-defined + * functions. *

* Stack after: (empty) */ EXEC_NEXTFILE, - /** - * Internal. Registers the address of the ENDFILE section so that a - * runtime {@code nextfile} can jump to it. - *

- * Argument: address of the ENDFILE section - *

- * Stack remains unchanged. - */ - SET_ENDFILE_ADDRESS, - - /** - * Internal. Registers the address of the {@link #NEXT_FILE} tuple so that - * a runtime {@code nextfile} can bypass the ENDFILE rules for input files - * that could not be opened. - *

- * Argument: address of the NEXT_FILE tuple - *

- * Stack remains unchanged. - */ - SET_NEXTFILE_ADDRESS, - /** * Assigns the top of the stack to ERRNO, managed by the JRT. *

From 95e51f28d662d01e30afa9c3a8b6220c23073703 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 23:04:03 +0200 Subject: [PATCH 10/11] refactor: carry the exit address as a tuple-stream property too Drop the SET_EXIT_ADDRESS opcode: like the ENDFILE/NEXT_FILE addresses, the END-blocks address is now a property of AwkTuples, read once by the AVM when it installs the program, remapped explicitly by the optimizer, and seeded as a reachability root so the END section survives dead-code elimination even when only a runtime exit can reach it. The intermediate serialVersionUID is bumped to 3; precompiled tuple files remain tied to the Jawk version that produced them. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 11 +++-- .../java/io/jawk/intermediate/AwkTuples.java | 44 ++++++++++++++----- .../java/io/jawk/intermediate/Opcode.java | 7 --- .../io/jawk/AwkTupleOptimizationTest.java | 1 - 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 943d06a7..4b03484c 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -586,6 +586,10 @@ private void initExtensions() { */ private int oldseed = 1; + /** + * Address of the END blocks section, read from the compiled program; + * {@code null} for expression streams. + */ private Address exitAddress = null; /** @@ -745,6 +749,7 @@ private void installProgramMetadata(AwkProgram compiledProgram) { globalVariableArrays = compiledProgram.getGlobalVariableAarrayMap(); functionNames = compiledProgram.getFunctionNameSet(); sourceDescription = compiledProgram.getSourceDescription(); + exitAddress = compiledProgram.getExitAddress(); endFileAddress = compiledProgram.getEndFileAddress(); nextFileAddress = compiledProgram.getNextFileAddress(); } @@ -2309,12 +2314,6 @@ private void executeTuples(PositionTracker position) position.next(); break; } - case SET_EXIT_ADDRESS: { - // arg[0] = exit address - exitAddress = tuple.getAddress(); - position.next(); - break; - } case SET_WITHIN_END_BLOCKS: { // arg[0] = whether within the END blocks section BooleanTuple endBlocksTuple = (BooleanTuple) tuple; diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 5fb5e882..f8b536c7 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -50,7 +50,7 @@ */ public class AwkTuples implements Serializable { - private static final long serialVersionUID = 2L; + private static final long serialVersionUID = 3L; /** Address manager */ private final AddressManager addressManager = new AddressManager(); @@ -111,10 +111,17 @@ public boolean add(Tuple t) { /** Whether this tuple stream was produced by {@code compileExpression()}. */ private boolean evalTupleStream; + /** + * Address of the END blocks section, where a runtime {@code exit} jumps; + * {@code null} for expression streams. Property addresses are remapped + * explicitly by the optimizer and seeded as reachability roots, so they + * stay valid even when no tuple references them. + */ + private Address exitAddress; + /** * Address of the ENDFILE section, or {@code null} when the program has - * no BEGINFILE/ENDFILE rules. Created addresses are remapped by the - * optimizer through the address manager, so this stays valid. + * no BEGINFILE/ENDFILE rules. */ private Address endFileAddress; @@ -1766,14 +1773,25 @@ public void deleteArray(int offset, boolean isGlobal) { } /** - *

- * setExitAddress. - *

+ * Registers the address of the END blocks section, so that a runtime + * {@code exit} statement can jump to it. A property of the tuple stream + * rather than a tuple: the interpreter reads it once when it installs + * the program. * - * @param addr a {@link io.jawk.intermediate.Address} object + * @param addr address of the END blocks section */ public void setExitAddress(Address addr) { - queue.add(new Tuple.AddressTuple(Opcode.SET_EXIT_ADDRESS, addr)); + exitAddress = addr; + } + + /** + * Returns the address of the END blocks section, or {@code null} when + * the tuple stream is an expression stream with no END blocks. + * + * @return address of the END blocks section, or {@code null} + */ + public Address getExitAddress() { + return exitAddress; } /** @@ -2438,9 +2456,10 @@ private void remapAddresses(int[] indexMapping) { for (Tuple tuple : queue) { remapAddress(tuple.getAddress(), indexMapping, processedAddresses); } - // The per-file property addresses may not be referenced by any tuple - // (e.g. after jump threading rewired the loop-back GOTO), so they - // must be remapped explicitly to stay valid. + // Property addresses may not be referenced by any tuple (e.g. after + // jump threading rewired the loop-back GOTO), so they must be + // remapped explicitly to stay valid. + remapAddress(exitAddress, indexMapping, processedAddresses); remapAddress(endFileAddress, indexMapping, processedAddresses); remapAddress(nextFileAddress, indexMapping, processedAddresses); addressManager.remapIndexes(indexMapping); @@ -2654,10 +2673,11 @@ private void optimizeQueue() { reachable[0] = true; worklist.add(0); } - // The per-file property addresses are runtime jump targets (nextfile, + // The property addresses are runtime jump targets (exit, nextfile, // and the ENDFILE section it resumes at) that no tuple may reference: // treat them as reachability roots so their sections are never // eliminated as dead code. + seedPropertyAddress(exitAddress, size, reachable, worklist); seedPropertyAddress(endFileAddress, size, reachable, worklist); seedPropertyAddress(nextFileAddress, size, reachable, worklist); diff --git a/src/main/java/io/jawk/intermediate/Opcode.java b/src/main/java/io/jawk/intermediate/Opcode.java index 5af45493..80fadff2 100644 --- a/src/main/java/io/jawk/intermediate/Opcode.java +++ b/src/main/java/io/jawk/intermediate/Opcode.java @@ -1279,13 +1279,6 @@ public enum Opcode { */ DELETE_MAP_ELEMENT, - /** - * Internal. - *

- * Stack remains unchanged. - */ - SET_EXIT_ADDRESS, - /** * Internal. *

diff --git a/src/test/java/io/jawk/AwkTupleOptimizationTest.java b/src/test/java/io/jawk/AwkTupleOptimizationTest.java index 8c55109d..2cec4e74 100644 --- a/src/test/java/io/jawk/AwkTupleOptimizationTest.java +++ b/src/test/java/io/jawk/AwkTupleOptimizationTest.java @@ -731,7 +731,6 @@ private static boolean usesAddress(Opcode opcode) { case IS_EMPTY_KEYLIST: case CONSUME_INPUT: case CALL_FUNCTION: - case SET_EXIT_ADDRESS: return true; default: return false; From 2f498a0fb959fef0b0b624e87125b812ce0e732c Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Fri, 17 Jul 2026 23:24:22 +0200 Subject: [PATCH 11/11] refactor: materialize ARGV with canonical Long keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the deliberate Integer-key trick in populateArgv: ARGV entries now always use Long keys, the canonical AWK array key form, on both the materialization and the synthetic-accessor paths. The precedence of a host-supplied ARGV over the operand list is now explicit — populateArgv leaves a non-empty injected map untouched — instead of depending on the key-type mismatch, which only protected plain maps with Long keys and silently clobbered Integer-keyed or AssocArray-backed injections. A new test pins the now key-type-independent behavior. Co-Authored-By: Claude Fable 5 --- src/main/java/io/jawk/backend/AVM.java | 33 +++++++++---------- .../java/io/jawk/StreamInputSourceTest.java | 19 +++++++++++ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 4b03484c..e3b510ff 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -40,6 +40,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; import java.util.regex.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.jawk.AwkExpression; @@ -2947,35 +2948,31 @@ private void populateArgc(long offset) { private void populateArgv(long offset) { argvOffset = offset; - // Integer keys, deliberately: a host-injected plain ARGV map uses - // Long keys, and the ARGV traversal gives those precedence, so the - // injected entries must not be overwritten here. + // A host-supplied ARGV takes precedence over the operand list: leave + // it untouched instead of overwriting its entries. + Object existing = runtimeStack.getVariable(argvOffset, true); + if (existing instanceof Map && !((Map) existing).isEmpty()) { + return; + } forEachArgvEntry((index, value) -> { - assignArray(argvOffset, Integer.valueOf(index), value, true); + assignArray(argvOffset, index, value, true); pop(); // clean up the stack after the assignment }); } - /** - * Receives one ARGV entry from {@link #forEachArgvEntry(ArgvEntryConsumer)}. - */ - @FunctionalInterface - private interface ArgvEntryConsumer { - void accept(int index, Object value); - } - /** * Supplies the ARGV entries to the given consumer, in index order: * ARGV[0] is the program name, ARGV[1..n] the command-line arguments. - * The count comes straight from the argument list because ARGC may not - * be materialized when the script does not reference it. + * Indexes are supplied as {@code Long}, the canonical AWK array key + * form. The count comes straight from the argument list because ARGC + * may not be materialized when the script does not reference it. * * @param consumer receives each (index, value) ARGV entry */ - private void forEachArgvEntry(ArgvEntryConsumer consumer) { - consumer.accept(0, "jawk"); + private void forEachArgvEntry(BiConsumer consumer) { + consumer.accept(Long.valueOf(0L), "jawk"); for (int i = 1; i <= arguments.size(); i++) { - consumer.accept(i, jrt.toInputScalar(arguments.get(i - 1))); + consumer.accept(Long.valueOf(i), jrt.toInputScalar(arguments.get(i - 1))); } } @@ -3648,7 +3645,7 @@ public void setFILENAME(String filename) { public Object getARGV() { if (argvOffset == NULL_OFFSET) { Map argv = newAwkArray(); - forEachArgvEntry((index, value) -> argv.put(Long.valueOf(index), value)); + forEachArgvEntry(argv::put); return argv; } return runtimeStack.getVariable(argvOffset, true); diff --git a/src/test/java/io/jawk/StreamInputSourceTest.java b/src/test/java/io/jawk/StreamInputSourceTest.java index 55ee55ce..a1b9eddb 100644 --- a/src/test/java/io/jawk/StreamInputSourceTest.java +++ b/src/test/java/io/jawk/StreamInputSourceTest.java @@ -84,6 +84,25 @@ public void testFileInputThroughInjectedLongKeyArgvMap() throws Exception { .runAndAssert(); } + @Test + public void testFileInputThroughInjectedIntegerKeyArgvMap() throws Exception { + // A host-supplied ARGV wins over the operand list regardless of the + // numeric key type it uses. + Path file = Files.createTempFile(AwkTestSupport.sharedTempDirectory(), "argv-int-", ".txt"); + Files.write(file, "mapped".getBytes(StandardCharsets.UTF_8)); + Map argv = new LinkedHashMap<>(); + argv.put(0, "jawk"); + argv.put(1, file.toString()); + + awkTest("StreamInputSource reads injected ARGV map with Integer keys") + .script("BEGIN { _ = ARGV[0] } { print $0 }") + .preassign("ARGV", argv) + .file("operand.txt", "operand\n") + .operand("{{operand.txt}}") + .expectLines("mapped") + .runAndAssert(); + } + @Test public void testMultipleFilesWithFnrReset() throws Exception { awkTest("StreamInputSource resets FNR per file")