diff --git a/README.md b/README.md index d8101178..12f3aafa 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,17 @@ Jawk is a pure Java implementation of [AWK](https://en.wikipedia.org/wiki/AWK). You can run it as a CLI, embed it directly in Java applications, compile scripts to reusable tuples, evaluate AWK expressions, feed it structured input, load extensions explicitly, and enable a sandboxed runtime when you need tighter execution constraints. -Gawk-specific builtins — `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, and `PROCINFO["sorted_in"]`-controlled array traversal — are available by default through the built-in GNU Awk compatibility extension. +## Support for POSIX AWK and Gawk + +Jawk fully implements POSIX AWK, and adds support for the most commonly used gawk-specific features: + +- Builtins, available by default through the built-in GNU Awk compatibility extension: `asort()`, `asorti()`, `typeof()`, `isarray()`, `mkbool()`, `gensub()`, and `PROCINFO["sorted_in"]`-controlled array traversal +- Arrays of arrays (`a[i][j]`) and typed regexp literals (`@/re/`) +- `BEGINFILE` / `ENDFILE` special patterns, with the `ERRNO` and `ARGIND` special variables, so a script can hook into the command-line file processing loop and skip unreadable files without a fatal error +- The `nextfile` statement +- The `IGNORECASE`, `SYMTAB`, and `FUNCTAB` special variables + +As in gawk, the gawk-specific syntax is not special in `--posix` mode. ## CLI Example diff --git a/src/it/java/io/jawk/gawk/GawkExtensionIT.java b/src/it/java/io/jawk/gawk/GawkExtensionIT.java index 650d6ff4..61836a34 100644 --- a/src/it/java/io/jawk/gawk/GawkExtensionIT.java +++ b/src/it/java/io/jawk/gawk/GawkExtensionIT.java @@ -22,6 +22,12 @@ * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ */ +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import io.jawk.AwkTestSupport; import org.junit.Test; @@ -1553,12 +1559,167 @@ public void test_mixed1() throws Exception { @Test public void test_beginfile1() throws Exception { - skip("BEGINFILE and ENDFILE are not implemented by Jawk yet."); + // Upstream: $AWK -f beginfile1.awk "$(srcdir)"/beginfile1.awk . ./no/such/file Makefile + // The script strips directories from FILENAME with gsub(/.*[/]/, ""), + // so operands are passed with forward slashes. "Makefile" is not part + // of the vendored fixtures (it is generated by gawk's configure), so + // an equivalent two-record file is materialized on the fly. + Path makefile = AwkTestSupport.sharedTempDirectory().resolve("Makefile"); + Files.write(makefile, "# first record\n# second record\n".getBytes(StandardCharsets.UTF_8)); + AwkTestSupport + .cliTest("GAWK beginfile1") + .argument("-f", gawkFile("beginfile1.awk")) + .operand( + slashed(gawkPath("beginfile1.awk")), + ".", + "./no/such/file", + slashed(makefile)) + .expectLines(gawkPath("beginfile1.ok")) + .expectExit(0) + .runAndAssert(); + } + + // The gawk beginfile2 target is a shell script running 16 sub-tests; the + // reproducible ones are transcribed below as test_beginfile2_test*. The + // remaining sub-tests rely on non-redirected getline in BEGIN/END driving + // the BEGINFILE/ENDFILE hooks mid-statement, or compare non-zero gawk CLI + // transcripts, which Jawk does not reproduce. + + @Test + public void test_beginfile2_test2() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 2--") + .argument(beginfile2Program(2)) + .operand(slashed(gawkPath("beginfile2.in")), "file/does/not/exist") + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("2")) + .expectExit(0) + .runAndAssert(); } @Test - public void test_beginfile2() throws Exception { - skip("BEGINFILE and ENDFILE are not implemented by Jawk yet."); + public void test_beginfile2_test5() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 5--") + .argument(beginfile2Program(5)) + .operand(slashed(gawkPath("beginfile2.in"))) + .expectLines(beginfile2ExpectedLines("5")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test9a() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 9a--") + .argument(beginfile2Program(9)) + .operand("file/does/not/exist", slashed(gawkPath("beginfile2.in"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("9a")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test9b() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 9b--") + .preassign("skip", 1) + .argument(beginfile2Program(9)) + .operand("file/does/not/exist", slashed(gawkPath("beginfile2.in"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("9b")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test12() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 12--") + .argument(beginfile2Program(12)) + .operand(slashed(gawkPath("beginfile2.in")), slashed(gawkPath("beginfile2.sh"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("12")) + .expectExit(0) + .runAndAssert(); + } + + @Test + public void test_beginfile2_test15() throws Exception { + AwkTestSupport + .cliTest("GAWK beginfile2 --Test 15--") + .argument(beginfile2Program(15)) + .operand(slashed(gawkPath("beginfile2.in")), slashed(gawkPath("beginfile2.sh"))) + .postProcessWith(GawkExtensionIT::stripGawkDirectory) + .expectLines(beginfile2ExpectedLines("15")) + .expectExit(0) + .runAndAssert(); + } + + /** + * Extracts one sub-test program from the vendored {@code beginfile2.in}, + * exactly like {@code beginfile2.sh} does with its awk range patterns. + * The {@code #TESTn#} marker lines are comments and stay harmless. + * + * @param testNumber sub-test number + * @return the AWK program of that sub-test + */ + private static String beginfile2Program(int testNumber) throws IOException { + String text = gawkText("beginfile2.in"); + String beginMarker = "#TEST" + testNumber + "#"; + String endMarker = "#TEST" + (testNumber + 1) + "#"; + return text.substring(text.indexOf(beginMarker), text.indexOf(endMarker) + endMarker.length()); + } + + /** + * Extracts the expected transcript of one beginfile2 sub-test from the + * vendored {@code beginfile2.ok}, i.e. the lines between the + * {@code --Test N--} banner and the next banner. + * + * @param testLabel sub-test label as it appears in the banner (e.g. "9a") + * @return the expected output lines of that sub-test + */ + private static List beginfile2ExpectedLines(String testLabel) throws IOException { + List lines = Files.readAllLines(gawkPath("beginfile2.ok"), StandardCharsets.UTF_8); + List section = new ArrayList<>(); + boolean inSection = false; + for (String line : lines) { + if (line.startsWith("--Test ")) { + if (inSection) { + break; + } + inSection = line.equals("--Test " + testLabel + "--"); + continue; + } + if (inSection) { + section.add(line); + } + } + return section; + } + + /** + * Removes the vendored gawk directory prefix from FILENAME occurrences in + * the transcript, so it can be compared with the {@code .ok} files that + * gawk produces while running inside that directory. + * + * @param output raw transcript + * @return transcript with bare fixture file names + */ + private static String stripGawkDirectory(String output) { + return output.replace(slashed(GAWK_DIRECTORY) + "/", ""); + } + + /** + * Returns the path as a string with forward slashes, so FILENAME-based + * basename extraction with {@code gsub(/.*[/]/, "")} works on Windows too. + * + * @param path path to convert + * @return the path with forward slashes + */ + private static String slashed(Path path) { + return path.toString().replace('\\', '/'); } @Test diff --git a/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java b/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java index 076b4482..2afb5737 100644 --- a/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java +++ b/src/it/java/io/jawk/gawk/GawkOptionalFeatureIT.java @@ -729,7 +729,10 @@ public void test_testext() throws Exception { @Test public void test_getfile() throws Exception { - skip(MANUAL_SKIP_REASON); + skip( + "Requires gawk's dynamic extension API (the get_file() input hand-off used by gawkextlib)," + + " which Jawk does not provide. The BEGINFILE/ENDFILE rules the fixture also exercises" + + " are covered by GawkExtensionIT.test_beginfile1 and test_beginfile2_test*."); } @Test diff --git a/src/main/java/io/jawk/Cli.java b/src/main/java/io/jawk/Cli.java index da0891a7..e56ba8ee 100644 --- a/src/main/java/io/jawk/Cli.java +++ b/src/main/java/io/jawk/Cli.java @@ -645,7 +645,10 @@ private static void usage(PrintStream dest) { .println( " -S, --sandbox = (extension) Enable sandbox mode (no system(), redirection, pipelines, or" + " dynamic extensions)."); - dest.println(" --posix = Enforce POSIX-compatible behavior such as disabling nested arrays."); + dest + .println( + " --posix = Enforce POSIX-compatible behavior such as disabling nested arrays and" + + " BEGINFILE/ENDFILE rules."); dest.println(" --dump-syntax = Print the syntax tree."); dest.println(" --dump-intermediate = Print the intermediate code."); dest.println(" -s, --no-optimize = (extension) Disable optimizations during compilation."); diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 27314790..e3b510ff 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -40,6 +40,7 @@ import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.function.BiConsumer; import java.util.regex.Pattern; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.jawk.AwkExpression; @@ -78,7 +79,6 @@ import io.jawk.jrt.BlockObject; import io.jawk.jrt.ConditionPair; import io.jawk.jrt.InputSource; -import io.jawk.jrt.StreamInputSource; import io.jawk.jrt.JRT; import io.jawk.jrt.VariableManager; import io.jawk.util.AwkSettings; @@ -150,7 +150,6 @@ private void push(Object o) { private final Map tupleProfilingStats; private final Map functionProfilingStats; private final Deque activeProfilingFunctions; - private boolean inputSourceFilelistAssignmentsApplied; private InputSource resolvedInputSource; private AwkExpression installedEvalExpression; private boolean mergedGlobalLayoutActive; @@ -588,8 +587,44 @@ private void initExtensions() { */ private int oldseed = 1; + /** + * Address of the END blocks section, read from the compiled program; + * {@code null} for expression streams. + */ private Address exitAddress = null; + /** + * Address of the ENDFILE section, read from the compiled program when it + * has BEGINFILE/ENDFILE rules; {@code null} otherwise. + */ + private Address endFileAddress = null; + + /** + * Address of the NEXT_FILE tuple that opens each input file, read from + * the compiled program when it requires per-file input stepping + * (BEGINFILE/ENDFILE rules or a {@code nextfile} statement); + * {@code null} otherwise. + */ + private Address nextFileAddress = null; + + /** + * true if execution position is within a BEGINFILE rule; + * false otherwise. + */ + private boolean withinBeginFileBlocks = false; + + /** + * true if execution position is within an ENDFILE rule; + * false otherwise. + */ + private boolean withinEndFileBlocks = false; + + /** + * true once the per-file main input loop has advanced to its + * first input file; false while still in the BEGIN blocks. + */ + private boolean inputFileLoopStarted = false; + /** * true if execution position is within an END block; * false otherwise. @@ -678,10 +713,14 @@ private void resetTransientRuntimeState(List runtimeArguments, Map globals) { */ private boolean isPersistentEligibleGlobal(String name) { return name != null - && !JRT.isJrtManagedSpecialVariable(name) + && !isManagedSpecialVariable(name) && !NON_PERSISTENT_GLOBALS.contains(name); } + /** + * Returns whether the named variable is a JRT-managed special variable in + * the current execution mode. Most special variables (NR, FS, FILENAME, + * ...) always are. The gawk-only ERRNO and ARGIND are special outside + * POSIX mode only: with {@code --posix} they are ordinary global + * variables, exactly like {@code gawk --posix} treats them. + * + * @param name variable name to inspect + * @return {@code true} when reads and writes of the variable go through + * the JRT instead of a global slot + */ + private boolean isManagedSpecialVariable(String name) { + if (!JRT.isJrtManagedSpecialVariable(name)) { + return false; + } + if (JRT.isGawkOnlySpecialVariable(name)) { + return !settings.isPosix(); + } + return true; + } + /** * Validates that a seeded global name is compatible with the compiled * metadata of the current program before any value normalization can mutate a @@ -1012,23 +1075,6 @@ private void validateSeededGlobalValue(String name, Object value) { } } - /** - * Parses a runtime {@code name=value} assignment. - * - * @param nameValue raw assignment text - * @return parsed assignment - */ - private NameValueAssignment parseNameValueAssignment(String nameValue) { - int eqIdx = nameValue.indexOf('='); - if (eqIdx == 0) { - throw new IllegalArgumentException( - "Must have a non-blank variable name in a name=value variable assignment argument."); - } - String name = nameValue.substring(0, eqIdx); - String value = nameValue.substring(eqIdx + 1); - return new NameValueAssignment(name, jrt.toInputScalar(value)); - } - /** * Executes the tuple stream after the runtime has been fully prepared. * @@ -2027,7 +2073,6 @@ private void executeTuples(PositionTracker position) case CONSUME_INPUT: { // arg[0] = address // store the next record into $0, $1, ... - applyInputSourceFilelistAssignmentsIfNeeded(); if (jrt.consumeInput(resolvedInputSource)) { position.next(); } else { @@ -2035,16 +2080,47 @@ private void executeTuples(PositionTracker position) } break; } + case CONSUME_FILE_INPUT: { + // arg[0] = address of the ENDFILE section + // store the next record of the current file into $0, $1, ... + withinBeginFileBlocks = false; + if (jrt.consumeCurrentFileInput(resolvedInputSource)) { + position.next(); + } else { + withinEndFileBlocks = true; + position.jump(tuple.getAddress()); + } + break; + } + case NEXT_FILE: { + // arg[0] = address to jump to when no input file remains + inputFileLoopStarted = true; + withinEndFileBlocks = false; + if (jrt.advanceToNextFile(resolvedInputSource)) { + withinBeginFileBlocks = true; + position.next(); + } else { + position.jump(tuple.getAddress()); + } + break; + } + case EXEC_NEXTFILE: { + executeNextfile(position); + break; + } case GETLINE_INPUT: { - applyInputSourceFilelistAssignmentsIfNeeded(); - push(jrt.consumeInput(resolvedInputSource) ? 1 : 0); + checkGetlineAllowed(position); + boolean consumed = isMainInputFileBounded() ? + jrt.consumeCurrentFileInput(resolvedInputSource) : jrt.consumeInput(resolvedInputSource); + push(consumed ? 1 : 0); position.next(); break; } case GETLINE_INPUT_TO_TARGET: { - applyInputSourceFilelistAssignmentsIfNeeded(); - Object input = jrt.consumeInputToTarget(resolvedInputSource); + checkGetlineAllowed(position); + Object input = isMainInputFileBounded() ? + jrt.consumeCurrentFileInputToTarget(resolvedInputSource) : jrt.consumeInputToTarget(resolvedInputSource); if (input != null) { push(1); push(input); @@ -2239,12 +2315,6 @@ private void executeTuples(PositionTracker position) position.next(); break; } - case SET_EXIT_ADDRESS: { - // arg[0] = exit address - exitAddress = tuple.getAddress(); - position.next(); - break; - } case SET_WITHIN_END_BLOCKS: { // arg[0] = whether within the END blocks section BooleanTuple endBlocksTuple = (BooleanTuple) tuple; @@ -2259,6 +2329,8 @@ private void executeTuples(PositionTracker position) exitCode = (int) JRT.toDouble(pop()); } throwExitException = true; + withinBeginFileBlocks = false; + withinEndFileBlocks = false; // If in BEGIN or in a rule, jump to the END section if (!withinEndBlocks && exitAddress != null) { @@ -2527,6 +2599,30 @@ private void executeTuples(PositionTracker position) position.next(); break; } + case ASSIGN_ERRNO: { + Object v = pop(); + jrt.setERRNO(v); + push(v == null ? "" : v); + position.next(); + break; + } + case PUSH_ERRNO: { + push(jrt.getERRNO()); + position.next(); + break; + } + case ASSIGN_ARGIND: { + Object v = pop(); + jrt.setARGIND(v); + push(v == null ? ZERO : v); + position.next(); + break; + } + case PUSH_ARGIND: { + push(jrt.getARGIND()); + position.next(); + break; + } case ASSIGN_SUBSEP: { Object v = pop(); jrt.setSUBSEP(v); @@ -2852,14 +2948,31 @@ private void populateArgc(long offset) { private void populateArgv(long offset) { argvOffset = offset; - // ARGV[0] is the program name, ARGV[1..n] the command-line arguments. - // The count comes straight from the argument list because ARGC may not - // be materialized when the script does not reference it. - assignArray(argvOffset, 0, "jawk", true); - pop(); - for (int i = 1; i <= arguments.size(); i++) { - assignArray(argvOffset, i, jrt.toInputScalar(arguments.get(i - 1)), true); + // A host-supplied ARGV takes precedence over the operand list: leave + // it untouched instead of overwriting its entries. + Object existing = runtimeStack.getVariable(argvOffset, true); + if (existing instanceof Map && !((Map) existing).isEmpty()) { + return; + } + forEachArgvEntry((index, value) -> { + assignArray(argvOffset, index, value, true); pop(); // clean up the stack after the assignment + }); + } + + /** + * Supplies the ARGV entries to the given consumer, in index order: + * ARGV[0] is the program name, ARGV[1..n] the command-line arguments. + * Indexes are supplied as {@code Long}, the canonical AWK array key + * form. The count comes straight from the argument list because ARGC + * may not be materialized when the script does not reference it. + * + * @param consumer receives each (index, value) ARGV entry + */ + private void forEachArgvEntry(BiConsumer consumer) { + consumer.accept(Long.valueOf(0L), "jawk"); + for (int i = 1; i <= arguments.size(); i++) { + consumer.accept(Long.valueOf(i), jrt.toInputScalar(arguments.get(i - 1))); } } @@ -3048,7 +3161,6 @@ public void close() throws IOException { jrt.jrtCloseAll(); closeResolvedInputSource(); resolvedInputSource = null; - inputSourceFilelistAssignmentsApplied = false; } /** @@ -3246,6 +3358,8 @@ public Set getFunctionNames() { "RSTART", "RLENGTH", "IGNORECASE", + "ERRNO", + "ARGIND", "ARGC", "ARGV"))); @@ -3294,6 +3408,18 @@ public final Object getVariable(String name) { return jrt.getRLENGTH(); case "IGNORECASE": return jrt.getIGNORECASEVar(); + case "ERRNO": + if (isManagedSpecialVariable(name)) { + return jrt.getERRNO(); + } + // POSIX mode: ordinary global, answered by the slot lookup below + break; + case "ARGIND": + if (isManagedSpecialVariable(name)) { + return jrt.getARGIND(); + } + // POSIX mode: ordinary global, answered by the slot lookup below + break; // lazily-materialized globals answered through their synthetic accessors case "ARGC": return getARGC(); @@ -3335,21 +3461,6 @@ public int getCurrentLineNumber() { return currentLineNumber; } - /** - * Performs the global variable assignment within the runtime environment. - * These assignments come from the ARGV list (bounded by ARGC), which, in - * turn, come from the command-line arguments passed into Awk. - * - * @param nameValue The variable assignment in name=value form. - */ - @SuppressWarnings("unused") - private void setFilelistVariable(String nameValue) { - // route through assignVariable so JRT-managed specials, global slots, - // and SYMTAB updates are handled in exactly one place - NameValueAssignment assignment = parseNameValueAssignment(nameValue); - assignVariable(assignment.name, assignment.value); - } - /** {@inheritDoc} */ @Override public final void assignVariable(String name, Object obj) { @@ -3358,7 +3469,7 @@ public final void assignVariable(String name, Object obj) { if (globalVariableOffsets == null || globalVariableArrays == null) { Object normalized = normalizeExternalVariableValue(obj); baseInitialVariables.put(name, normalized); - if (JRT.isJrtManagedSpecialVariable(name)) { + if (isManagedSpecialVariable(name)) { baseSpecialVariables.put(name, normalized); } return; @@ -3374,7 +3485,8 @@ public final void assignVariable(String name, Object obj) { // FS=: between input files) must reach the JRT, not a global slot. // ARGC is excluded: JRT.setARGC delegates back to this method, and its // authoritative storage is the compiled slot below. - if (!"ARGC".equals(name) && jrt.applySpecialVariable(name, normalized)) { + if (!"ARGC".equals(name) && isManagedSpecialVariable(name)) { + jrt.applySpecialVariable(name, normalized); updateSymtabEntry(name, normalized); return; } @@ -3395,16 +3507,93 @@ public final void assignVariable(String name, Object obj) { updateSymtabEntry(name, normalized); } - private void applyInputSourceFilelistAssignmentsIfNeeded() { - if (inputSourceFilelistAssignmentsApplied || resolvedInputSource instanceof StreamInputSource) { - return; + /** + * Executes the {@code nextfile} statement: abandons the current input file + * and resumes the per-file input loop at the appropriate point. The + * runtime and operand stacks are cleared, so {@code nextfile} unwinds + * user-defined function calls, mirroring {@code exit}. + *

+ * A {@code nextfile} written directly inside a BEGIN, END, or ENDFILE + * rule is already rejected at compile time by the parser. The checks + * below cover the uses reached through user-defined functions, where the + * calling rule cannot be known statically (the same function may be + * called from both an ordinary rule and an END rule); gawk performs the + * same checks at runtime. + *

+ * + * @param position the tuple position tracker to redirect + */ + private void executeNextfile(PositionTracker position) { + if (nextFileAddress == null || !inputFileLoopStarted) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from a BEGIN rule"); + } + if (withinEndBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from an END rule"); + } + if (withinEndFileBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "`nextfile' cannot be called from an ENDFILE rule"); + } + // nextfile can be invoked from user-defined functions: unwind them. + runtimeStack.popAllFrames(); + operandStack.clear(); + if (endFileAddress == null + || withinBeginFileBlocks && jrt.hasPendingInputFileError(resolvedInputSource)) { + // No ENDFILE rules to run, or the file could not be opened: skip + // the ENDFILE section (gawk BEGINFILE error handling) and go + // straight to the next file. + withinBeginFileBlocks = false; + position.jump(nextFileAddress); + } else { + withinBeginFileBlocks = false; + withinEndFileBlocks = true; + position.jump(endFileAddress); } - for (String argument : arguments) { - if (argument.indexOf('=') > 0) { - setFilelistVariable(argument); - } + } + + /** + * Returns whether a non-redirected {@code getline} must be confined to + * the current input file. While the per-file main input loop of a program + * with BEGINFILE/ENDFILE rules is running, only the loop itself may cross + * file boundaries, so that no file's hooks are ever skipped; a + * {@code getline} in an action therefore reports end-of-input at the end + * of the current file. In BEGIN and END rules — before the loop starts or + * after it ends — {@code getline} keeps streaming across the remaining + * input. + * + * @return {@code true} when getline must not advance to the next file + */ + private boolean isMainInputFileBounded() { + return endFileAddress != null && inputFileLoopStarted && !withinEndBlocks; + } + + /** + * Raises the gawk-compatible fatal error when a non-redirected + * {@code getline} executes inside a BEGINFILE or ENDFILE rule. + *

+ * A non-redirected {@code getline} written directly inside a + * BEGINFILE/ENDFILE rule is already rejected at compile time by the + * parser. This runtime check covers the uses reached through + * user-defined functions, where the calling rule cannot be known + * statically; it lives here rather than in {@link JRT} because the + * current-rule flags are interpreter execution state. + *

+ * + * @param position the tuple position tracker, for error reporting + */ + private void checkGetlineAllowed(PositionTracker position) { + if (withinBeginFileBlocks || withinEndFileBlocks) { + throw new AwkRuntimeException( + position.lineNumber(), + "non-redirected `getline' invalid inside `" + + (withinBeginFileBlocks ? "BEGINFILE" : "ENDFILE") + + "' rule"); } - inputSourceFilelistAssignmentsApplied = true; } /** {@inheritDoc} */ @@ -3456,10 +3645,7 @@ public void setFILENAME(String filename) { public Object getARGV() { if (argvOffset == NULL_OFFSET) { Map argv = newAwkArray(); - argv.put(0L, "jawk"); - for (int i = 0; i < arguments.size(); i++) { - argv.put(Long.valueOf(i + 1L), jrt.toInputScalar(arguments.get(i))); - } + forEachArgvEntry(argv::put); return argv; } return runtimeStack.getVariable(argvOffset, true); @@ -3575,16 +3761,6 @@ private Object normalizeExternalVariableValue(Object value) { private static final Set NON_PERSISTENT_GLOBALS = new HashSet<>( Arrays.asList("ARGV", "ARGC", "ENVIRON", "RSTART", "RLENGTH", "IGNORECASE")); - private static final class NameValueAssignment { - private final String name; - private final Object value; - - private NameValueAssignment(String name, Object value) { - this.name = name; - this.value = value; - } - } - private static final class SingleRecordInputSource implements InputSource { private final String record; diff --git a/src/main/java/io/jawk/frontend/AwkParser.java b/src/main/java/io/jawk/frontend/AwkParser.java index 0bad1b36..134085fd 100644 --- a/src/main/java/io/jawk/frontend/AwkParser.java +++ b/src/main/java/io/jawk/frontend/AwkParser.java @@ -40,6 +40,7 @@ import io.jawk.ext.ExtensionFunction; import io.jawk.intermediate.Address; import io.jawk.intermediate.AwkTuples; +import io.jawk.jrt.JRT; import io.jawk.util.ScriptSource; import io.jawk.frontend.ast.LexerException; import io.jawk.frontend.ast.ParserException; @@ -131,6 +132,8 @@ enum Token { KW_FUNCTION, KW_BEGIN, KW_END, + KW_BEGINFILE, + KW_ENDFILE, KW_IN, KW_IF, KW_ELSE, @@ -140,6 +143,7 @@ enum Token { KW_RETURN, KW_EXIT, KW_NEXT, + KW_NEXTFILE, KW_CONTINUE, KW_DELETE, KW_BREAK, @@ -169,6 +173,8 @@ enum Token { KEYWORDS.put("function", Token.KW_FUNCTION); KEYWORDS.put("BEGIN", Token.KW_BEGIN); KEYWORDS.put("END", Token.KW_END); + KEYWORDS.put("BEGINFILE", Token.KW_BEGINFILE); + KEYWORDS.put("ENDFILE", Token.KW_ENDFILE); KEYWORDS.put("in", Token.KW_IN); // statements @@ -180,6 +186,7 @@ enum Token { KEYWORDS.put("return", Token.KW_RETURN); KEYWORDS.put("exit", Token.KW_EXIT); KEYWORDS.put("next", Token.KW_NEXT); + KEYWORDS.put("nextfile", Token.KW_NEXTFILE); KEYWORDS.put("continue", Token.KW_CONTINUE); KEYWORDS.put("delete", Token.KW_DELETE); KEYWORDS.put("break", Token.KW_BREAK); @@ -283,6 +290,8 @@ private static BuiltinFunction of(String name) { SPECIAL_VAR_NAMES.put("ARGC", SP_IDX); SPECIAL_VAR_NAMES.put("ARGV", SP_IDX); SPECIAL_VAR_NAMES.put("IGNORECASE", SP_IDX); + SPECIAL_VAR_NAMES.put("ERRNO", SP_IDX); + SPECIAL_VAR_NAMES.put("ARGIND", SP_IDX); } /** @@ -310,6 +319,19 @@ public AwkParser(Map extensions, boolean posix) { this.posix = posix; } + /** + * Returns whether the keyword is disabled in the current compile-time + * mode. BEGINFILE and ENDFILE are gawk extensions: in POSIX mode they are + * not special and lex as plain identifiers, exactly like + * {@code gawk --posix}. + * + * @param keywordToken the keyword token to inspect + * @return {@code true} when the keyword must be treated as an identifier + */ + private boolean isDisabledKeyword(Token keywordToken) { + return posix && (keywordToken == Token.KW_BEGINFILE || keywordToken == Token.KW_ENDFILE); + } + private List scriptSources; private int scriptSourcesCurrentIndex; private LineNumberReader reader; @@ -515,18 +537,64 @@ private void readString() throws IOException { /** * Reads the regular expression (between slashes '/') and handle '\/'. + * A slash within a bracket expression (e.g. {@code /[/]/}) does not + * terminate the regular expression, per POSIX ERE bracket semantics. * * @throws IOException */ private void readRegexp() throws IOException { regexp.setLength(0); - while (token != Token.EOF && c > 0 && c != '/' && c != '\n') { + boolean inBracket = false; + while (token != Token.EOF && c > 0 && (c != '/' || inBracket) && c != '\n') { if (c == '\\') { read(); if (c != '/') { regexp.append('\\'); } + regexp.append((char) c); + read(); + continue; + } + if (!inBracket && c == '[') { + inBracket = true; + regexp.append((char) c); + read(); + // a ']' right after '[' (or after '[^') is a literal ']' + if (c == '^') { + regexp.append((char) c); + read(); + } + if (c == ']') { + regexp.append((char) c); + read(); + } + continue; + } + if (inBracket && c == '[') { + regexp.append((char) c); + read(); + // POSIX character class, collating element, or equivalence + // class ([:alpha:], [.x.], [=e=]): its closing ']' does not + // end the outer bracket expression. + if (c == ':' || c == '.' || c == '=') { + int delimiter = c; + boolean closed = false; + while (token != Token.EOF && c > 0 && c != '\n' && !closed) { + int previous = c; + regexp.append((char) c); + read(); + if (previous == delimiter && c == ']') { + regexp.append((char) c); + read(); + closed = true; + } + } + } + continue; + } + if (inBracket && c == ']') { + inBracket = false; } regexp.append((char) c); read(); @@ -839,7 +907,7 @@ private Token lexer() throws IOException { return token; } Token kwToken = KEYWORDS.get(text.toString()); - if (kwToken != null) { + if (kwToken != null && !isDisabledKeyword(kwToken)) { token = kwToken; return token; } @@ -1043,6 +1111,12 @@ AST RULE() throws IOException { } else if (token == Token.KW_END) { lexer(); optExpr = symbolTable.addEND(); + } else if (token == Token.KW_BEGINFILE) { + lexer(); + optExpr = symbolTable.addBEGINFILE(); + } else if (token == Token.KW_ENDFILE) { + lexer(); + optExpr = symbolTable.addENDFILE(); } else if (token != Token.OPEN_BRACE && token != Token.SEMICOLON && token != Token.NEWLINE && token != Token.EOF) { // true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions optExpr = ASSIGNMENT_EXPRESSION(null, true, true, false); @@ -1676,6 +1750,8 @@ AST STATEMENT() throws IOException { stmt = PRINTF_STATEMENT(); } else if (token == Token.KW_NEXT) { stmt = NEXT_STATEMENT(); + } else if (token == Token.KW_NEXTFILE) { + stmt = NEXTFILE_STATEMENT(); } else if (token == Token.KW_CONTINUE) { stmt = CONTINUE_STATEMENT(); } else if (token == Token.KW_BREAK) { @@ -2096,6 +2172,12 @@ AST NEXT_STATEMENT() throws IOException { return new NextStatementAst(); } + AST NEXTFILE_STATEMENT() throws IOException { + expectKeyword("nextfile"); + nextfileEncountered = true; + return new NextfileStatementAst(); + } + AST CONTINUE_STATEMENT() throws IOException { expectKeyword("continue"); return new ContinueStatementAst(); @@ -2614,6 +2696,52 @@ private boolean isEnd() { return result; } + private boolean isBeginFile = isBeginFile(); + + protected final void setBeginFileFlag(boolean flag) { + isBeginFile = flag; + } + + private boolean isBeginFile() { + boolean result = isBeginFile; + if (!result && ast1 != null) { + result = ast1.isBeginFile(); + } + if (!result && ast2 != null) { + result = ast2.isBeginFile(); + } + if (!result && ast3 != null) { + result = ast3.isBeginFile(); + } + if (!result && ast4 != null) { + result = ast4.isBeginFile(); + } + return result; + } + + private boolean isEndFile = isEndFile(); + + protected final void setEndFileFlag(boolean flag) { + isEndFile = flag; + } + + private boolean isEndFile() { + boolean result = isEndFile; + if (!result && ast1 != null) { + result = ast1.isEndFile(); + } + if (!result && ast2 != null) { + result = ast2.isEndFile(); + } + if (!result && ast3 != null) { + result = ast3.isEndFile(); + } + if (!result && ast4 != null) { + result = ast4.isEndFile(); + } + return result; + } + private boolean isFunction = isFunction(); @SuppressWarnings("unused") @@ -2719,7 +2847,12 @@ public boolean isScalar() { } private static boolean isRule(AST ast) { - return ast != null && !ast.isBegin() && !ast.isEnd() && !ast.isFunction(); + return ast != null + && !ast.isBegin() + && !ast.isEnd() + && !ast.isBeginFile() + && !ast.isEndFile() + && !ast.isFunction(); } /** @@ -2775,6 +2908,14 @@ private static boolean containsASTType(AST ast, Class[] clsArray) { private Address nextAddress; + /** + * Whether the program contains at least one {@code nextfile} statement, + * anywhere (rules or user-defined functions). When set, the main input + * loop is compiled with per-file stepping so the runtime can jump to the + * ENDFILE section and advance to the next input file. + */ + private boolean nextfileEncountered; + private final class RuleListAst extends AST { private RuleListAst(AST rule, AST rest) { @@ -2857,20 +2998,26 @@ public int populateTuples(AwkTuples tuples) { Address exitAddr = tuples.createAddress("end blocks start address"); tuples.setExitAddress(exitAddr); - // grab all BEGINs + // Does the program use BEGINFILE/ENDFILE rules or nextfile? If so, + // the main input loop must step through the input one file at a + // time (per-file scaffolding) instead of streaming across files. + boolean hasBeginFileRules = false; + boolean hasEndFileRules = false; ptr = this; - // ptr.getAst1() == blank rule condition (i.e.: { print }) while (ptr != null) { - if (ptr.getAst1() != null && ptr.getAst1().isBegin()) { - ptr.getAst1().populateTuples(tuples); + if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) { + hasBeginFileRules = true; + } + if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) { + hasEndFileRules = true; } - ptr = ptr.getAst2(); } // Do we have rules? (apart from BEGIN) - // If we have rules or END, we need to parse the input - boolean reqInput = false; + // If we have rules, END, BEGINFILE, or ENDFILE, we need to parse + // the input + boolean reqInput = hasBeginFileRules || hasEndFileRules; // Check for "normal" rules ptr = this; @@ -2890,19 +3037,70 @@ public int populateTuples(AwkTuples tuples) { ptr = ptr.getAst2(); } - if (reqInput) { - Address inputLoopAddress = null; - Address noMoreInput = null; + boolean perFileScaffolding = reqInput + && (hasBeginFileRules || hasEndFileRules || nextfileEncountered); + + // The per-file addresses are carried as properties of the tuple + // stream, so a nextfile executed from a user-defined function can + // resolve them at runtime. The begin_file address labels the + // NEXT_FILE tuple that opens each input file. + Address beginFileAddress = null; + Address endFileAddress = null; + if (perFileScaffolding) { + beginFileAddress = tuples.createAddress("begin_file"); + endFileAddress = tuples.createAddress("end_file"); + // The ENDFILE address is registered only when BEGINFILE or + // ENDFILE rules exist: its presence also confines a + // non-redirected getline to the current input file, which + // only matters when there are per-file hooks to protect. + if (hasBeginFileRules || hasEndFileRules) { + tuples.setEndFileAddress(endFileAddress); + } + tuples.setNextFileAddress(beginFileAddress); + } - inputLoopAddress = tuples.createAddress("input_loop_address"); - tuples.address(inputLoopAddress); + // grab all BEGINs + ptr = this; + // ptr.getAst1() == blank rule condition (i.e.: { print }) + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isBegin()) { + ptr.getAst1().populateTuples(tuples); + } - ptr = this; + ptr = ptr.getAst2(); + } - noMoreInput = tuples.createAddress("no_more_input"); - tuples.consumeInput(noMoreInput); + if (reqInput) { + Address inputLoopAddress = tuples.createAddress("input_loop_address"); + Address noMoreInput = tuples.createAddress("no_more_input"); + + if (perFileScaffolding) { + // Advance to the next input file so the BEGINFILE rules + // observe its FILENAME, FNR, ARGIND, and ERRNO. The + // ENDFILE section loops back here for the following files. + tuples.address(beginFileAddress); + tuples.nextFile(noMoreInput); + + // BEGINFILE rules, in the order they were read + ptr = this; + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) { + ptr.getAst1().populateTuples(tuples); + } + ptr = ptr.getAst2(); + } + + // per-file input loop: at end of the current file, run the + // ENDFILE rules instead of silently opening the next file + tuples.address(inputLoopAddress); + tuples.consumeFileInput(endFileAddress); + } else { + tuples.address(inputLoopAddress); + tuples.consumeInput(noMoreInput); + } // grab all INPUT RULES + ptr = this; while (ptr != null) { // the first one of these is an input rule if (isRule(ptr.getAst1())) { @@ -2914,11 +3112,26 @@ public int populateTuples(AwkTuples tuples) { tuples.gotoAddress(inputLoopAddress); - if (reqInput) { - tuples.address(noMoreInput); - // compiler has issue with missing nop here - tuples.nop(); + if (perFileScaffolding) { + // ENDFILE rules, in the order they were read + tuples.address(endFileAddress); + ptr = this; + while (ptr != null) { + if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) { + ptr.getAst1().populateTuples(tuples); + } + ptr = ptr.getAst2(); + } + + // then loop back to the NEXT_FILE tuple at begin_file, + // which falls through to the END rules once the input is + // exhausted + tuples.gotoAddress(beginFileAddress); } + + tuples.address(noMoreInput); + // compiler has issue with missing nop here + tuples.nop(); } // indicate where the first end block resides @@ -3008,7 +3221,9 @@ public int populateTuples(AwkTuples tuples) { pushSourceLineNumber(tuples); boolean unconditionalRule = getAst1() == null || getAst1().isBegin() - || getAst1().isEnd(); + || getAst1().isEnd() + || getAst1().isBeginFile() + || getAst1().isEndFile(); if (!unconditionalRule) { getAst1().populateTuples(tuples); // result of whether to execute or not is on the stack @@ -3026,12 +3241,12 @@ public int populateTuples(AwkTuples tuples) { private void populateRuleBody(AwkTuples tuples) { // execute the optRule here! if (getAst2() == null) { - if (getAst1() == null || (!getAst1().isBegin() && !getAst1().isEnd())) { + if (isRule(this)) { // display $0 tuples.print(0); } // else, don't populate it with anything - // (i.e., blank BEGIN/END rule) + // (i.e., blank BEGIN/END/BEGINFILE/ENDFILE rule) } else { // execute it, and leave nothing on the stack getAst2().populateTuples(tuples); @@ -3041,13 +3256,35 @@ private void populateRuleBody(AwkTuples tuples) { @Override public Address nextAddress() { if (!isRule(this)) { - throw new SemanticException("Must call next within an input rule."); + throw new SemanticException( + "`next' cannot be called from a `" + specialRuleName() + "' rule."); } if (nextAddress == null) { throw new SemanticException("Cannot call next here."); } return nextAddress; } + + /** + * Names the special (non-input) rule this AST represents, for + * gawk-compatible diagnostics. + */ + private String specialRuleName() { + AST pattern = getAst1(); + if (pattern != null && pattern.isBegin()) { + return "BEGIN"; + } + if (pattern != null && pattern.isEnd()) { + return "END"; + } + if (pattern != null && pattern.isBeginFile()) { + return "BEGINFILE"; + } + if (pattern != null && pattern.isEndFile()) { + return "ENDFILE"; + } + return "special"; + } } private final class IfStatementAst extends AST { @@ -4461,7 +4698,7 @@ public void semanticAnalysis() throws SemanticException { FunctionDefParamListAst ptr = this; while (ptr != null) { - if (SPECIAL_VAR_NAMES.get(ptr.id) != null) { + if (isSpecialVariableName(ptr.id)) { throw new SemanticException("Special variable " + ptr.id + " cannot be used as a formal parameter"); } ptr = (FunctionDefParamListAst) ptr.getAst1(); @@ -4770,6 +5007,38 @@ public int populateTuples(AwkTuples tuples) { } } + private final class BeginFileAst extends AST { + + private BeginFileAst() { + super(); + setBeginFileFlag(true); + } + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + tuples.push(1); + popSourceLineNumber(tuples); + return 1; + } + } + + private final class EndFileAst extends AST { + + private EndFileAst() { + super(); + setEndFileFlag(true); + } + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + tuples.push(1); + popSourceLineNumber(tuples); + return 1; + } + } + private final class PreIncAst extends ScalarExpressionAst { private PreIncAst(AST symbolAst) { @@ -5023,7 +5292,28 @@ public int populateTuples(AwkTuples tuples) { * them. */ private boolean isJrtManagedSpecialName(String id) { - return SPECIAL_VAR_NAMES.containsKey(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id); + return isSpecialVariableName(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id); + } + + /** + * Returns whether the identifier names a special variable in the current + * compile-time mode. Most special names (NR, FS, FILENAME, ...) always + * are. The gawk-only ERRNO and ARGIND are special outside POSIX mode + * only: with {@code --posix} they are plain identifiers — usable as + * function parameters and compiled as ordinary globals — exactly like + * {@code gawk --posix} treats them. + * + * @param id the identifier to inspect + * @return {@code true} when the name is special in the current mode + */ + private boolean isSpecialVariableName(String id) { + if (!SPECIAL_VAR_NAMES.containsKey(id)) { + return false; + } + if (JRT.isGawkOnlySpecialVariable(id)) { + return !posix; + } + return true; } /** Emits the tuple pushing the value of a JRT-managed special variable. */ @@ -5074,6 +5364,12 @@ private void pushSpecialVariable(AwkTuples tuples, String id) { case "ARGC": tuples.pushARGC(); break; + case "ERRNO": + tuples.pushERRNO(); + break; + case "ARGIND": + tuples.pushARGIND(); + break; default: throw new Error("Unhandled special var: " + id); } @@ -5127,6 +5423,12 @@ private void assignSpecialVariable(AwkTuples tuples, String id) { case "ARGC": tuples.assignARGC(); break; + case "ERRNO": + tuples.assignERRNO(); + break; + case "ARGIND": + tuples.assignARGIND(); + break; default: throw new Error("Unhandled special var: " + id); } @@ -5327,6 +5629,21 @@ private GetlineAst(AST pipeExpr, AST lvalueAst, AST inRedirect) { @Override public int populateTuples(AwkTuples tuples) { pushSourceLineNumber(tuples); + // gawk restriction: only redirected forms of getline may be used + // inside BEGINFILE/ENDFILE rules. Direct uses are rejected here at + // compile time; uses reached through user-defined functions are + // caught at runtime by the interpreter, which knows the current + // rule. + if (getAst1() == null && getAst3() == null) { + AST enclosingRule = searchFor(AstFlag.NEXTABLE); + AST pattern = enclosingRule == null ? null : enclosingRule.getAst1(); + if (pattern != null && (pattern.isBeginFile() || pattern.isEndFile())) { + throw new SemanticException( + "non-redirected `getline' invalid inside `" + + (pattern.isBeginFile() ? "BEGINFILE" : "ENDFILE") + + "' rule"); + } + } if (getAst1() == null && getAst3() == null && getAst2() == null) { tuples.getlineInput(); popSourceLineNumber(tuples); @@ -5494,6 +5811,29 @@ public int populateTuples(AwkTuples tuples) { } } + private class NextfileStatementAst extends AST { + + @Override + public int populateTuples(AwkTuples tuples) { + pushSourceLineNumber(tuples); + AST nextable = searchFor(AstFlag.NEXTABLE); + if (nextable != null) { + // Direct use inside a rule: BEGIN, END, and ENDFILE reject + // nextfile at compile time, mirroring gawk's fatal errors. + // (Uses within user-defined functions are checked at runtime.) + AST pattern = nextable.getAst1(); + if (pattern != null && (pattern.isBegin() || pattern.isEnd() || pattern.isEndFile())) { + String ruleName = pattern.isBegin() ? "BEGIN" : pattern.isEnd() ? "END" : "ENDFILE"; + throw new SemanticException( + "`nextfile' cannot be called from a `" + ruleName + "' rule."); + } + } + tuples.execNextfile(); + popSourceLineNumber(tuples); + return 0; + } + } + private final class ContinueStatementAst extends AST { private ContinueStatementAst() { @@ -5589,6 +5929,8 @@ int numGlobals() { // "constants" private BeginAst beginAst = null; private EndAst endAst = null; + private BeginFileAst beginFileAst = null; + private EndFileAst endFileAst = null; // functions (proxies) private Map functionProxies = new HashMap(); @@ -5625,6 +5967,20 @@ AST addEND() { return endAst; } + AST addBEGINFILE() { + if (beginFileAst == null) { + beginFileAst = new BeginFileAst(); + } + return beginFileAst; + } + + AST addENDFILE() { + if (endFileAst == null) { + endFileAst = new EndFileAst(); + } + return endFileAst; + } + /** * Returns whether the script references the named global variable, * without creating a symbol for it. diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 396c2891..f8b536c7 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -50,7 +50,7 @@ */ public class AwkTuples implements Serializable { - private static final long serialVersionUID = 2L; + private static final long serialVersionUID = 3L; /** Address manager */ private final AddressManager addressManager = new AddressManager(); @@ -111,6 +111,26 @@ public boolean add(Tuple t) { /** Whether this tuple stream was produced by {@code compileExpression()}. */ private boolean evalTupleStream; + /** + * Address of the END blocks section, where a runtime {@code exit} jumps; + * {@code null} for expression streams. Property addresses are remapped + * explicitly by the optimizer and seeded as reachability roots, so they + * stay valid even when no tuple references them. + */ + private Address exitAddress; + + /** + * Address of the ENDFILE section, or {@code null} when the program has + * no BEGINFILE/ENDFILE rules. + */ + private Address endFileAddress; + + /** + * Address of the {@code NEXT_FILE} tuple that opens each input file, or + * {@code null} when the program does not use per-file input stepping. + */ + private Address nextFileAddress; + /** *

* toOpcodeString. @@ -1492,6 +1512,36 @@ public void assignIGNORECASE() { queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_IGNORECASE)); } + /** + * Emits the tuple pushing the value of ERRNO, managed by the JRT. + */ + public void pushERRNO() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_ERRNO)); + } + + /** + * Emits the tuple assigning the top of the stack to ERRNO, managed by the + * JRT. + */ + public void assignERRNO() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_ERRNO)); + } + + /** + * Emits the tuple pushing the value of ARGIND, managed by the JRT. + */ + public void pushARGIND() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_ARGIND)); + } + + /** + * Emits the tuple assigning the top of the stack to ARGIND, managed by + * the JRT. + */ + public void assignARGIND() { + queue.add(new Tuple.BuiltinVarTuple(Opcode.ASSIGN_ARGIND)); + } + /** Pushes the current value of {@code RS} onto the operand stack. */ public void pushRS() { queue.add(new Tuple.BuiltinVarTuple(Opcode.PUSH_RS)); @@ -1723,14 +1773,25 @@ public void deleteArray(int offset, boolean isGlobal) { } /** - *

- * setExitAddress. - *

+ * Registers the address of the END blocks section, so that a runtime + * {@code exit} statement can jump to it. A property of the tuple stream + * rather than a tuple: the interpreter reads it once when it installs + * the program. * - * @param addr a {@link io.jawk.intermediate.Address} object + * @param addr address of the END blocks section */ public void setExitAddress(Address addr) { - queue.add(new Tuple.AddressTuple(Opcode.SET_EXIT_ADDRESS, addr)); + exitAddress = addr; + } + + /** + * Returns the address of the END blocks section, or {@code null} when + * the tuple stream is an expression stream with no END blocks. + * + * @return address of the END blocks section, or {@code null} + */ + public Address getExitAddress() { + return exitAddress; } /** @@ -1744,6 +1805,79 @@ public void setWithinEndBlocks(boolean b) { queue.add(new Tuple.BooleanTuple(Opcode.SET_WITHIN_END_BLOCKS, b)); } + /** + * Registers the address of the ENDFILE section, so that a runtime + * {@code nextfile} statement can jump to it. A property of the tuple + * stream rather than a tuple: the interpreter reads it once when it + * installs the program. + * + * @param addr address of the ENDFILE section + */ + public void setEndFileAddress(Address addr) { + endFileAddress = addr; + } + + /** + * Returns the address of the ENDFILE section, or {@code null} when the + * program has no BEGINFILE/ENDFILE rules. + * + * @return address of the ENDFILE section, or {@code null} + */ + public Address getEndFileAddress() { + return endFileAddress; + } + + /** + * Registers the address of the {@code NEXT_FILE} tuple that opens each + * input file, so that a runtime {@code nextfile} statement can bypass + * the ENDFILE rules for input files that could not be opened. A property + * of the tuple stream rather than a tuple: the interpreter reads it once + * when it installs the program. + * + * @param addr address of the NEXT_FILE tuple + */ + public void setNextFileAddress(Address addr) { + nextFileAddress = addr; + } + + /** + * Returns the address of the {@code NEXT_FILE} tuple that opens each + * input file, or {@code null} when the program does not use per-file + * input stepping. + * + * @return address of the NEXT_FILE tuple, or {@code null} + */ + public Address getNextFileAddress() { + return nextFileAddress; + } + + /** + * Emits the tuple advancing the main input to the next input file, or + * jumping to the given address when no input file remains. + * + * @param address address to jump to when no more input files remain + */ + public void nextFile(Address address) { + queue.add(new Tuple.AddressTuple(Opcode.NEXT_FILE, address)); + } + + /** + * Emits the tuple consuming one record of the current input file only, + * jumping to the given address at end of the current file. + * + * @param address address to jump to at end of the current input file + */ + public void consumeFileInput(Address address) { + queue.add(new Tuple.AddressTuple(Opcode.CONSUME_FILE_INPUT, address)); + } + + /** + * Emits the tuple executing the {@code nextfile} statement at runtime. + */ + public void execNextfile() { + queue.add(new Tuple.NoOperandTuple(Opcode.EXEC_NEXTFILE)); + } + /** *

* exitWithCode. @@ -2320,19 +2454,44 @@ private void remapAddresses(int[] indexMapping) { } Set

processedAddresses = Collections.newSetFromMap(new IdentityHashMap()); for (Tuple tuple : queue) { - Address address = tuple.getAddress(); - if (address != null && processedAddresses.add(address)) { - int oldIndex = address.index(); - if (oldIndex >= 0 && oldIndex < indexMapping.length) { - int mappedIndex = indexMapping[oldIndex]; - if (mappedIndex < 0) { - throw new Error("Address " + address + " references removed tuple " + oldIndex); - } - address.assignIndex(mappedIndex); - } + remapAddress(tuple.getAddress(), indexMapping, processedAddresses); + } + // Property addresses may not be referenced by any tuple (e.g. after + // jump threading rewired the loop-back GOTO), so they must be + // remapped explicitly to stay valid. + remapAddress(exitAddress, indexMapping, processedAddresses); + remapAddress(endFileAddress, indexMapping, processedAddresses); + remapAddress(nextFileAddress, indexMapping, processedAddresses); + addressManager.remapIndexes(indexMapping); + } + + private static void seedPropertyAddress( + Address address, + int size, + boolean[] reachable, + Deque worklist) { + if (address == null) { + return; + } + int targetIndex = address.index(); + if (targetIndex >= 0 && targetIndex < size && !reachable[targetIndex]) { + reachable[targetIndex] = true; + worklist.addLast(targetIndex); + } + } + + private static void remapAddress(Address address, int[] indexMapping, Set
processedAddresses) { + if (address == null || !processedAddresses.add(address)) { + return; + } + int oldIndex = address.index(); + if (oldIndex >= 0 && oldIndex < indexMapping.length) { + int mappedIndex = indexMapping[oldIndex]; + if (mappedIndex < 0) { + throw new Error("Address " + address + " references removed tuple " + oldIndex); } + address.assignIndex(mappedIndex); } - addressManager.remapIndexes(indexMapping); } private void reprocessQueue() { @@ -2514,6 +2673,13 @@ private void optimizeQueue() { reachable[0] = true; worklist.add(0); } + // The property addresses are runtime jump targets (exit, nextfile, + // and the ENDFILE section it resumes at) that no tuple may reference: + // treat them as reachability roots so their sections are never + // eliminated as dead code. + seedPropertyAddress(exitAddress, size, reachable, worklist); + seedPropertyAddress(endFileAddress, size, reachable, worklist); + seedPropertyAddress(nextFileAddress, size, reachable, worklist); while (!worklist.isEmpty()) { int index = worklist.removeFirst(); diff --git a/src/main/java/io/jawk/intermediate/Opcode.java b/src/main/java/io/jawk/intermediate/Opcode.java index 8057f103..80fadff2 100644 --- a/src/main/java/io/jawk/intermediate/Opcode.java +++ b/src/main/java/io/jawk/intermediate/Opcode.java @@ -1279,13 +1279,6 @@ public enum Opcode { */ DELETE_MAP_ELEMENT, - /** - * Internal. - *

- * Stack remains unchanged. - */ - SET_EXIT_ADDRESS, - /** * Internal. *

@@ -1540,7 +1533,79 @@ public enum Opcode { * Argument: offset of the FUNCTAB global
* Stack unchanged. */ - UPDATE_FUNCTAB; + UPDATE_FUNCTAB, + + /** + * Advances the main input to the next input file, applying pending + * {@code name=value} command-line assignments along the way. On success, + * FILENAME, FNR, ARGIND, and ERRNO are updated and execution falls through + * to the BEGINFILE rules. When no input file remains, it jumps to the + * specified address. Emitted only when BEGINFILE/ENDFILE rules or a + * {@code nextfile} statement require per-file input stepping. + *

+ * Argument: address to jump to when no more input files remain + *

+ * Stack unchanged. + */ + NEXT_FILE, + + /** + * Consume the next record of the current input file only; assigning $0 and + * recalculating $1, $2, etc. Unlike {@link #CONSUME_INPUT}, it never + * advances to the next input file: at end of the current file it jumps to + * the specified address so the ENDFILE rules can run. + *

+ * Argument: address to jump to at end of the current input file + *

+ * Stack unchanged. + */ + CONSUME_FILE_INPUT, + + /** + * Executes the {@code nextfile} statement: abandons the current input + * file and resumes the per-file input loop. The runtime jumps to the + * ENDFILE rules when the current file was opened successfully, or + * directly past them when the file could not be opened (BEGINFILE error + * handling). The jump targets are carried as properties of the tuple + * stream, not as tuples. The runtime stack and operand stack are + * cleared, allowing {@code nextfile} to be invoked from user-defined + * functions. + *

+ * Stack after: (empty) + */ + EXEC_NEXTFILE, + + /** + * Assigns the top of the stack to ERRNO, managed by the JRT. + *

+ * Stack before: value ...
+ * Stack after: value ... + */ + ASSIGN_ERRNO, + + /** + * Pushes the value of ERRNO, managed by the JRT. + *

+ * Stack before: ...
+ * Stack after: errno-value ... + */ + PUSH_ERRNO, + + /** + * Assigns the top of the stack to ARGIND, managed by the JRT. + *

+ * Stack before: value ...
+ * Stack after: value ... + */ + ASSIGN_ARGIND, + + /** + * Pushes the value of ARGIND, managed by the JRT. + *

+ * Stack before: ...
+ * Stack after: argind-value ... + */ + PUSH_ARGIND; private static final Opcode[] VALUES = values(); diff --git a/src/main/java/io/jawk/jrt/InputSource.java b/src/main/java/io/jawk/jrt/InputSource.java index a9932cc1..d850473a 100644 --- a/src/main/java/io/jawk/jrt/InputSource.java +++ b/src/main/java/io/jawk/jrt/InputSource.java @@ -98,6 +98,12 @@ default String getRecord() { /** * Indicates whether the current record originates from a named file in the * argument list. + *

+ * For custom implementations, this flag also controls whether consuming a + * record advances the per-file record counter {@code FNR}. Records read by + * the built-in stream input always advance {@code FNR} — including + * standard input — per POSIX. + *

* * @return {@code true} when sourced from a filename argument */ diff --git a/src/main/java/io/jawk/jrt/JRT.java b/src/main/java/io/jawk/jrt/JRT.java index 92ad9962..11ddfc08 100644 --- a/src/main/java/io/jawk/jrt/JRT.java +++ b/src/main/java/io/jawk/jrt/JRT.java @@ -136,6 +136,9 @@ public class JRT { private int rstart; // last match start (1-based) private int rlength; // last match length private Object filename; // current input filename scalar (or empty for stdin/pipe) + private Object errno; // last input I/O error description (gawk ERRNO) + private Object argind; // ARGV index of the current input file (gawk ARGIND) + private boolean syntheticFilePresented; // custom InputSource already presented as a single "file" private String fs; // field separator private String rs; // record separator (regexp) private String ofs; // output field separator @@ -320,12 +323,26 @@ public static boolean isJrtManagedSpecialVariable(String name) { case "FNR": case "ARGC": case "IGNORECASE": + case "ERRNO": + case "ARGIND": return true; default: return false; } } + /** + * Returns whether the name is a gawk-only special variable that POSIX + * mode treats as an ordinary identifier, like {@code gawk --posix} does. + * Shared by the parser and the interpreter so both stay in sync. + * + * @param name variable name to inspect + * @return {@code true} when POSIX mode must treat the name as ordinary + */ + public static boolean isGawkOnlySpecialVariable(String name) { + return "ERRNO".equals(name) || "ARGIND".equals(name); + } + /** * Copies only the JRT-managed special variables from the supplied map. * @@ -375,6 +392,9 @@ public void prepareForExecution(String defaultFs, String defaultRs) { rstart = 0; rlength = 0; filename = ""; + errno = ""; + argind = ZERO; + syntheticFilePresented = false; // Apply default runtime special variables. setFS(defaultFs == null ? Awk.DEFAULT_FS : defaultFs); @@ -457,6 +477,12 @@ public boolean applySpecialVariable(String name, Object value) { case "IGNORECASE": setIGNORECASE(value); return true; + case "ERRNO": + setERRNO(value); + return true; + case "ARGIND": + setARGIND(value); + return true; default: return false; } @@ -1585,6 +1611,42 @@ public void setFILENAMEViaJrt(Object name) { this.filename = normalizeRecordValue(name); } + /** + * Get ERRNO as tracked by JRT. + * + * @return current ERRNO (empty string when no input error is pending) + */ + public Object getERRNO() { + return errno == null ? "" : errno; + } + + /** + * Set ERRNO tracked by JRT. + * + * @param value new ERRNO value + */ + public void setERRNO(Object value) { + this.errno = normalizeRecordValue(value); + } + + /** + * Get ARGIND as tracked by JRT. + * + * @return ARGV index of the current input file (0 before any file is open) + */ + public Object getARGIND() { + return argind == null ? ZERO : argind; + } + + /** + * Set ARGIND tracked by JRT. + * + * @param value new ARGIND value + */ + public void setARGIND(Object value) { + this.argind = normalizeRecordValue(value); + } + /** * Get SUBSEP from the VariableManager. * @@ -1731,14 +1793,163 @@ public boolean consumeInput(final InputSource source) throws IOException { return false; } + bindConsumedRecord(source); + return true; + } + + /** + * Attempt to consume one record from the current input file only, without + * ever advancing to the next input file. Used by the per-file main input + * loop when BEGINFILE/ENDFILE rules or {@code nextfile} are present, so + * that the ENDFILE rules can run at each file boundary. + *

+ * When the current input file could not be opened (a pending ERRNO set by + * {@link #advanceToNextFile(InputSource)} that no {@code nextfile} + * consumed), the usual fatal error is raised, mirroring gawk. + *

+ * + * @param source source strategy that provides records and optional + * pre-split fields + * @return {@code true} if a record was consumed; {@code false} at the end + * of the current input file + * @throws IOException if the source raises an I/O error + */ + public boolean consumeCurrentFileInput(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (!(source instanceof StreamInputSource)) { + // Custom input sources behave as a single unnamed input file. + return consumeInput(source); + } + StreamInputSource streamSource = (StreamInputSource) source; + throwIfCurrentFileUnopened(streamSource); + activeSource = source; + if (!streamSource.nextRecordInCurrentFile()) { + return false; + } + bindConsumedRecord(source); + return true; + } + + /** + * Attempt to consume one record of the current input file only for + * {@code getline target}, returning the input value and leaving the + * current input record state untouched. Used instead of + * {@link #consumeInputToTarget(InputSource)} while the per-file main + * input loop is active, so a {@code getline} in an action never crosses a + * file boundary behind the BEGINFILE/ENDFILE rules' back. + * + * @param source source strategy that provides records and optional + * pre-split fields + * @return the consumed input value, or {@code null} at the end of the + * current input file + * @throws IOException if the source raises an I/O error + */ + public Object consumeCurrentFileInputToTarget(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (!(source instanceof StreamInputSource)) { + // Custom input sources behave as a single unnamed input file. + return consumeInputToTarget(source); + } + StreamInputSource streamSource = (StreamInputSource) source; + throwIfCurrentFileUnopened(streamSource); + activeSource = source; + materializeCurrentRecord(); + if (!streamSource.nextRecordInCurrentFile()) { + return null; + } + + RecordState inputState = new RecordState(source); + this.nr++; + if (countsTowardFNR(source)) { + this.fnr++; + } + return new StrNum(inputState.getRecordText(), decimalSeparator); + } + + /** + * Raises the gawk-compatible fatal error when the current input file + * could not be opened and no BEGINFILE rule bypassed it with + * {@code nextfile}. + * + * @param streamSource the main input source to check + */ + private void throwIfCurrentFileUnopened(StreamInputSource streamSource) { + String openError = streamSource.getCurrentFileOpenError(); + if (openError != null) { + throw new AwkRuntimeException( + "cannot open file `" + toAwkString(getFILENAME()) + "' for reading: " + openError); + } + } + + /** + * Advance the main input to the next input file, applying pending + * {@code name=value} command-line assignments along the way. On success, + * FILENAME, FNR, ARGIND, and ERRNO are updated and {@code $0} is cleared, + * so the BEGINFILE rules observe the new file. A file that cannot be + * opened is still reported as available, with ERRNO carrying the error + * description (gawk BEGINFILE error handling). + * + * @param source source strategy that provides records and optional + * pre-split fields + * @return {@code true} when a new input file (or the initial stdin + * stream) is available; {@code false} when input is exhausted + * @throws IOException if an I/O error occurs while traversing ARGV + */ + public boolean advanceToNextFile(final InputSource source) throws IOException { + Objects.requireNonNull(source, "source"); + if (source instanceof StreamInputSource) { + return ((StreamInputSource) source).advanceToNextFile(); + } + // Custom input sources behave as a single unnamed input file. + if (syntheticFilePresented) { + return false; + } + syntheticFilePresented = true; + return true; + } + + /** + * Returns whether the current input file of the given source failed to + * open, leaving a pending error that only a {@code nextfile} statement in + * a BEGINFILE rule may bypass. + * + * @param source source strategy that provides records + * @return {@code true} when the current input file could not be opened + */ + public boolean hasPendingInputFileError(InputSource source) { + return source instanceof StreamInputSource + && ((StreamInputSource) source).getCurrentFileOpenError() != null; + } + + /** + * Binds the record just consumed from the given source as the current + * input record and updates the NR/FNR counters. + * + * @param source the source a record was just consumed from + */ + private void bindConsumedRecord(InputSource source) { inputLine = null; recordState = new RecordState(source); this.nr++; - if (source.isFromFilenameList()) { + if (countsTowardFNR(source)) { this.fnr++; } - return true; + } + + /** + * Returns whether consuming a record from the given source advances FNR, + * the per-file record counter. All records of the main command-line input + * flow count, including standard input (POSIX defines FNR as the record + * number in the current input file, which stdin is). For custom + * {@link InputSource} implementations, {@link InputSource#isFromFilenameList()} + * keeps controlling FNR, as documented. + * + * @param source the source a record was just consumed from + * @return {@code true} when the record advances FNR + */ + private static boolean countsTowardFNR(InputSource source) { + return source instanceof StreamInputSource || source.isFromFilenameList(); } /** @@ -1762,7 +1973,7 @@ public Object consumeInputToTarget(final InputSource source) throws IOException RecordState inputState = new RecordState(source); this.nr++; - if (source.isFromFilenameList()) { + if (countsTowardFNR(source)) { this.fnr++; } return new StrNum(inputState.getRecordText(), decimalSeparator); diff --git a/src/main/java/io/jawk/jrt/StreamInputSource.java b/src/main/java/io/jawk/jrt/StreamInputSource.java index f3cfa72e..610297ea 100644 --- a/src/main/java/io/jawk/jrt/StreamInputSource.java +++ b/src/main/java/io/jawk/jrt/StreamInputSource.java @@ -23,6 +23,7 @@ */ import java.io.Closeable; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; @@ -73,6 +74,11 @@ public class StreamInputSource implements InputSource, Closeable { private String currentRecord; private boolean currentReaderExhausted; + // Per-file stepping state (BEGINFILE/ENDFILE and nextfile support) + private String currentFileOpenError; + private boolean currentPresentedToLoop; + private int lastArgumentIndex; + /** * Creates a stream-backed input source. * @@ -274,6 +280,7 @@ private String nextArgument() { } String arg = jrt.toAwkString(argValue); if (!arg.isEmpty()) { + lastArgumentIndex = idx; return arg; } } @@ -331,6 +338,9 @@ private boolean prepareNextReader() throws IOException { new InputStreamReader(defaultInput, StandardCharsets.UTF_8), jrt.getRSString()); jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); return true; } closeCurrentReaderIfFileStream(); @@ -346,11 +356,11 @@ private boolean prepareNextReader() throws IOException { new InputStreamReader(defaultInput, StandardCharsets.UTF_8), jrt.getRSString()); jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); return true; } - if (partitioningReader != null) { - jrt.setNR(jrt.getNR() + 1); - } } else { closeCurrentReaderIfFileStream(); partitioningReader = new PartitioningReader( @@ -359,12 +369,186 @@ private boolean prepareNextReader() throws IOException { true); jrt.setFILENAMEViaJrt(jrt.toInputScalar(arg)); jrt.setFNR(0L); + jrt.setARGIND(Long.valueOf(lastArgumentIndex)); + // gawk clears ERRNO whenever the main input advances + // successfully + jrt.setERRNO(""); ready = true; } } return true; } + /** + * Advance to the next input file for the per-file main input loop used + * when BEGINFILE/ENDFILE rules or {@code nextfile} are present. Variable + * assignment arguments are applied along the way, exactly like + * {@link #nextRecord()} does when it crosses a file boundary. + *

+ * On success, FILENAME, FNR, ARGIND, and ERRNO are updated and {@code $0} + * is cleared, so the BEGINFILE rules observe the new file before any + * record is read. A file that cannot be opened is still reported as + * available: ERRNO carries the error description and + * {@link #getCurrentFileOpenError()} returns it until the next advance, + * enabling gawk's non-fatal BEGINFILE error handling. + *

+ * + * @return {@code true} when a new input file (or the initial stdin + * stream) is current; {@code false} when input is exhausted + * @throws IOException if an I/O error occurs while traversing ARGV + */ + public boolean advanceToNextFile() throws IOException { + initializeArgList(); + + // Adopt a reader already opened by a non-redirected getline that ran + // before the per-file loop (e.g. in a BEGIN rule): it is the current + // input file, already positioned after the records getline consumed. + if (!currentPresentedToLoop + && partitioningReader != null + && !currentReaderExhausted + && currentFileOpenError == null) { + currentPresentedToLoop = true; + return true; + } + + currentFileOpenError = null; + arglistMaxKey = computeMaxArgvKey(); + hasFilenames = detectFilenames(); + while (true) { + String arg = nextArgument(); + if (arg == null) { + // ARGC/ARGV may have changed while evaluating assignments. + hasFilenames = detectFilenames(); + if (partitioningReader == null && !hasFilenames) { + return presentDefaultInput(); + } + closeCurrentReaderIfFileStream(); + return false; + } + if (arg.indexOf('=') > 0) { + setFilelistVariable(arg); + // Recompute bounds so ARGC changes are reflected immediately. + arglistMaxKey = computeMaxArgvKey(); + hasFilenames = detectFilenames(); + if (partitioningReader == null && !hasFilenames) { + return presentDefaultInput(); + } + } else { + closeCurrentReaderIfFileStream(); + partitioningReader = null; + currentReaderExhausted = false; + currentPresentedToLoop = true; + jrt.setFILENAMEViaJrt(jrt.toInputScalar(arg)); + beginFileState(lastArgumentIndex); + currentFileOpenError = openCurrentFile(arg); + if (currentFileOpenError != null) { + jrt.setERRNO(currentFileOpenError); + } + return true; + } + } + } + + /** + * Reads the next record of the current input file only, never advancing + * to the next input file. Used by the per-file main input loop so that + * ENDFILE rules can run at each file boundary. + * + * @return {@code true} when a record is available; {@code false} at the + * end of the current input file + * @throws IOException if an I/O error occurs + */ + public boolean nextRecordInCurrentFile() throws IOException { + if (partitioningReader == null || currentReaderExhausted || currentFileOpenError != null) { + return false; + } + String nextRecord = partitioningReader.readRecord(); + if (nextRecord == null) { + currentReaderExhausted = true; + return false; + } + currentRecord = nextRecord; + currentFromFilenameList = partitioningReader.fromFilenameList(); + return true; + } + + /** + * Returns the error description recorded when the current input file + * could not be opened by {@link #advanceToNextFile()}, or {@code null} + * when the current input is readable. + * + * @return the pending open error, or {@code null} + */ + public String getCurrentFileOpenError() { + return currentFileOpenError; + } + + /** + * Presents the default input stream (usually stdin) as the current and + * only input "file" for the per-file main input loop. + * + * @return always {@code true} + */ + private boolean presentDefaultInput() { + partitioningReader = new PartitioningReader( + new InputStreamReader(defaultInput, StandardCharsets.UTF_8), + jrt.getRSString()); + currentPresentedToLoop = true; + jrt.setFILENAMEViaJrt(jrt.toInputScalar("")); + beginFileState(0); + return true; + } + + /** + * Resets the per-file special variables observed by BEGINFILE rules: FNR + * and {@code $0} are cleared, ERRNO is emptied, and ARGIND designates the + * ARGV entry being processed. + * + * @param argvIndex the ARGV index of the new current file, or {@code 0} + * for the default input stream + */ + private void beginFileState(int argvIndex) { + jrt.setFNR(0L); + jrt.setERRNO(""); + jrt.setARGIND(Long.valueOf(argvIndex)); + jrt.setInputLine(""); + } + + /** + * Attempts to open the given filename as the current input file. + * + * @param arg the filename to open + * @return {@code null} on success, or a gawk-style error description when + * the file cannot be opened for reading + */ + private String openCurrentFile(String arg) { + File file = new File(arg); + if (file.isDirectory()) { + return "Is a directory"; + } + if (!file.exists()) { + return "No such file or directory"; + } + try { + partitioningReader = new PartitioningReader( + new InputStreamReader(new FileInputStream(arg), StandardCharsets.UTF_8), + jrt.getRSString(), + true); + return null; + } catch (IOException e) { + String message = e.getMessage(); + if (message == null || message.isEmpty()) { + return "Permission denied"; + } + // Java prefixes the failing path: "path (reason)". Keep the reason. + int open = message.lastIndexOf('('); + if (open >= 0 && message.endsWith(")")) { + return message.substring(open + 1, message.length() - 1); + } + return message; + } + } + /** * Closes the current {@link PartitioningReader} if it wraps a file stream * (not {@code defaultInput}). This prevents file-descriptor leaks when diff --git a/src/main/java/io/jawk/util/AwkSettings.java b/src/main/java/io/jawk/util/AwkSettings.java index 909843da..f30df0cd 100644 --- a/src/main/java/io/jawk/util/AwkSettings.java +++ b/src/main/java/io/jawk/util/AwkSettings.java @@ -277,7 +277,9 @@ public void setUseSortedArrayKeys(boolean useSortedArrayKeys) { /** * Whether POSIX compile-time behavior is enforced, rejecting gawk syntax * such as arrays of arrays ({@code a[i][j]}, subarray operands like - * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}). + * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}), + * and treating the gawk-specific {@code BEGINFILE} / {@code ENDFILE} + * patterns as ordinary identifiers. * * @return {@code true} when POSIX compile-time behavior is enforced */ @@ -289,7 +291,8 @@ public boolean isPosix() { * Enables or disables POSIX compile-time behavior. When enabled, gawk * syntax such as arrays of arrays ({@code a[i][j]}, subarray operands like * {@code split(..., a[i])} or {@code for (k in a[i])}) and typed regexp - * literals ({@code @/re/}) is rejected. + * literals ({@code @/re/}) is rejected, and the gawk-specific + * {@code BEGINFILE} / {@code ENDFILE} patterns are not special. * * @param posix {@code true} to enforce POSIX compile-time behavior */ diff --git a/src/site/markdown/cli-reference.md b/src/site/markdown/cli-reference.md index fd514a4b..b3d9338a 100644 --- a/src/site/markdown/cli-reference.md +++ b/src/site/markdown/cli-reference.md @@ -49,7 +49,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext > - `-r` disables Jawk's default trapping of `IllegalFormatException` for `printf` and `sprintf`. > - `--locale ` sets the locale through `Locale.forLanguageTag(...)`. > - `-t` keeps associative array keys sorted. -> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays and typed regexp literals (`@/re/`). +> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns. > - `JAWK_PERSISTENT_MEMORY` can also point at the persistent-memory file when you do not want to pass `--persist` explicitly. `--persist` wins when both are present. > > - Extensions and sandbox @@ -78,7 +78,7 @@ java -jar jawk-${project.version}-standalone.jar --list-ext - `--profile` is an executing mode. It keeps normal AWK output on stdout and writes the profiling report to stderr after execution finishes. - `--profile=` keeps normal AWK output on stdout and writes only the profiling report to the file. - `-S` affects compilation and execution, not just runtime behavior. -- `--posix` currently disables arrays-of-arrays syntax and related subarray-only operands in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. +- `--posix` currently disables arrays-of-arrays syntax and related subarray-only operands, and stops treating gawk's `BEGINFILE` / `ENDFILE` patterns as special, in order to keep CLI compilation aligned with classic POSIX-style AWK expectations. - `--posix` is rejected together with `-L`, because loading precompiled tuples bypasses source compilation entirely. - `-L` lets you skip source compilation, but the loaded tuples must still be compatible with the current runtime. - `-f` and `-L` are distinct paths: source files compile now, tuple files load now. diff --git a/src/site/markdown/cli.md b/src/site/markdown/cli.md index 9a577444..197d1903 100644 --- a/src/site/markdown/cli.md +++ b/src/site/markdown/cli.md @@ -75,6 +75,26 @@ $ java -jar jawk-${project.version}-standalone.jar -F : '{ print FILENAME ":" FN Jawk follows the usual AWK distinction between the script itself and the remaining operands. Files and `name=value` operands after the script are visible through `ARGV` and `ARGC`. +### BEGINFILE and ENDFILE Rules + +Jawk supports gawk's `BEGINFILE` and `ENDFILE` special patterns, which hook into the command-line file processing loop. `BEGINFILE` rules run just before the first record of each input file is read, with `FILENAME` set, `FNR` at 0, `$0` cleared, and `ARGIND` designating the `ARGV` entry being processed. `ENDFILE` rules run when the last record of a file has been consumed — even for empty files — and before the `END` rules for the last file. + +When a `BEGINFILE` rule is present, a file that cannot be opened is no longer an immediate fatal error: `ERRNO` carries the error description (for example `No such file or directory` or `Is a directory`), and the script can skip the file with `nextfile`. If the `BEGINFILE` rule does not skip it, the usual fatal error is raised: + +```shell-session +$ java -jar jawk-${project.version}-standalone.jar ' + BEGINFILE { if (ERRNO != "") { printf "skipping %s (%s)\n", FILENAME, ERRNO; nextfile } } + { print FILENAME ":" FNR ":" $0 }' good.txt missing.txt +good.txt:1:hello +skipping missing.txt (No such file or directory) +``` + +The `nextfile` statement is also available in ordinary rules — including from user-defined functions — and abandons the rest of the current input file after running the `ENDFILE` rules. `next` is rejected inside `BEGINFILE`/`ENDFILE` rules, `nextfile` is rejected inside `ENDFILE`, `BEGIN`, and `END` rules, and only redirected forms of `getline` (such as `getline line < "file"`) may be used inside `BEGINFILE`/`ENDFILE`, all matching gawk's restrictions. + +When `BEGINFILE`/`ENDFILE` rules are present, a non-redirected `getline` in an ordinary rule never crosses a file boundary: it reports end-of-input at the end of the current file, and the main loop then runs the `ENDFILE` and `BEGINFILE` rules before the next file's records are processed. (gawk instead lets such a `getline` pull in the next file's first record, firing the hooks mid-statement.) Without `BEGINFILE`/`ENDFILE` rules, `getline` keeps the classic AWK behavior of streaming across input files. + +As in gawk, `BEGINFILE` and `ENDFILE` are gawk extensions: with `--posix` they are not special and parse as ordinary identifiers. + ## Pass Variables Use `-v` for variables that must exist before `BEGIN` runs: diff --git a/src/site/markdown/extensions.md b/src/site/markdown/extensions.md index bacc7f2f..86bcf1d1 100644 --- a/src/site/markdown/extensions.md +++ b/src/site/markdown/extensions.md @@ -76,6 +76,8 @@ That keeps extension availability explicit and local to the embedding code. `asort()`, `asorti()`, and the `for (index in array)` statement honor `PROCINFO["sorted_in"]` with gawk's predefined comparison modes: `@unsorted`, `@ind_str_asc`, `@ind_num_asc`, `@val_str_asc`, `@val_num_asc`, `@val_type_asc`, and their `_desc` counterparts. String comparisons ignore case when `IGNORECASE` is non-zero. +Beyond the extension functions, the interpreter itself implements gawk's `BEGINFILE` / `ENDFILE` special patterns, the `nextfile` statement, and the `ERRNO` and `ARGIND` special variables (see the [CLI guide](cli.html#BEGINFILE_and_ENDFILE_Rules)). Like the other gawk-specific syntax, `BEGINFILE` and `ENDFILE` are not special in POSIX mode. + Scripts that reference `SYMTAB` or `FUNCTAB` get honest, Jawk-shaped content, populated by the runtime itself (outside POSIX mode): `SYMTAB` holds the names of the program's globals, Jawk's special variables, and `-v`/host-supplied variables; `FUNCTAB` holds the names of the program's user-defined functions plus the loaded extensions' function keywords. Command-line `name=value` operand assignments update `SYMTAB` live, as in gawk; ordinary in-script assignments are not reflected (the array is a startup snapshot, not gawk's live view). As in gawk, assigning a scalar to `SYMTAB` or `FUNCTAB` is a runtime error. > [!NOTE] diff --git a/src/site/markdown/java-input.md b/src/site/markdown/java-input.md index b3f7bdae..641d6fbd 100644 --- a/src/site/markdown/java-input.md +++ b/src/site/markdown/java-input.md @@ -21,7 +21,9 @@ An `InputSource` implementation controls four things: - `nextRecord()` advances to the next record and returns `true` while input remains - `getRecordText()` provides the current `$0`, or `null` if you only expose fields - `getFields()` provides `$1..$NF`, or `null` if Jawk should split `$0` using `FS` -- `isFromFilenameList()` tells Jawk whether the current record should behave like file-list input for counters such as `FNR` +- `isFromFilenameList()` tells Jawk whether the current record should behave like file-list input for counters such as `FNR` (records from Jawk's built-in stream input always advance `FNR`, including standard input, per POSIX) + +A custom `InputSource` replaces the ARGV file-list traversal entirely: `name=value` runtime arguments are not applied and remain visible only as plain `ARGV` entries. Use `-v`-style variables or `putVariable(...)` to pass values to the script. When both record text and fields are available, Jawk uses: diff --git a/src/site/markdown/java.md b/src/site/markdown/java.md index e38ba7bf..a4000abc 100644 --- a/src/site/markdown/java.md +++ b/src/site/markdown/java.md @@ -36,14 +36,14 @@ Awk awk = new Awk(settings); | `setLocale(Locale)` | `Locale.US` | Locale for numeric output formatting | | `setDefaultRS(String)` | Platform line separator | Default value for `RS`, the record separator | | `setUseSortedArrayKeys(boolean)` | `false` | Whether to keep associative array keys in sorted order | -| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, and typed regexp literals (`@/re/`) | +| `setPosix(boolean)` | `false` | Enforce POSIX compile-time behavior, rejecting gawk syntax such as `a[i][j]`, `split(..., a[i])`, typed regexp literals (`@/re/`), and the `BEGINFILE` / `ENDFILE` special patterns | | `putVariable(String, Object)` | Empty map | Pre-set variables available before `BEGIN` | Output destination is specified per-call on the builder (`execute()`, `execute(PrintStream)`, `execute(OutputStream)`, `execute(Appendable)`, or `execute(AwkSink)`). See the [Custom Output](java-output.html) guide for details. For more on passing variables to scripts, see [Variables and Arguments](java-variables.html). -By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`). Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, and typed regexp literals (`@/re/`): +By default, Jawk accepts both classic multi-dimensional array syntax (`a[i, j]`) and gawk-style arrays of arrays (`a[i][j]`), and treats gawk's `BEGINFILE` / `ENDFILE` patterns as special. Enable POSIX mode when you need strict classic AWK parsing; it rejects arrays of arrays, subarray operands in array-only positions such as `split(..., a[i])`, `for (k in a[i])`, and `"x" in a[i]`, and typed regexp literals (`@/re/`), and it stops treating `BEGINFILE` / `ENDFILE` as special patterns: ```java AwkSettings settings = new AwkSettings(); diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 55ab6b62..88686f90 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1508,20 +1508,23 @@ public void executePersistingGlobalsPersistsSeededCompiledAndUncompiledUserGloba } @Test - public void executePersistingGlobalsDefersRuntimeArgumentAssignmentsUntilInputTraversal() throws Exception { + public void executePersistingGlobalsIgnoresRuntimeArgumentAssignmentsWithCustomInput() throws Exception { + // name=value runtime arguments belong to the ARGV file-list traversal, + // which a custom InputSource replaces entirely: they are never applied, + // so nothing is persisted for them either. AwkProgram seedFromRuntimeArguments = AWK.compile("BEGIN { print x } { x = x } END { print x }"); AwkProgram readValue = AWK.compile("BEGIN { print x }"); try (AVM avm = AWK.createAvm()) { assertEquals( - "\n1\n", + "\n\n", executePersistent( avm, seedFromRuntimeArguments, Collections.singletonList("x=1"), Collections.emptyMap(), new SingleRecordInputSource("record"))); - assertEquals("1\n", executePersistent(avm, readValue)); + assertEquals("\n", executePersistent(avm, readValue)); } } diff --git a/src/test/java/io/jawk/AwkTupleOptimizationTest.java b/src/test/java/io/jawk/AwkTupleOptimizationTest.java index 8c55109d..2cec4e74 100644 --- a/src/test/java/io/jawk/AwkTupleOptimizationTest.java +++ b/src/test/java/io/jawk/AwkTupleOptimizationTest.java @@ -731,7 +731,6 @@ private static boolean usesAddress(Opcode opcode) { case IS_EMPTY_KEYLIST: case CONSUME_INPUT: case CALL_FUNCTION: - case SET_EXIT_ADDRESS: return true; default: return false; diff --git a/src/test/java/io/jawk/BeginFileEndFileTest.java b/src/test/java/io/jawk/BeginFileEndFileTest.java new file mode 100644 index 00000000..b4a5ecef --- /dev/null +++ b/src/test/java/io/jawk/BeginFileEndFileTest.java @@ -0,0 +1,500 @@ +package io.jawk; + +/*- + * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲ + * Jawk + * ჻჻჻჻჻჻ + * Copyright (C) 2006 - 2026 MetricsHub + * ჻჻჻჻჻჻ + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱ + */ + +import org.junit.Test; +import io.jawk.jrt.AwkRuntimeException; + +/** + * Tests for the gawk-specific BEGINFILE and ENDFILE special patterns, the + * {@code nextfile} statement, and the ERRNO / ARGIND special variables that + * support them. BEGINFILE and ENDFILE are gawk extensions and are therefore + * not special in POSIX mode. + */ +public class BeginFileEndFileTest { + + @Test + public void beginFileAndEndFileFireOncePerFile() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE/ENDFILE fire once per input file, in order") + .script( + "BEGIN { print \"begin\" }" + + " BEGINFILE { print \"bf\", FNR }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\", FNR }" + + " END { print \"end\", NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines( + "begin", + "bf 0", + "r:a1", + "r:a2", + "ef 2", + "bf 0", + "r:b1", + "ef 1", + "end 3") + .runAndAssert(); + } + + @Test + public void endFileRunsForEmptyFiles() throws Exception { + AwkTestSupport + .awkTest("ENDFILE runs even for empty input files") + .script("BEGINFILE { print \"bf\" } ENDFILE { print \"ef\", FNR }") + .file("empty", "") + .operand("{{empty}}") + .expectLines("bf", "ef 0") + .runAndAssert(); + } + + @Test + public void beginFileSeesFilenameAndClearedRecord() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE sees FILENAME, FNR=0, and a cleared $0") + .script( + "BEGINFILE { ok = (FILENAME == \"{{f1}}\"); print ok, FNR, \"[\" $0 \"]\", NF }" + + " { x = $0 }") + .file("f1", "one two\n") + .operand("{{f1}}") + .expectLines("1 0 [] 0") + .runAndAssert(); + } + + @Test + public void beginFileAndEndFileFireForStandardInput() throws Exception { + AwkTestSupport + .awkTest("BEGINFILE/ENDFILE fire for standard input") + .script( + "BEGINFILE { print \"bf[\" FILENAME \"]\", FNR }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef[\" FILENAME \"]\", FNR }") + .stdin("s1\ns2\n") + .expectLines("bf[] 0", "r:s1", "r:s2", "ef[] 2") + .runAndAssert(); + } + + @Test + public void errnoAllowsSkippingUnreadableFile() throws Exception { + AwkTestSupport + .cliTest("ERRNO + nextfile skips a file that cannot be opened") + .script( + "BEGINFILE { if (ERRNO != \"\") { print \"skipping\"; nextfile } print \"bf\" }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\n") + .operand("{{f1}}", "no/such/file", "{{f1}}") + .expectLines("bf", "r:a1", "ef", "skipping", "bf", "r:a1", "ef") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void unreadableFileIsFatalWhenBeginFileDoesNotSkipIt() throws Exception { + AwkTestSupport + .awkTest("open failure stays fatal when BEGINFILE does not call nextfile") + .script("BEGINFILE { print \"bf\" } { print }") + .operand("no/such/file") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileInBeginFileRunsEndFileForReadableFile() throws Exception { + AwkTestSupport + .awkTest("nextfile in BEGINFILE still runs ENDFILE for a readable file") + .script( + "BEGINFILE { print \"bf\"; nextfile }" + + " { print \"r:\" $0 }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("bf", "ef") + .runAndAssert(); + } + + @Test + public void nextfileSkipsRestOfCurrentFile() throws Exception { + AwkTestSupport + .awkTest("nextfile skips the rest of the current file") + .script("{ print FILENAME \":\" $0; if (FNR == 1) nextfile }") + .file("f1", "A1\nA2\n") + .file("f2", "B1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("{{f1}}:A1", "{{f2}}:B1") + .runAndAssert(); + } + + @Test + public void nextfileWorksFromUserDefinedFunction() throws Exception { + AwkTestSupport + .awkTest("nextfile unwinds user-defined function calls") + .script( + "function skip() { nextfile; print \"unreached\" }" + + " BEGINFILE { print \"bf\" }" + + " { print \"r:\" $0; skip() }" + + " ENDFILE { print \"ef\" }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("bf", "r:a1", "ef", "bf", "r:b1", "ef") + .runAndAssert(); + } + + @Test + public void nextfileWithoutBeginFileOrEndFileRules() throws Exception { + AwkTestSupport + .awkTest("nextfile works without any BEGINFILE/ENDFILE rules") + .script("function f() { nextfile } { print $0; f() } END { print \"nr=\" NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("a1", "b1", "nr=2") + .runAndAssert(); + } + + @Test + public void nextIsRejectedInsideBeginFile() throws Exception { + AwkTestSupport + .awkTest("next cannot be called from a BEGINFILE rule") + .script("BEGINFILE { next } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextIsRejectedInsideEndFile() throws Exception { + AwkTestSupport + .awkTest("next cannot be called from an ENDFILE rule") + .script("ENDFILE { next } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileIsRejectedInsideEndFile() throws Exception { + AwkTestSupport + .awkTest("nextfile cannot be called from an ENDFILE rule") + .script("ENDFILE { nextfile } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileIsRejectedInsideBeginAndEnd() throws Exception { + AwkTestSupport + .awkTest("nextfile cannot be called from a BEGIN rule") + .script("BEGIN { nextfile } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + AwkTestSupport + .awkTest("nextfile cannot be called from an END rule") + .script("{ print } END { nextfile }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nextfileInEndViaFunctionIsARuntimeError() throws Exception { + AwkTestSupport + .awkTest("nextfile reached from an END rule through a function is fatal") + .script("function f() { nextfile } { } END { f() }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsRejectedInsideBeginFile() throws Exception { + // Direct use is a compile-time error; see the ViaFunction cases for + // the runtime detection. + AwkTestSupport + .awkTest("non-redirected getline is invalid inside BEGINFILE") + .script("BEGINFILE { getline } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsRejectedInsideEndFile() throws Exception { + // Direct use of the "getline var" form is a compile-time error too. + AwkTestSupport + .awkTest("non-redirected getline is invalid inside ENDFILE") + .script("ENDFILE { getline line } { print }") + .stdin("x\n") + .expectThrow(RuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsFatalInsideBeginFileViaFunction() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline through a function is invalid inside BEGINFILE") + .script("function g() { getline } BEGINFILE { g() } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void nonRedirectedGetlineIsFatalInsideEndFileViaFunction() throws Exception { + AwkTestSupport + .awkTest("non-redirected getline through a function is invalid inside ENDFILE") + .script("function g() { getline } ENDFILE { g() } { print }") + .stdin("x\n") + .expectThrow(AwkRuntimeException.class) + .runAndAssert(); + } + + @Test + public void getlineInActionStopsAtFileBoundaryWhenHooksArePresent() throws Exception { + // The per-file loop is the only place allowed to cross file + // boundaries when BEGINFILE/ENDFILE rules exist, so no file's hooks + // are ever skipped: getline reports end-of-input at the end of the + // current file instead of silently opening the next one. + AwkTestSupport + .awkTest("getline in an action does not cross file boundaries behind the hooks") + .script( + "BEGINFILE { print \"bf:\" FILENAME }" + + " { print \"r:\" $0; if ((getline x) > 0) print \"g:\" x; else print \"g:eof\" }" + + " ENDFILE { print \"ef:\" FILENAME }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines( + "bf:{{f1}}", + "r:a1", + "g:a2", + "ef:{{f1}}", + "bf:{{f2}}", + "r:b1", + "g:eof", + "ef:{{f2}}") + .runAndAssert(); + } + + @Test + public void getlineInActionStillCrossesFilesWithoutHooks() throws Exception { + // Without BEGINFILE/ENDFILE rules there are no per-file hooks to + // protect, so getline keeps the classic AWK behavior of streaming + // across input files — even when nextfile forces the per-file loop. + AwkTestSupport + .awkTest("getline in an action crosses file boundaries without hooks") + .script( + "function unused() { nextfile }" + + " { print \"r:\" $0; if ((getline x) > 0) print \"g:\" x }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("r:a1", "g:b1") + .runAndAssert(); + } + + @Test + public void redirectedGetlineIsAllowedInsideBeginFile() throws Exception { + AwkTestSupport + .awkTest("redirected getline is allowed inside BEGINFILE") + .script( + "BEGINFILE { if ((getline line < \"{{f2}}\") > 0) print \"got:\" line;" + + " close(\"{{f2}}\") }" + + " { print \"r:\" $0 }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}") + .expectLines("got:b1", "r:a1") + .runAndAssert(); + } + + @Test + public void argindTracksTheArgvIndexOfTheCurrentFile() throws Exception { + AwkTestSupport + .awkTest("ARGIND designates the ARGV entry of the current file") + .script("BEGINFILE { print \"bf\", ARGIND } END { print \"end\", ARGIND }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "x=1", "{{f2}}") + .expectLines("bf 1", "bf 3", "end 3") + .runAndAssert(); + } + + @Test + public void errnoIsAnAssignableSpecialVariable() throws Exception { + AwkTestSupport + .awkTest("ERRNO starts empty and is assignable") + .script("BEGIN { print \"[\" ERRNO \"]\"; ERRNO = \"boom\"; print ERRNO }") + .expectLines("[]", "boom") + .runAndAssert(); + } + + @Test + public void errnoIsClearedWhenMainInputOpensAFile() throws Exception { + // gawk clears ERRNO whenever the main input advances successfully, + // even without BEGINFILE/ENDFILE rules. + AwkTestSupport + .awkTest("ERRNO is cleared when an input file opens successfully") + .script("BEGIN { ERRNO = \"boom\" } { print \"[\" ERRNO \"]\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("[]") + .runAndAssert(); + } + + @Test + public void errnoIsClearedWhenMainInputReadsStandardInput() throws Exception { + AwkTestSupport + .awkTest("ERRNO is cleared when standard input is selected as main input") + .script("BEGIN { ERRNO = \"boom\" } { print \"[\" ERRNO \"]\" }") + .stdin("s1\n") + .expectLines("[]") + .runAndAssert(); + } + + @Test + public void exitInsideBeginFileRunsEndRules() throws Exception { + AwkTestSupport + .cliTest("exit in BEGINFILE skips ENDFILE and runs END") + .script( + "BEGINFILE { print \"bf\"; exit(3) }" + + " { print \"r\" }" + + " ENDFILE { print \"ef\" }" + + " END { print \"end\" }") + .file("f1", "a1\n") + .operand("{{f1}}") + .expectLines("bf", "end") + .expectExit(3) + .runAndAssert(); + } + + @Test + public void beginFileIsNotSpecialInPosixMode() throws Exception { + AwkTestSupport + .cliTest("--posix treats BEGINFILE/ENDFILE as plain identifiers") + .argument("--posix") + .script("BEGINFILE { print \"bf\" } ENDFILE { print \"ef\" } { print \"r:\" $0 }") + .stdin("x\n") + .expectLines("r:x") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void errnoAndArgindAreOrdinaryVariablesInPosixMode() throws Exception { + // Like gawk --posix: ERRNO and ARGIND are plain identifiers, usable + // as function parameters and as ordinary globals. + AwkTestSupport + .cliTest("--posix treats ERRNO and ARGIND as ordinary identifiers") + .argument("--posix") + .script( + "function f(ARGIND) { return ARGIND * 2 }" + + " function g(ERRNO) { return ERRNO \"x\" }" + + " BEGIN { ERRNO = \"a\"; print f(3), g(ERRNO) }") + .expectLines("6 ax") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void argindIsNotUpdatedAtFileBoundariesInPosixMode() throws Exception { + // Like gawk --posix: reading input files does not touch a + // user-assigned ARGIND. + AwkTestSupport + .cliTest("--posix keeps a user-assigned ARGIND across input files") + .argument("--posix") + .script("BEGIN { ARGIND = 99 } { print ARGIND \":\" $0 }") + .file("f1", "a1\n") + .file("f2", "b1\n") + .operand("{{f1}}", "{{f2}}") + .expectLines("99:a1", "99:b1") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void errnoIsAssignableThroughDashVInPosixMode() throws Exception { + // -v ERRNO=... must reach the ordinary global slot in POSIX mode. + AwkTestSupport + .cliTest("--posix -v ERRNO seeds the ordinary global") + .argument("--posix", "-v", "ERRNO=foo") + .script("BEGIN { print ERRNO }") + .expectLines("foo") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void beginFileIsUsableAsVariableInPosixMode() throws Exception { + AwkTestSupport + .cliTest("--posix allows BEGINFILE as an ordinary variable name") + .argument("--posix") + .script("BEGIN { BEGINFILE = 42; ENDFILE = 8; print BEGINFILE + ENDFILE }") + .expectLines("50") + .expectExit(0) + .runAndAssert(); + } + + @Test + public void regexpBracketExpressionMayContainSlash() throws Exception { + AwkTestSupport + .awkTest("a slash inside a bracket expression does not end the regexp") + .script("BEGIN { s = \"a/b\"; gsub(/[/]/, \"-\", s); print s }") + .expectLines("a-b") + .runAndAssert(); + } + + @Test + public void regexpBracketExpressionMayContainSlashAfterCharacterClass() throws Exception { + // The ']' closing a POSIX character class such as [:digit:] must not + // be taken as the end of the enclosing bracket expression, so the + // slash that follows still belongs to the regexp literal. + AwkTestSupport + .awkTest("a slash after a POSIX character class does not end the regexp") + .script("BEGIN { s = \"a/b\"; gsub(/[[:digit:]/]/, \"-\", s); print s }") + .expectLines("a-b") + .runAndAssert(); + } + + @Test + public void assignmentOperandsDoNotConsumeRecordsInPerFileLoop() throws Exception { + // A name=value operand between input files applies its assignment + // without reading a record, so NR stays contiguous across files. + AwkTestSupport + .awkTest("assignment operands leave NR untouched in the per-file loop") + .script( + "BEGINFILE { print \"bf\", NR }" + + " { print NR \":\" $0 }" + + " END { print \"end\", NR }") + .file("f1", "a1\na2\n") + .file("f2", "b1\n") + .operand("{{f1}}", "x=1", "{{f2}}") + .expectLines("bf 0", "1:a1", "2:a2", "bf 2", "3:b1", "end 3") + .runAndAssert(); + } +} diff --git a/src/test/java/io/jawk/InputSourceTest.java b/src/test/java/io/jawk/InputSourceTest.java index f73ff62b..d733fb41 100644 --- a/src/test/java/io/jawk/InputSourceTest.java +++ b/src/test/java/io/jawk/InputSourceTest.java @@ -83,6 +83,23 @@ public void testNrCountsRecordsFromInputSource() throws Exception { .runAndAssert(); } + @Test + public void testFnrFollowsIsFromFilenameListContract() throws Exception { + // For custom InputSource implementations, isFromFilenameList() keeps + // controlling whether records advance FNR (unlike the built-in + // stream input, where stdin also counts). + awkTest("input source FNR gated by isFromFilenameList") + .script("{ print FNR, NR }") + .withInputSource( + new TableInputSource( + Arrays + .asList( + Collections.singletonList("r1"), + Collections.singletonList("r2")))) + .expectLines("0 1", "0 2") + .runAndAssert(); + } + @Test public void testDollarZeroUsesInputSourceRecord() throws Exception { awkTest("input source controls dollar zero text") @@ -216,12 +233,15 @@ public void testGetlineIntoVariableMaterializesCurrentRecordBeforeAdvance() thro } @Test - public void testFileListVariableAssignmentsAreAppliedWithInputSource() throws Exception { - awkTest("input source still applies name=value operands") - .script("{ print x }") + public void testFileListVariableAssignmentsAreIgnoredWithInputSource() throws Exception { + // name=value operands belong to the ARGV file-list traversal, which a + // custom InputSource replaces entirely: they are never applied and + // remain visible only as plain ARGV entries. + awkTest("input source ignores name=value operands") + .script("{ print \"[\" x \"]\" ARGV[1] }") .operand("x=42") .withInputSource(new TableInputSource(Collections.singletonList(Collections.singletonList("row")))) - .expectLines("42") + .expectLines("[]x=42") .runAndAssert(); } diff --git a/src/test/java/io/jawk/JRTConsumeInputTest.java b/src/test/java/io/jawk/JRTConsumeInputTest.java index 3ff035c6..74808449 100644 --- a/src/test/java/io/jawk/JRTConsumeInputTest.java +++ b/src/test/java/io/jawk/JRTConsumeInputTest.java @@ -32,19 +32,20 @@ public class JRTConsumeInputTest { /** * Ensures that variable assignments interleaved with filenames in - * {@code ARGV} correctly advance {@code NR}. + * {@code ARGV} apply without consuming a record: per POSIX (and gawk), + * {@code NR} only counts records actually read. * * @throws Exception if the AWK invocation fails */ @Test - public void testVariableAssignmentBetweenFilesIncrementsNR() throws Exception { + public void testVariableAssignmentBetweenFilesDoesNotAdvanceNR() throws Exception { AwkTestSupport - .awkTest("variable assignments interleaved with filenames advance NR") + .awkTest("variable assignments interleaved with filenames leave NR untouched") .file("file1", "a\n") .file("file2", "b\n") .script("{ next } \nEND { print NR }") .operand("{{file1}}", "X=1", "{{file2}}") - .expectLines("3") + .expectLines("2") .runAndAssert(); } diff --git a/src/test/java/io/jawk/PosixConformanceTest.java b/src/test/java/io/jawk/PosixConformanceTest.java index 88a1def1..d8ba064a 100644 --- a/src/test/java/io/jawk/PosixConformanceTest.java +++ b/src/test/java/io/jawk/PosixConformanceTest.java @@ -387,6 +387,26 @@ public void posix51NrAndFnrCounts() throws Exception { .runAndAssert(); } + @Test + public void posix51FnrCountsRecordsOnStandardInput() throws Exception { + AwkTestSupport + .awkTest("POSIX 5.1 FNR counts records read from standard input") + .script("{ print FNR, NR }") + .stdin("a\nb\nc\n") + .expectLines("1 1", "2 2", "3 3") + .runAndAssert(); + } + + @Test + public void posix51FnrCountsGetlineRecordsOnStandardInput() throws Exception { + AwkTestSupport + .awkTest("POSIX 5.1 plain getline from standard input advances FNR and NR") + .script("BEGIN { getline; getline line; print FNR, NR }") + .stdin("a\nb\nc\n") + .expectLines("2 2") + .runAndAssert(); + } + @Test public void posix52FilenameUnsetInBegin() throws Exception { AwkTestSupport @@ -584,7 +604,6 @@ public void posix87NextSkipsCurrentRecord() throws Exception { @Test public void posix88NextfileSkipsRestOfCurrentFile() throws Exception { - Assume.assumeTrue("nextfile is not supported by Jawk", false); AwkTestSupport .awkTest("POSIX 8.8 nextfile skips rest of current file") .script("{print FILENAME \":\" $0; if (FNR==1) nextfile}") diff --git a/src/test/java/io/jawk/StreamInputSourceTest.java b/src/test/java/io/jawk/StreamInputSourceTest.java index 7200cafa..a1b9eddb 100644 --- a/src/test/java/io/jawk/StreamInputSourceTest.java +++ b/src/test/java/io/jawk/StreamInputSourceTest.java @@ -84,6 +84,25 @@ public void testFileInputThroughInjectedLongKeyArgvMap() throws Exception { .runAndAssert(); } + @Test + public void testFileInputThroughInjectedIntegerKeyArgvMap() throws Exception { + // A host-supplied ARGV wins over the operand list regardless of the + // numeric key type it uses. + Path file = Files.createTempFile(AwkTestSupport.sharedTempDirectory(), "argv-int-", ".txt"); + Files.write(file, "mapped".getBytes(StandardCharsets.UTF_8)); + Map argv = new LinkedHashMap<>(); + argv.put(0, "jawk"); + argv.put(1, file.toString()); + + awkTest("StreamInputSource reads injected ARGV map with Integer keys") + .script("BEGIN { _ = ARGV[0] } { print $0 }") + .preassign("ARGV", argv) + .file("operand.txt", "operand\n") + .operand("{{operand.txt}}") + .expectLines("mapped") + .runAndAssert(); + } + @Test public void testMultipleFilesWithFnrReset() throws Exception { awkTest("StreamInputSource resets FNR per file") @@ -107,6 +126,19 @@ public void testVariableAssignmentInArgv() throws Exception { .runAndAssert(); } + @Test + public void testVariableAssignmentBetweenFilesDoesNotConsumeARecord() throws Exception { + // A name=value operand between two input files applies its assignment + // without reading a record: NR stays contiguous across the boundary. + awkTest("StreamInputSource name=value operands leave NR untouched") + .script("{ print NR \":\" x \":\" $0 } END { print \"end:\" NR }") + .file("a.txt", "A1\nA2\n") + .file("b.txt", "B1\n") + .operand("{{a.txt}}", "x=1", "{{b.txt}}") + .expectLines("1::A1", "2::A2", "3:1:B1", "end:3") + .runAndAssert(); + } + @Test public void testGetlineFromStdin() throws Exception { awkTest("StreamInputSource getline from stdin")