From 0b5425b8d23df488c154f540e058a643d8cd13cc Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 8 Oct 2026 14:48:38 +0200 Subject: [PATCH 1/3] Follow gawk's backslash rules in sub()/gsub() replacement text In the replacement text of sub() and gsub(), Jawk applied the POSIX backslash rules (\\ is a literal backslash) in every mode. gawk's default mode follows its own documented rules instead: \\\& is a literal \&, \\\\ a literal \\, \\& a literal backslash followed by the match, and any other backslash is kept as is, so \\ stays \\. sub() and gsub() now follow gawk's rules by default and the POSIX ones under --posix, which is what gawk --posix, mawk and One True Awk do. The POSIX switch reaches the runtime through JRT.setPosix(), set by AVM from the settings. gensub() keeps its own gawk rules. JRT.prepareReplacement() is split into prepareSubReplacement(), which takes the POSIX switch, and prepareGensubReplacement(), which takes the highest capture group number, instead of one method whose boolean and int overloads meant different things. The gawk backgsub compatibility test now passes. Closes #614 Co-Authored-By: Claude Fable 5.1 --- src/main/java/io/jawk/backend/AVM.java | 1 + src/main/java/io/jawk/ext/GawkExtension.java | 2 +- src/main/java/io/jawk/jrt/JRT.java | 167 ++++++++++++------ src/main/java/io/jawk/util/AwkSettings.java | 4 +- src/site/markdown/behavior-changes.md | 8 + src/site/markdown/cli-reference.md | 2 +- src/site/markdown/compatibility.md.vm | 1 + src/test/java/io/jawk/AwkTest.java | 16 ++ .../io/jawk/jrt/PrepareReplacementTest.java | 92 +++++++--- 9 files changed, 205 insertions(+), 88 deletions(-) diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index 9c5d0dc2..fa549a9c 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -246,6 +246,7 @@ public AVM( executionSpecialVariables = baseSpecialVariables; jrt = createJrt(); + jrt.setPosix(this.settings.isPosix()); initExtensions(); } diff --git a/src/main/java/io/jawk/ext/GawkExtension.java b/src/main/java/io/jawk/ext/GawkExtension.java index 6bcd79eb..ff56d2eb 100644 --- a/src/main/java/io/jawk/ext/GawkExtension.java +++ b/src/main/java/io/jawk/ext/GawkExtension.java @@ -238,7 +238,7 @@ public String gensub(@JawkRegexp Object regexp, Object replacement, Object how, pattern = getJrt().caseAwarePattern(pattern); Object targetValue = target == null ? getJrt().getInputLine() : target; Matcher matcher = pattern.matcher(toAwkString(targetValue)); - String repl = JRT.prepareReplacement(toAwkString(replacement), pattern.matcher("").groupCount()); + String repl = JRT.prepareGensubReplacement(toAwkString(replacement), pattern.matcher("").groupCount()); String selector = toAwkString(how); // gawk: any string beginning with 'g' or 'G' selects a global replacement if (!selector.isEmpty() && (selector.charAt(0) == 'g' || selector.charAt(0) == 'G')) { diff --git a/src/main/java/io/jawk/jrt/JRT.java b/src/main/java/io/jawk/jrt/JRT.java index 7dc44910..a39de718 100644 --- a/src/main/java/io/jawk/jrt/JRT.java +++ b/src/main/java/io/jawk/jrt/JRT.java @@ -143,6 +143,9 @@ public class JRT { private Object ignorecase = Long.valueOf(0L); /** Precomputed truth of IGNORECASE, consulted by every regexp operation. */ private boolean ignoreCase; + + /** Whether sub()/gsub() replacement text follows the POSIX backslash rules instead of gawk's. */ + private boolean posix; /** Case-insensitive twins of precompiled patterns; created on first use. */ private Map caseInsensitivePatterns; /** Compiled case-sensitive dynamic (string) regexps, keyed by expression text; created on first use. */ @@ -299,6 +302,17 @@ public JRT(VariableManager vm, Locale locale, AwkSink awkSink, PrintStream error this.subsep = Awk.DEFAULT_SUBSEP; } + /** + * Selects the POSIX backslash rules for {@code sub()} and {@code gsub()} + * replacement text instead of gawk's; see + * {@link #prepareSubReplacement(String, boolean)}. + * + * @param posix {@code true} for the POSIX rules + */ + public void setPosix(boolean posix) { + this.posix = posix; + } + /** * Sets the sink used by default {@code print} and {@code printf} * operations. @@ -1719,7 +1733,7 @@ public int replaceAll(String orig, String repl, String ere) { private int replace(String orig, String repl, String ere, boolean global) { replaceResult.setLength(0); - String preparedReplacement = prepareReplacement(repl, false); + String preparedReplacement = prepareSubReplacement(repl, posix); Matcher matcher = dynamicPattern(ere).matcher(orig); int count = 0; while (matcher.find()) { @@ -1816,93 +1830,130 @@ public Enumeration splitTokenizer(String input, Object separator) { } /** - * Converts an AWK replacement text into a Java {@link Matcher} replacement: - * {@code &} becomes the whole match, {@code \&} a literal ampersand, - * {@code \\} a literal backslash, and {@code $} is escaped. + * Converts the replacement text of {@code sub()} or {@code gsub()} into a + * Java {@link Matcher} replacement. {@code &} becomes the whole match and + * {@code \&} a literal ampersand. The remaining backslashes follow gawk's + * documented rules by default: {@code \\\&} is a literal {@code \&}, + * {@code \\\\} a literal {@code \\}, {@code \\&} a literal backslash + * followed by the match, and any other backslash is kept as is. Under the + * POSIX rules, as in mawk and One True Awk, {@code \\} is a literal + * backslash and any other backslash is kept as is. * * @param awkRepl AWK replacement text - * @param backreferences whether {@code \N} denotes capture group {@code N} - * and any other {@code \c} a plain {@code c}, as in gawk's - * {@code gensub()}; when {@code false}, {@code \c} stays a literal - * {@code \c} as in {@code sub()} and {@code gsub()} + * @param posix whether to apply the POSIX rules instead of gawk's * @return the equivalent Java replacement string */ - public static String prepareReplacement(String awkRepl, boolean backreferences) { - return prepareReplacement(awkRepl, backreferences ? Integer.MAX_VALUE : -1); + public static String prepareSubReplacement(String awkRepl, boolean posix) { + if (awkRepl == null) { + return ""; + } + if (awkRepl.indexOf('\\') == -1 && awkRepl.indexOf('$') == -1 && awkRepl.indexOf('&') == -1) { + return awkRepl; + } + StringBuilder javaRepl = new StringBuilder(); + int length = awkRepl.length(); + for (int i = 0; i < length; i++) { + char c = awkRepl.charAt(i); + if (c != '\\') { + appendLiteral(javaRepl, c); + continue; + } + char next = i + 1 < length ? awkRepl.charAt(i + 1) : 0; + if (next == '&') { + // \& is a literal ampersand + javaRepl.append('&'); + i++; + } else if (posix) { + // \\ is a literal backslash; any other backslash is kept as + // is, and must be escaped for Matcher.appendReplacement + javaRepl.append("\\\\"); + if (next == '\\') { + i++; + } + } else if (awkRepl.startsWith("\\\\\\&", i)) { + // \\\& is a literal \& + javaRepl.append("\\\\&"); + i += 3; + } else if (awkRepl.startsWith("\\\\\\\\", i)) { + // \\\\ is a literal \\ + javaRepl.append("\\\\\\\\"); + i += 3; + } else if (awkRepl.startsWith("\\\\&", i)) { + // \\& is a literal backslash followed by the match + javaRepl.append("\\\\$0"); + i += 2; + } else { + // any other backslash is kept as is, and must be escaped for + // Matcher.appendReplacement + javaRepl.append("\\\\"); + } + } + return javaRepl.toString(); } /** - * Converts an AWK replacement text into a Java {@link Matcher} replacement, - * resolving gensub-style backreferences against a known number of capture - * groups: {@code \N} beyond {@code maxGroup} is replaced by the empty - * string, as gawk does, instead of producing a group reference that would - * make the matcher throw. + * Converts the replacement text of gawk's {@code gensub()} into a Java + * {@link Matcher} replacement: {@code &} and {@code \0} become the whole + * match, {@code \N} capture group {@code N}, {@code \&} a literal + * ampersand, {@code \\} a literal backslash, and any other {@code \c} a + * plain {@code c}, as in gawk. A {@code \N} beyond {@code maxGroup} + * expands to the empty string, as gawk does, instead of producing a group + * reference that would make the matcher throw. * * @param awkRepl AWK replacement text - * @param maxGroup highest valid capture group number, or a negative value - * to disable backreferences entirely ({@code sub()}/{@code gsub()} - * semantics) + * @param maxGroup highest valid capture group number * @return the equivalent Java replacement string */ - public static String prepareReplacement(String awkRepl, int maxGroup) { - boolean backreferences = maxGroup >= 0; + public static String prepareGensubReplacement(String awkRepl, int maxGroup) { if (awkRepl == null) { return ""; } - - if ((awkRepl.indexOf('\\') == -1) && (awkRepl.indexOf('$') == -1) && (awkRepl.indexOf('&') == -1)) { + if (awkRepl.indexOf('\\') == -1 && awkRepl.indexOf('$') == -1 && awkRepl.indexOf('&') == -1) { return awkRepl; } - StringBuilder javaRepl = new StringBuilder(); - for (int i = 0; i < awkRepl.length(); i++) { + int length = awkRepl.length(); + for (int i = 0; i < length; i++) { char c = awkRepl.charAt(i); - - if (c == '\\') { - if (i == awkRepl.length() - 1) { - // a trailing backslash is a literal backslash; left bare - // it would make Matcher.appendReplacement throw - javaRepl.append("\\\\"); - continue; - } - i++; - c = awkRepl.charAt(i); + if (c != '\\') { + appendLiteral(javaRepl, c); + } else if (i == length - 1) { + // a trailing backslash is a literal backslash; left bare it + // would make Matcher.appendReplacement throw + javaRepl.append("\\\\"); + } else { + c = awkRepl.charAt(++i); if (c == '&') { javaRepl.append('&'); - continue; } else if (c == '\\') { javaRepl.append("\\\\"); - continue; - } else if (backreferences) { - if (Character.isDigit(c)) { - if (c - '0' <= maxGroup) { - javaRepl.append('$').append(c); - } - // references beyond the pattern's groups expand to - // the empty string, as in gawk - continue; + } else if (Character.isDigit(c)) { + if (c - '0' <= maxGroup) { + javaRepl.append('$').append(c); } - // gensub(): any other \c is a plain c, as in gawk } else { - // sub()/gsub(): any other \c stays a literal \c, as in - // POSIX, gawk and mawk; a bare backslash would be eaten - // by Matcher.appendReplacement as an escape - javaRepl.append("\\\\"); + appendLiteral(javaRepl, c); } } - - if (c == '$') { - javaRepl.append("\\$"); - } else if (c == '&') { - javaRepl.append("$0"); - } else { - javaRepl.append(c); - } } - return javaRepl.toString(); } + /** + * Appends a character that is literal in AWK replacement text, escaping + * it for {@link Matcher#appendReplacement} and mapping {@code &} to the + * whole match. + */ + private static void appendLiteral(StringBuilder javaRepl, char c) { + if (c == '$') { + javaRepl.append("\\$"); + } else if (c == '&') { + javaRepl.append("$0"); + } else { + javaRepl.append(c); + } + } + /** * Returns the pattern itself, or its case-insensitive twin when * {@code IGNORECASE} is set. Twins are compiled once and cached here: diff --git a/src/main/java/io/jawk/util/AwkSettings.java b/src/main/java/io/jawk/util/AwkSettings.java index 03670a78..63c6321b 100644 --- a/src/main/java/io/jawk/util/AwkSettings.java +++ b/src/main/java/io/jawk/util/AwkSettings.java @@ -291,7 +291,9 @@ public void setUseSortedArrayKeys(boolean useSortedArrayKeys) { * such as arrays of arrays ({@code a[i][j]}, subarray operands like * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}), * and treating the gawk-specific {@code BEGINFILE} / {@code ENDFILE} - * patterns as ordinary identifiers. + * patterns as ordinary identifiers. At run time, the backslashes in + * {@code sub()} / {@code gsub()} replacement text then follow the POSIX + * rules instead of gawk's. * * @return {@code true} when POSIX compile-time behavior is enforced */ diff --git a/src/site/markdown/behavior-changes.md b/src/site/markdown/behavior-changes.md index 88842f83..3dd9ec97 100644 --- a/src/site/markdown/behavior-changes.md +++ b/src/site/markdown/behavior-changes.md @@ -20,6 +20,14 @@ released version automatically via .github/scripts/stamp-behavior-changes.sh. ## Unreleased +- Backslashes in the replacement text of `sub()` and `gsub()` now follow gawk's documented rules + instead of the POSIX ones: `\\` (the string written `"\\\\"` in the source) now produces `\\` + where it previously produced `\`, `\\\&` produces `\&`, and `\\\\` produces `\\`. `\&` and `\\&` + are unchanged. With `--posix`, the previous POSIX behavior is kept, as in mawk and One True Awk. + `gensub()` is unchanged ([#614](https://github.com/jawkio/jawk/issues/614)). +- For Java embedders, `JRT.prepareReplacement()` is replaced by `JRT.prepareSubReplacement()`, + which takes the POSIX switch, and `JRT.prepareGensubReplacement()`, which takes the highest + capture group number ([#614](https://github.com/jawkio/jawk/issues/614)). - In the replacement text of `sub()` and `gsub()`, a backslash followed by any character other than `&` or `\` is now kept literally, as in POSIX, gawk and mawk: `gsub(/'/, "'\\''")` on `'` now yields `'\''`. Previously the backslash was dropped (`'''`), and a `"\\$"` sequence or a diff --git a/src/site/markdown/cli-reference.md b/src/site/markdown/cli-reference.md index 31bb14eb..99819ad0 100644 --- a/src/site/markdown/cli-reference.md +++ b/src/site/markdown/cli-reference.md @@ -55,7 +55,7 @@ jawk --version > - `-F ` sets the initial field separator. > - `--locale ` sets the locale used to format numbers, through `Locale.forLanguageTag(...)`. > - `-t` keeps associative array keys sorted. -> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, all gawk `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns. +> - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, all gawk `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns, and makes `sub()` / `gsub()` follow the POSIX backslash rules in their replacement text (see [Where Jawk follows gawk](compatibility.html#where-jawk-follows-gawk)). > > - Extensions and sandbox > diff --git a/src/site/markdown/compatibility.md.vm b/src/site/markdown/compatibility.md.vm index 176f4949..2714b97a 100644 --- a/src/site/markdown/compatibility.md.vm +++ b/src/site/markdown/compatibility.md.vm @@ -375,6 +375,7 @@ Jawk follows gawk in several places where gawk departs from historical AWK: - A regexp literal used as an ordinary expression evaluates as `$0 ~ /re/`, per POSIX; `x = /re/` assigns 0 or 1, not the pattern text. - Typed regexp literals (`@/re/`) are accepted; `--posix` rejects them. +- Backslashes in the replacement text of `sub()` and `gsub()` follow gawk's rules: `\&` is a literal `&`, `\\&` a literal backslash followed by the matched text, `\\\&` a literal `\&`, `\\\\` a literal `\\`, and any other backslash is kept as is, so the replacement `\\` (written `"\\\\"` in the source) produces `\\` and `\q` produces `\q`. With `--posix`, `\\` is a single literal backslash, as in mawk and One True Awk, and everything else is kept as is. `gensub()` always uses its own gawk rules: `\N` is capture group N, `\\` a backslash, `\&` an ampersand, and any other `\c` a plain `c`. - `IGNORECASE` is a built-in special variable managed by the runtime, like `FS` or `RS`: a truthy value makes regexp matching, field and `split()` separators, string comparisons, `index()`, and gawk array sorting case-insensitive. - Because the [gawk compatibility extension](extensions.html#gawk) is enabled by default, `gensub`, `typeof`, `isarray`, `asort`, `asorti`, `mkbool`, `patsplit`, `strtonum`, `systime`, `mktime`, `strftime`, `bindtextdomain`, `dcgettext`, and `dcngettext` are function names and can no longer be used as variables. Load an explicit extension list without `GawkExtension` to reclaim those identifiers. - Calling a user-defined function with more arguments than it declares is accepted: the extra expressions are evaluated and discarded, and a gawk-style warning is printed to stderr. diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 551046d0..883c7994 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -821,6 +821,22 @@ public void testSubReplaceWithMatchReference() throws Exception { .script("BEGIN { s = \"x x x\"; gsub(/x/, \"\\\\$\\\\1\\\\\", s); print s }") .expect("\\$\\1\\ \\$\\1\\ \\$\\1\\\n") .runAndAssert(); + + // issue #614: the replacement text \\\&|\\\\|\\&|\\q|\\ follows gawk's + // rules by default and the POSIX rules (as in mawk) with --posix + String backslashes = "BEGIN { s = \"x\"; gsub(/x/, \"\\\\\\\\\\\\&|\\\\\\\\\\\\\\\\|\\\\\\\\&|\\\\\\\\q|\\\\\\\\\", s); print s }"; + AwkTestSupport + .awkTest("gsub follows gawk's backslash rules") + .script(backslashes) + .expect("\\&|\\\\|\\x|\\\\q|\\\\\n") + .runAndAssert(); + + AwkTestSupport + .cliTest("gsub follows the POSIX backslash rules with --posix") + .argument("--posix") + .script(backslashes) + .expect("\\&|\\\\|\\x|\\q|\\\n") + .runAndAssert(); } @Test diff --git a/src/test/java/io/jawk/jrt/PrepareReplacementTest.java b/src/test/java/io/jawk/jrt/PrepareReplacementTest.java index 8342d302..ef22ae70 100644 --- a/src/test/java/io/jawk/jrt/PrepareReplacementTest.java +++ b/src/test/java/io/jawk/jrt/PrepareReplacementTest.java @@ -27,39 +27,77 @@ import org.junit.Test; /** - * Tests for {@link JRT#prepareReplacement(String, boolean)}. + * Tests for {@link JRT#prepareSubReplacement(String, boolean)} and + * {@link JRT#prepareGensubReplacement(String, int)}. */ public class PrepareReplacementTest { @Test - public void testPrepareReplacement() { - assertEquals("don't change", JRT.prepareReplacement("don't change", false)); - assertEquals("a$0a", JRT.prepareReplacement("a&a", false)); - assertEquals("1$01", JRT.prepareReplacement("1&1", false)); - assertEquals("a$0b$0c", JRT.prepareReplacement("a&b&c", false)); - // sub()/gsub(): \c stays a literal \c, including \digit and a trailing \ - assertEquals("a\\\\b", JRT.prepareReplacement("a\\b", false)); - assertEquals("a\\\\1", JRT.prepareReplacement("a\\1", false)); - assertEquals("a&b", JRT.prepareReplacement("a\\&b", false)); - assertEquals("a\\\\", JRT.prepareReplacement("a\\", false)); - assertEquals("a\\$", JRT.prepareReplacement("a$", false)); - assertEquals("a\\\\\\$", JRT.prepareReplacement("a\\$", false)); - assertEquals("a\\\\\\$", JRT.prepareReplacement("a\\\\$", false)); - assertEquals("a\\\\$0", JRT.prepareReplacement("a\\\\&", false)); - assertEquals("a\\\\&", JRT.prepareReplacement("a\\\\\\&", false)); - assertEquals("", JRT.prepareReplacement("", false)); - assertEquals("", JRT.prepareReplacement(null, false)); + public void testPrepareSubReplacementGawkRules() { + assertEquals("don't change", JRT.prepareSubReplacement("don't change", false)); + assertEquals("a$0a", JRT.prepareSubReplacement("a&a", false)); + assertEquals("1$01", JRT.prepareSubReplacement("1&1", false)); + assertEquals("a$0b$0c", JRT.prepareSubReplacement("a&b&c", false)); + assertEquals("a\\$", JRT.prepareSubReplacement("a$", false)); + // \& is a literal & + assertEquals("a&b", JRT.prepareSubReplacement("a\\&b", false)); + // \\& is a literal \ followed by the match + assertEquals("a\\\\$0", JRT.prepareSubReplacement("a\\\\&", false)); + // \\\& is a literal \& + assertEquals("a\\\\&", JRT.prepareSubReplacement("a\\\\\\&", false)); + // \\\\ is a literal \\, also when followed by & + assertEquals("a\\\\\\\\", JRT.prepareSubReplacement("a\\\\\\\\", false)); + assertEquals("a\\\\\\\\$0", JRT.prepareSubReplacement("a\\\\\\\\&", false)); + assertEquals("a\\\\\\\\&", JRT.prepareSubReplacement("a\\\\\\\\\\&", false)); + // any other backslash is kept as is: \\ stays \\, \c stays \c + assertEquals("a\\\\\\\\", JRT.prepareSubReplacement("a\\\\", false)); + assertEquals("a\\\\b", JRT.prepareSubReplacement("a\\b", false)); + assertEquals("a\\\\\\\\q", JRT.prepareSubReplacement("a\\\\q", false)); + assertEquals("a\\\\1", JRT.prepareSubReplacement("a\\1", false)); + assertEquals("a\\\\", JRT.prepareSubReplacement("a\\", false)); + assertEquals("a\\\\\\$", JRT.prepareSubReplacement("a\\$", false)); + assertEquals("a\\\\\\\\\\$", JRT.prepareSubReplacement("a\\\\$", false)); + assertEquals("", JRT.prepareSubReplacement("", false)); + assertEquals("", JRT.prepareSubReplacement(null, false)); } @Test - public void testPrepareReplacementWithBackreferences() { - // gensub mode: \N becomes a group reference, trailing backslash is literal - assertEquals("a$1b", JRT.prepareReplacement("a\\1b", true)); - assertEquals("a\\\\", JRT.prepareReplacement("a\\", true)); - assertEquals("a$0a", JRT.prepareReplacement("a&a", true)); - assertEquals("a&b", JRT.prepareReplacement("a\\&b", true)); - // gensub mode: any other \c is a plain c - assertEquals("aq", JRT.prepareReplacement("a\\q", true)); - assertEquals("a\\$", JRT.prepareReplacement("a\\$", true)); + public void testPrepareSubReplacementPosixRules() { + assertEquals("a$0a", JRT.prepareSubReplacement("a&a", true)); + assertEquals("a\\$", JRT.prepareSubReplacement("a$", true)); + // \& is a literal &, \\ a literal \ + assertEquals("a&b", JRT.prepareSubReplacement("a\\&b", true)); + assertEquals("a\\\\", JRT.prepareSubReplacement("a\\\\", true)); + assertEquals("a\\\\$0", JRT.prepareSubReplacement("a\\\\&", true)); + assertEquals("a\\\\&", JRT.prepareSubReplacement("a\\\\\\&", true)); + assertEquals("a\\\\\\\\", JRT.prepareSubReplacement("a\\\\\\\\", true)); + assertEquals("a\\\\q", JRT.prepareSubReplacement("a\\\\q", true)); + // any other backslash is kept as is + assertEquals("a\\\\b", JRT.prepareSubReplacement("a\\b", true)); + assertEquals("a\\\\1", JRT.prepareSubReplacement("a\\1", true)); + assertEquals("a\\\\", JRT.prepareSubReplacement("a\\", true)); + assertEquals("a\\\\\\$", JRT.prepareSubReplacement("a\\$", true)); + assertEquals("a\\\\\\$", JRT.prepareSubReplacement("a\\\\$", true)); + assertEquals("", JRT.prepareSubReplacement(null, true)); + } + + @Test + public void testPrepareGensubReplacement() { + assertEquals("don't change", JRT.prepareGensubReplacement("don't change", 0)); + assertEquals("a$0a", JRT.prepareGensubReplacement("a&a", 0)); + assertEquals("a\\$", JRT.prepareGensubReplacement("a$", 0)); + // \N is a group reference, the empty string beyond the pattern's groups + assertEquals("a$1b", JRT.prepareGensubReplacement("a\\1b", 1)); + assertEquals("a$0b", JRT.prepareGensubReplacement("a\\0b", 0)); + assertEquals("ab", JRT.prepareGensubReplacement("a\\2b", 1)); + // \& is a literal &, \\ a literal \, a trailing \ a literal \ + assertEquals("a&b", JRT.prepareGensubReplacement("a\\&b", 0)); + assertEquals("a\\\\b", JRT.prepareGensubReplacement("a\\\\b", 0)); + assertEquals("a\\\\", JRT.prepareGensubReplacement("a\\", 0)); + // any other \c is a plain c + assertEquals("aq", JRT.prepareGensubReplacement("a\\q", 0)); + assertEquals("a\\$", JRT.prepareGensubReplacement("a\\$", 0)); + assertEquals("", JRT.prepareGensubReplacement("", 0)); + assertEquals("", JRT.prepareGensubReplacement(null, 0)); } } From 42293827137d52ab740e4ce61466e527e6fbbbc5 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 8 Oct 2026 14:58:26 +0200 Subject: [PATCH 2/3] Record the POSIX mode in the compiled program The runtime took the sub()/gsub() backslash rules from the settings of the AVM that runs the program. A program compiled with --posix -K and loaded with -L, which cannot be combined with --posix, would therefore silently switch to gawk's rules. AwkProgram now records whether it was compiled in POSIX mode, and AVM takes the rules from the program when it executes it. Co-Authored-By: Claude Fable 5.1 --- src/main/java/io/jawk/Awk.java | 1 + src/main/java/io/jawk/AwkProgram.java | 21 ++++++++++++++++++- src/main/java/io/jawk/backend/AVM.java | 3 ++- src/site/markdown/behavior-changes.md | 6 ++++-- src/site/markdown/cli-reference.md | 2 +- src/test/java/io/jawk/AwkTest.java | 29 ++++++++++++++++++++++++++ 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/jawk/Awk.java b/src/main/java/io/jawk/Awk.java index edc8912e..ea9df6d2 100644 --- a/src/main/java/io/jawk/Awk.java +++ b/src/main/java/io/jawk/Awk.java @@ -556,6 +556,7 @@ protected final T compileProgram( parser.populateGlobalVariableNameToOffsetMappings(tuples); } } + tuples.setPosix(settings.isPosix()); tuples.freezeMetadata(); return tuples; diff --git a/src/main/java/io/jawk/AwkProgram.java b/src/main/java/io/jawk/AwkProgram.java index c75a0f94..f3ea6dac 100644 --- a/src/main/java/io/jawk/AwkProgram.java +++ b/src/main/java/io/jawk/AwkProgram.java @@ -31,9 +31,28 @@ */ public class AwkProgram extends io.jawk.intermediate.AwkTuples { - private static final long serialVersionUID = 1L; + private static final long serialVersionUID = 2L; + + /** Whether the program was compiled in POSIX mode. */ + private boolean posix; AwkProgram() { super(); } + + /** + * Returns whether the program was compiled in POSIX mode, in which case + * {@code sub()} and {@code gsub()} follow the POSIX backslash rules in + * their replacement text at run time, even when the program is loaded + * from a file. + * + * @return {@code true} when the program was compiled in POSIX mode + */ + public boolean isPosix() { + return posix; + } + + void setPosix(boolean posix) { + this.posix = posix; + } } diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index fa549a9c..b47cafa1 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -246,7 +246,6 @@ public AVM( executionSpecialVariables = baseSpecialVariables; jrt = createJrt(); - jrt.setPosix(this.settings.isPosix()); initExtensions(); } @@ -456,6 +455,7 @@ public void execute( resetRuntimeState(runtimeArguments, variableOverrides); installProgramMetadata(compiledProgram); + jrt.setPosix(compiledProgram.isPosix()); jrt.prepareForExecution(settings.getFieldSeparator(), settings.getDefaultRS()); if (!executionSpecialVariables.isEmpty()) { jrt.applySpecialVariables(executionSpecialVariables); @@ -533,6 +533,7 @@ public void executePersistingGlobals( InputSource resolvedSource = Objects.requireNonNull(inputSource, "inputSource"); mergeRuntimeState(runtimeArguments, variableOverrides, compiledProgram); + jrt.setPosix(compiledProgram.isPosix()); jrt.prepareForExecution(settings.getFieldSeparator(), settings.getDefaultRS()); if (!executionSpecialVariables.isEmpty()) { jrt.applySpecialVariables(executionSpecialVariables); diff --git a/src/site/markdown/behavior-changes.md b/src/site/markdown/behavior-changes.md index 3dd9ec97..37e99f7d 100644 --- a/src/site/markdown/behavior-changes.md +++ b/src/site/markdown/behavior-changes.md @@ -23,8 +23,10 @@ released version automatically via .github/scripts/stamp-behavior-changes.sh. - Backslashes in the replacement text of `sub()` and `gsub()` now follow gawk's documented rules instead of the POSIX ones: `\\` (the string written `"\\\\"` in the source) now produces `\\` where it previously produced `\`, `\\\&` produces `\&`, and `\\\\` produces `\\`. `\&` and `\\&` - are unchanged. With `--posix`, the previous POSIX behavior is kept, as in mawk and One True Awk. - `gensub()` is unchanged ([#614](https://github.com/jawkio/jawk/issues/614)). + are unchanged. With `--posix`, the previous POSIX behavior is kept, as in mawk and One True Awk, + and a program compiled with `--posix -K` keeps it when loaded with `-L` (the compiled program now + records its mode, so programs compiled by earlier versions must be recompiled). `gensub()` is + unchanged ([#614](https://github.com/jawkio/jawk/issues/614)). - For Java embedders, `JRT.prepareReplacement()` is replaced by `JRT.prepareSubReplacement()`, which takes the POSIX switch, and `JRT.prepareGensubReplacement()`, which takes the highest capture group number ([#614](https://github.com/jawkio/jawk/issues/614)). diff --git a/src/site/markdown/cli-reference.md b/src/site/markdown/cli-reference.md index 99819ad0..4cd280b3 100644 --- a/src/site/markdown/cli-reference.md +++ b/src/site/markdown/cli-reference.md @@ -82,7 +82,7 @@ jawk --version ## Execution Notes - `--dump-syntax`, `--dump-intermediate`, `-K`, `-h`, `-?`, `-V`, `--version`, and `--list-ext` do not execute the script, and ignore an ambient `JAWK_PERSISTENT_MEMORY`. `--dump-syntax`, `--dump-intermediate`, and `-K` also ignore `--persist`, while the standalone modes (`-h`, `-?`, `-V`, `--version`, `--list-ext`) reject it like any other extra argument. `--profile` does execute the script, and keeps the normal AWK output on stdout. -- `-f` compiles source now, `-L` loads a program compiled earlier. `--posix` is rejected together with `-L`: loading a precompiled program bypasses source compilation, so there is no compile-time behavior left to restrict. +- `-f` compiles source now, `-L` loads a program compiled earlier. `--posix` is rejected together with `-L`: loading a precompiled program bypasses source compilation, so there is no compile-time behavior left to restrict. A program compiled with `--posix -K` records that mode and keeps the POSIX `sub()` / `gsub()` replacement rules when loaded with `-L`. - `-S` applies at compile time as well as at run time — a sandboxed run rejects the forbidden constructs while compiling the script. ## See Also diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 883c7994..2467ddd5 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1643,6 +1643,35 @@ public void compileTuplesViaCLI() throws Exception { assertEquals("ABC\n", result); } + /** + * A program compiled with --posix -K keeps the POSIX + * sub()/gsub() replacement rules when loaded with -L, which + * cannot be combined with --posix (issue #614). + */ + @Test + public void loadedProgramKeepsPosixMode() throws Exception { + File tmp = File.createTempFile("jawk", ".tpl"); + // the replacement \\ is a single backslash in POSIX mode, \\ in gawk's + String script = "BEGIN { s = \"x\"; gsub(/x/, \"\\\\\\\\\", s); print s }"; + Cli + .create( + new String[] + { "--posix", "-K", tmp.getAbsolutePath(), script }, + new ByteArrayInputStream(new byte[0]), + System.out, + System.err); + + Cli cli = Cli.parseCommandLineArguments(new String[] { "-L", tmp.getAbsolutePath() }); + assertTrue(cli.getPrecompiledProgram().isPosix()); + + String result = new Awk() + .script(cli.getPrecompiledProgram()) + .execute(); + + assertEquals("\\\n", result); + assertEquals("\\\\\n", new Awk().script(script).execute()); + } + /** * Ensures that the CLI rejects variable assignments with invalid identifiers. */ From 6796339fe8e37de632b450c80375e6496fa83784 Mon Sep 17 00:00:00 2001 From: Bertrand Martin Date: Thu, 8 Oct 2026 15:07:48 +0200 Subject: [PATCH 3/3] Carry the POSIX mode on compiled expressions too Expressions compiled with POSIX settings and evaluated through Awk.eval() never reached the execute() paths that set the runtime mode, so sub()/gsub() in them used gawk's rules. The mode now lives on AwkTuples, the base of both AwkProgram and AwkExpression, is recorded by both compile paths, and is applied by AVM.eval() as well. The loaded-program test now runs through AwkTestSupport. Co-Authored-By: Claude Fable 5.1 --- src/main/java/io/jawk/Awk.java | 1 + src/main/java/io/jawk/AwkProgram.java | 21 +------------ src/main/java/io/jawk/backend/AVM.java | 1 + .../java/io/jawk/intermediate/AwkTuples.java | 26 +++++++++++++++- src/test/java/io/jawk/AwkEvalTest.java | 11 +++++++ src/test/java/io/jawk/AwkTest.java | 31 ++++++++++--------- 6 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/main/java/io/jawk/Awk.java b/src/main/java/io/jawk/Awk.java index ea9df6d2..6982e0ca 100644 --- a/src/main/java/io/jawk/Awk.java +++ b/src/main/java/io/jawk/Awk.java @@ -636,6 +636,7 @@ protected final T compileExpression( // so that the interpreter can assign variables parser.populateGlobalVariableNameToOffsetMappings(tuples); } + tuples.setPosix(settings.isPosix()); tuples.freezeMetadata(); return tuples; diff --git a/src/main/java/io/jawk/AwkProgram.java b/src/main/java/io/jawk/AwkProgram.java index f3ea6dac..c75a0f94 100644 --- a/src/main/java/io/jawk/AwkProgram.java +++ b/src/main/java/io/jawk/AwkProgram.java @@ -31,28 +31,9 @@ */ public class AwkProgram extends io.jawk.intermediate.AwkTuples { - private static final long serialVersionUID = 2L; - - /** Whether the program was compiled in POSIX mode. */ - private boolean posix; + private static final long serialVersionUID = 1L; AwkProgram() { super(); } - - /** - * Returns whether the program was compiled in POSIX mode, in which case - * {@code sub()} and {@code gsub()} follow the POSIX backslash rules in - * their replacement text at run time, even when the program is loaded - * from a file. - * - * @return {@code true} when the program was compiled in POSIX mode - */ - public boolean isPosix() { - return posix; - } - - void setPosix(boolean posix) { - this.posix = posix; - } } diff --git a/src/main/java/io/jawk/backend/AVM.java b/src/main/java/io/jawk/backend/AVM.java index b47cafa1..f6273e67 100644 --- a/src/main/java/io/jawk/backend/AVM.java +++ b/src/main/java/io/jawk/backend/AVM.java @@ -359,6 +359,7 @@ protected Locale getLocale() { public Object eval(AwkExpression expression) throws IOException { AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression"); installExpressionMetadata(compiledExpression); + jrt.setPosix(compiledExpression.isPosix()); try { executeTuples(compiledExpression.top()); diff --git a/src/main/java/io/jawk/intermediate/AwkTuples.java b/src/main/java/io/jawk/intermediate/AwkTuples.java index 20ddcfc2..5f205576 100644 --- a/src/main/java/io/jawk/intermediate/AwkTuples.java +++ b/src/main/java/io/jawk/intermediate/AwkTuples.java @@ -53,7 +53,7 @@ public class AwkTuples implements Serializable { // Bumped to 6 when the getline opcodes gained a no-record jump address: // older tuple streams emit them without one and must be recompiled. // (5 = exact 64-bit integral arithmetic.) - private static final long serialVersionUID = 7L; + private static final long serialVersionUID = 8L; /** Address manager */ private final AddressManager addressManager = new AddressManager(); @@ -61,6 +61,9 @@ public class AwkTuples implements Serializable { /** Description of the primary script source, used for runtime diagnostics. */ private String sourceDescription; + /** Whether the tuples were compiled in POSIX mode. */ + private boolean posix; + /** * Creates an empty tuple list, ready for the front end to append to. */ @@ -87,6 +90,27 @@ public String getSourceDescription() { return sourceDescription; } + /** + * Records whether the tuples were compiled in POSIX mode, so that the + * runtime applies the matching semantics even when they are loaded from a + * file: {@code sub()} and {@code gsub()} then follow the POSIX backslash + * rules in their replacement text. + * + * @param posixParam {@code true} when compiled in POSIX mode + */ + public void setPosix(boolean posixParam) { + this.posix = posixParam; + } + + /** + * Returns whether the tuples were compiled in POSIX mode. + * + * @return {@code true} when compiled in POSIX mode + */ + public boolean isPosix() { + return posix; + } + // made public to access static members of AwkTuples via Java Reflection // made public to be accessable via Java Reflection diff --git a/src/test/java/io/jawk/AwkEvalTest.java b/src/test/java/io/jawk/AwkEvalTest.java index 77e7ad71..d4877a67 100644 --- a/src/test/java/io/jawk/AwkEvalTest.java +++ b/src/test/java/io/jawk/AwkEvalTest.java @@ -255,6 +255,17 @@ public void testEvalHonorsSettingsChangesBetweenCalls() throws Exception { assertEquals("right", awk.eval(expression, "left:right")); } + @Test + public void testEvalAppliesPosixModeOfTheExpression() throws Exception { + // the replacement \\ is a single backslash in POSIX mode, \\ in gawk's + String expression = "gsub(/x/, \"\\\\\\\\\") ? $0 : $0"; + assertEquals("\\\\", new Awk().eval(expression, "x")); + + AwkSettings settings = new AwkSettings(); + settings.setPosix(true); + assertEquals("\\", new Awk(settings).eval(expression, "x")); + } + @Test public void testEvalStringInputRepresentsSingleRecord() throws Exception { Awk awk = new Awk(); diff --git a/src/test/java/io/jawk/AwkTest.java b/src/test/java/io/jawk/AwkTest.java index 2467ddd5..19f15285 100644 --- a/src/test/java/io/jawk/AwkTest.java +++ b/src/test/java/io/jawk/AwkTest.java @@ -1653,23 +1653,24 @@ public void loadedProgramKeepsPosixMode() throws Exception { File tmp = File.createTempFile("jawk", ".tpl"); // the replacement \\ is a single backslash in POSIX mode, \\ in gawk's String script = "BEGIN { s = \"x\"; gsub(/x/, \"\\\\\\\\\", s); print s }"; - Cli - .create( - new String[] - { "--posix", "-K", tmp.getAbsolutePath(), script }, - new ByteArrayInputStream(new byte[0]), - System.out, - System.err); - - Cli cli = Cli.parseCommandLineArguments(new String[] { "-L", tmp.getAbsolutePath() }); - assertTrue(cli.getPrecompiledProgram().isPosix()); + AwkTestSupport + .cliTest("gsub follows gawk's backslash rules by default") + .script(script) + .expect("\\\\\n") + .runAndAssert(); - String result = new Awk() - .script(cli.getPrecompiledProgram()) - .execute(); + AwkTestSupport + .cliTest("--posix -K compiles the program to a file") + .argument("--posix", "-K", tmp.getAbsolutePath()) + .script(script) + .expect("") + .runAndAssert(); - assertEquals("\\\n", result); - assertEquals("\\\\\n", new Awk().script(script).execute()); + AwkTestSupport + .cliTest("-L keeps the POSIX backslash rules of the compiled program") + .argument("-L", tmp.getAbsolutePath()) + .expect("\\\n") + .runAndAssert(); } /**