Follow gawk's backslash rules in sub()/gsub() replacement text

src/main/java/io/jawk/Awk.java

@@ -556,6 +556,7 @@ protected final <T extends AwkProgram> T compileProgram(
 				parser.populateGlobalVariableNameToOffsetMappings(tuples);
 			}
 		}
+		tuples.setPosix(settings.isPosix());
 		tuples.freezeMetadata();
 
 		return tuples;
@@ -635,6 +636,7 @@ protected final <T extends AwkExpression> T compileExpression(
 			// so that the interpreter can assign variables
 			parser.populateGlobalVariableNameToOffsetMappings(tuples);
 		}
+		tuples.setPosix(settings.isPosix());
 		tuples.freezeMetadata();
 
 		return tuples;

src/main/java/io/jawk/backend/AVM.java

@@ -359,6 +359,7 @@ protected Locale getLocale() {
 	public Object eval(AwkExpression expression) throws IOException {
 		AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
 		installExpressionMetadata(compiledExpression);
+		jrt.setPosix(compiledExpression.isPosix());
 
 		try {
 			executeTuples(compiledExpression.top());
@@ -455,6 +456,7 @@ public void execute(
 		resetRuntimeState(runtimeArguments, variableOverrides);
 		installProgramMetadata(compiledProgram);
 
+		jrt.setPosix(compiledProgram.isPosix());
 		jrt.prepareForExecution(settings.getFieldSeparator(), settings.getDefaultRS());
 		if (!executionSpecialVariables.isEmpty()) {
 			jrt.applySpecialVariables(executionSpecialVariables);
@@ -532,6 +534,7 @@ public void executePersistingGlobals(
 		InputSource resolvedSource = Objects.requireNonNull(inputSource, "inputSource");
 		mergeRuntimeState(runtimeArguments, variableOverrides, compiledProgram);
 
+		jrt.setPosix(compiledProgram.isPosix());
 		jrt.prepareForExecution(settings.getFieldSeparator(), settings.getDefaultRS());
 		if (!executionSpecialVariables.isEmpty()) {
 			jrt.applySpecialVariables(executionSpecialVariables);

src/main/java/io/jawk/ext/GawkExtension.java

@@ -238,7 +238,7 @@ public String gensub(@JawkRegexp Object regexp, Object replacement, Object how,
 		pattern = getJrt().caseAwarePattern(pattern);
 		Object targetValue = target == null ? getJrt().getInputLine() : target;
 		Matcher matcher = pattern.matcher(toAwkString(targetValue));
-		String repl = JRT.prepareReplacement(toAwkString(replacement), pattern.matcher("").groupCount());
+		String repl = JRT.prepareGensubReplacement(toAwkString(replacement), pattern.matcher("").groupCount());
 		String selector = toAwkString(how);
 		// gawk: any string beginning with 'g' or 'G' selects a global replacement
 		if (!selector.isEmpty() && (selector.charAt(0) == 'g' || selector.charAt(0) == 'G')) {

src/main/java/io/jawk/intermediate/AwkTuples.java

@@ -53,14 +53,17 @@ public class AwkTuples implements Serializable {
 	// Bumped to 6 when the getline opcodes gained a no-record jump address:
 	// older tuple streams emit them without one and must be recompiled.
 	// (5 = exact 64-bit integral arithmetic.)
-	private static final long serialVersionUID = 7L;
+	private static final long serialVersionUID = 8L;
 
 	/** Address manager */
 	private final AddressManager addressManager = new AddressManager();
 
 	/** Description of the primary script source, used for runtime diagnostics. */
 	private String sourceDescription;
 
+	/** Whether the tuples were compiled in POSIX mode. */
+	private boolean posix;
+
 	/**
 	 * Creates an empty tuple list, ready for the front end to append to.
 	 */
@@ -87,6 +90,27 @@ public String getSourceDescription() {
 		return sourceDescription;
 	}
 
+	/**
+	 * Records whether the tuples were compiled in POSIX mode, so that the
+	 * runtime applies the matching semantics even when they are loaded from a
+	 * file: {@code sub()} and {@code gsub()} then follow the POSIX backslash
+	 * rules in their replacement text.
+	 *
+	 * @param posixParam {@code true} when compiled in POSIX mode
+	 */
+	public void setPosix(boolean posixParam) {
+		this.posix = posixParam;
+	}
+
+	/**
+	 * Returns whether the tuples were compiled in POSIX mode.
+	 *
+	 * @return {@code true} when compiled in POSIX mode
+	 */
+	public boolean isPosix() {
+		return posix;
+	}
+
 	// made public to access static members of AwkTuples via Java Reflection
 
 	// made public to be accessable via Java Reflection

src/main/java/io/jawk/jrt/JRT.java

@@ -143,6 +143,9 @@ public class JRT {
 	private Object ignorecase = Long.valueOf(0L);
 	/** Precomputed truth of IGNORECASE, consulted by every regexp operation. */
 	private boolean ignoreCase;
+
+	/** Whether sub()/gsub() replacement text follows the POSIX backslash rules instead of gawk's. */
+	private boolean posix;
 	/** Case-insensitive twins of precompiled patterns; created on first use. */
 	private Map<Pattern, Pattern> caseInsensitivePatterns;
 	/** Compiled case-sensitive dynamic (string) regexps, keyed by expression text; created on first use. */
@@ -299,6 +302,17 @@ public JRT(VariableManager vm, Locale locale, AwkSink awkSink, PrintStream error
 		this.subsep = Awk.DEFAULT_SUBSEP;
 	}
 
+	/**
+	 * Selects the POSIX backslash rules for {@code sub()} and {@code gsub()}
+	 * replacement text instead of gawk's; see
+	 * {@link #prepareSubReplacement(String, boolean)}.
+	 *
+	 * @param posix {@code true} for the POSIX rules
+	 */
+	public void setPosix(boolean posix) {
+		this.posix = posix;
+	}
+
 	/**
 	 * Sets the sink used by default {@code print} and {@code printf}
 	 * operations.
@@ -1719,7 +1733,7 @@ public int replaceAll(String orig, String repl, String ere) {
 
 	private int replace(String orig, String repl, String ere, boolean global) {
 		replaceResult.setLength(0);
-		String preparedReplacement = prepareReplacement(repl, false);
+		String preparedReplacement = prepareSubReplacement(repl, posix);
 		Matcher matcher = dynamicPattern(ere).matcher(orig);
 		int count = 0;
 		while (matcher.find()) {
@@ -1816,93 +1830,130 @@ public Enumeration<Object> splitTokenizer(String input, Object separator) {
 	}
 
 	/**
-	 * Converts an AWK replacement text into a Java {@link Matcher} replacement:
-	 * {@code &} becomes the whole match, {@code \&} a literal ampersand,
-	 * {@code \\} a literal backslash, and {@code $} is escaped.
+	 * Converts the replacement text of {@code sub()} or {@code gsub()} into a
+	 * Java {@link Matcher} replacement. {@code &} becomes the whole match and
+	 * {@code \&} a literal ampersand. The remaining backslashes follow gawk's
+	 * documented rules by default: {@code \\\&} is a literal {@code \&},
+	 * {@code \\\\} a literal {@code \\}, {@code \\&} a literal backslash
+	 * followed by the match, and any other backslash is kept as is. Under the
+	 * POSIX rules, as in mawk and One True Awk, {@code \\} is a literal
+	 * backslash and any other backslash is kept as is.
 	 *
 	 * @param awkRepl AWK replacement text
-	 * @param backreferences whether {@code \N} denotes capture group {@code N}
-	 *        and any other {@code \c} a plain {@code c}, as in gawk's
-	 *        {@code gensub()}; when {@code false}, {@code \c} stays a literal
-	 *        {@code \c} as in {@code sub()} and {@code gsub()}
+	 * @param posix whether to apply the POSIX rules instead of gawk's
 	 * @return the equivalent Java replacement string
 	 */
-	public static String prepareReplacement(String awkRepl, boolean backreferences) {
-		return prepareReplacement(awkRepl, backreferences ? Integer.MAX_VALUE : -1);
+	public static String prepareSubReplacement(String awkRepl, boolean posix) {
+		if (awkRepl == null) {
+			return "";
+		}
+		if (awkRepl.indexOf('\\') == -1 && awkRepl.indexOf('$') == -1 && awkRepl.indexOf('&') == -1) {
+			return awkRepl;
+		}
+		StringBuilder javaRepl = new StringBuilder();
+		int length = awkRepl.length();
+		for (int i = 0; i < length; i++) {
+			char c = awkRepl.charAt(i);
+			if (c != '\\') {
+				appendLiteral(javaRepl, c);
+				continue;
+			}
+			char next = i + 1 < length ? awkRepl.charAt(i + 1) : 0;
+			if (next == '&') {
+				// \& is a literal ampersand
+				javaRepl.append('&');
+				i++;
+			} else if (posix) {
+				// \\ is a literal backslash; any other backslash is kept as
+				// is, and must be escaped for Matcher.appendReplacement
+				javaRepl.append("\\\\");
+				if (next == '\\') {
+					i++;
+				}
+			} else if (awkRepl.startsWith("\\\\\\&", i)) {
+				// \\\& is a literal \&
+				javaRepl.append("\\\\&");
+				i += 3;
+			} else if (awkRepl.startsWith("\\\\\\\\", i)) {
+				// \\\\ is a literal \\
+				javaRepl.append("\\\\\\\\");
+				i += 3;
+			} else if (awkRepl.startsWith("\\\\&", i)) {
+				// \\& is a literal backslash followed by the match
+				javaRepl.append("\\\\$0");
+				i += 2;
+			} else {
+				// any other backslash is kept as is, and must be escaped for
+				// Matcher.appendReplacement
+				javaRepl.append("\\\\");
+			}
+		}
+		return javaRepl.toString();
 	}
 
 	/**
-	 * Converts an AWK replacement text into a Java {@link Matcher} replacement,
-	 * resolving gensub-style backreferences against a known number of capture
-	 * groups: {@code \N} beyond {@code maxGroup} is replaced by the empty
-	 * string, as gawk does, instead of producing a group reference that would
-	 * make the matcher throw.
+	 * Converts the replacement text of gawk's {@code gensub()} into a Java
+	 * {@link Matcher} replacement: {@code &} and {@code \0} become the whole
+	 * match, {@code \N} capture group {@code N}, {@code \&} a literal
+	 * ampersand, {@code \\} a literal backslash, and any other {@code \c} a
+	 * plain {@code c}, as in gawk. A {@code \N} beyond {@code maxGroup}
+	 * expands to the empty string, as gawk does, instead of producing a group
+	 * reference that would make the matcher throw.
 	 *
 	 * @param awkRepl AWK replacement text
-	 * @param maxGroup highest valid capture group number, or a negative value
-	 *        to disable backreferences entirely ({@code sub()}/{@code gsub()}
-	 *        semantics)
+	 * @param maxGroup highest valid capture group number
 	 * @return the equivalent Java replacement string
 	 */
-	public static String prepareReplacement(String awkRepl, int maxGroup) {
-		boolean backreferences = maxGroup >= 0;
+	public static String prepareGensubReplacement(String awkRepl, int maxGroup) {
 		if (awkRepl == null) {
 			return "";
 		}
-
-		if ((awkRepl.indexOf('\\') == -1) && (awkRepl.indexOf('$') == -1) && (awkRepl.indexOf('&') == -1)) {
+		if (awkRepl.indexOf('\\') == -1 && awkRepl.indexOf('$') == -1 && awkRepl.indexOf('&') == -1) {
 			return awkRepl;
 		}
-
 		StringBuilder javaRepl = new StringBuilder();
-		for (int i = 0; i < awkRepl.length(); i++) {
+		int length = awkRepl.length();
+		for (int i = 0; i < length; i++) {
 			char c = awkRepl.charAt(i);
-
-			if (c == '\\') {
-				if (i == awkRepl.length() - 1) {
-					// a trailing backslash is a literal backslash; left bare
-					// it would make Matcher.appendReplacement throw
-					javaRepl.append("\\\\");
-					continue;
-				}
-				i++;
-				c = awkRepl.charAt(i);
+			if (c != '\\') {
+				appendLiteral(javaRepl, c);
+			} else if (i == length - 1) {
+				// a trailing backslash is a literal backslash; left bare it
+				// would make Matcher.appendReplacement throw
+				javaRepl.append("\\\\");
+			} else {
+				c = awkRepl.charAt(++i);
 				if (c == '&') {
 					javaRepl.append('&');
-					continue;
 				} else if (c == '\\') {
 					javaRepl.append("\\\\");
-					continue;
-				} else if (backreferences) {
-					if (Character.isDigit(c)) {
-						if (c - '0' <= maxGroup) {
-							javaRepl.append('$').append(c);
-						}
-						// references beyond the pattern's groups expand to
-						// the empty string, as in gawk
-						continue;
+				} else if (Character.isDigit(c)) {
+					if (c - '0' <= maxGroup) {
+						javaRepl.append('$').append(c);
 					}
-					// gensub(): any other \c is a plain c, as in gawk
 				} else {
-					// sub()/gsub(): any other \c stays a literal \c, as in
-					// POSIX, gawk and mawk; a bare backslash would be eaten
-					// by Matcher.appendReplacement as an escape
-					javaRepl.append("\\\\");
+					appendLiteral(javaRepl, c);
 				}
 			}
-
-			if (c == '$') {
-				javaRepl.append("\\$");
-			} else if (c == '&') {
-				javaRepl.append("$0");
-			} else {
-				javaRepl.append(c);
-			}
 		}
-
 		return javaRepl.toString();
 	}
 
+	/**
+	 * Appends a character that is literal in AWK replacement text, escaping
+	 * it for {@link Matcher#appendReplacement} and mapping {@code &} to the
+	 * whole match.
+	 */
+	private static void appendLiteral(StringBuilder javaRepl, char c) {
+		if (c == '$') {
+			javaRepl.append("\\$");
+		} else if (c == '&') {
+			javaRepl.append("$0");
+		} else {
+			javaRepl.append(c);
+		}
+	}
+
 	/**
 	 * Returns the pattern itself, or its case-insensitive twin when
 	 * {@code IGNORECASE} is set. Twins are compiled once and cached here:

src/main/java/io/jawk/util/AwkSettings.java

@@ -291,7 +291,9 @@ public void setUseSortedArrayKeys(boolean useSortedArrayKeys) {
 	 * such as arrays of arrays ({@code a[i][j]}, subarray operands like
 	 * {@code split(..., a[i])}) and typed regexp literals ({@code @/re/}),
 	 * and treating the gawk-specific {@code BEGINFILE} / {@code ENDFILE}
-	 * patterns as ordinary identifiers.
+	 * patterns as ordinary identifiers. At run time, the backslashes in
+	 * {@code sub()} / {@code gsub()} replacement text then follow the POSIX
+	 * rules instead of gawk's.
 	 *
 	 * @return {@code true} when POSIX compile-time behavior is enforced
 	 */

src/site/markdown/behavior-changes.md

@@ -20,6 +20,16 @@ released version automatically via .github/scripts/stamp-behavior-changes.sh.
 
 ## Unreleased
 
+- Backslashes in the replacement text of `sub()` and `gsub()` now follow gawk's documented rules
+  instead of the POSIX ones: `\\` (the string written `"\\\\"` in the source) now produces `\\`
+  where it previously produced `\`, `\\\&` produces `\&`, and `\\\\` produces `\\`. `\&` and `\\&`
+  are unchanged. With `--posix`, the previous POSIX behavior is kept, as in mawk and One True Awk,
+  and a program compiled with `--posix -K` keeps it when loaded with `-L` (the compiled program now
+  records its mode, so programs compiled by earlier versions must be recompiled). `gensub()` is
+  unchanged ([#614](https://github.com/jawkio/jawk/issues/614)).
+- For Java embedders, `JRT.prepareReplacement()` is replaced by `JRT.prepareSubReplacement()`,
+  which takes the POSIX switch, and `JRT.prepareGensubReplacement()`, which takes the highest
+  capture group number ([#614](https://github.com/jawkio/jawk/issues/614)).
 - In the replacement text of `sub()` and `gsub()`, a backslash followed by any character other
   than `&` or `\` is now kept literally, as in POSIX, gawk and mawk: `gsub(/'/, "'\\''")` on `'`
   now yields `'\''`. Previously the backslash was dropped (`'''`), and a `"\\$"` sequence or a

src/site/markdown/cli-reference.md

@@ -55,7 +55,7 @@ jawk --version
 >   - `-F <fs>` sets the initial field separator.
 >   - `--locale <locale>` sets the locale used to format numbers, through `Locale.forLanguageTag(...)`.
 >   - `-t` keeps associative array keys sorted.
->   - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, all gawk `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns.
+>   - `--posix` enforces POSIX-oriented compile-time behavior such as disabling gawk-style nested arrays, all gawk `@` forms, and the `BEGINFILE` / `ENDFILE` special patterns, and makes `sub()` / `gsub()` follow the POSIX backslash rules in their replacement text (see [Where Jawk follows gawk](compatibility.html#where-jawk-follows-gawk)).
 >
 > - Extensions and sandbox
 >
@@ -82,7 +82,7 @@ jawk --version
 ## Execution Notes
 
 - `--dump-syntax`, `--dump-intermediate`, `-K`, `-h`, `-?`, `-V`, `--version`, and `--list-ext` do not execute the script, and ignore an ambient `JAWK_PERSISTENT_MEMORY`. `--dump-syntax`, `--dump-intermediate`, and `-K` also ignore `--persist`, while the standalone modes (`-h`, `-?`, `-V`, `--version`, `--list-ext`) reject it like any other extra argument. `--profile` does execute the script, and keeps the normal AWK output on stdout.
-- `-f` compiles source now, `-L` loads a program compiled earlier. `--posix` is rejected together with `-L`: loading a precompiled program bypasses source compilation, so there is no compile-time behavior left to restrict.
+- `-f` compiles source now, `-L` loads a program compiled earlier. `--posix` is rejected together with `-L`: loading a precompiled program bypasses source compilation, so there is no compile-time behavior left to restrict. A program compiled with `--posix -K` records that mode and keeps the POSIX `sub()` / `gsub()` replacement rules when loaded with `-L`.
 - `-S` applies at compile time as well as at run time — a sandboxed run rejects the forbidden constructs while compiling the script.
 
 ## See Also

src/site/markdown/compatibility.md.vm

@@ -375,6 +375,7 @@ Jawk follows gawk in several places where gawk departs from historical AWK:
 
 - A regexp literal used as an ordinary expression evaluates as `$0 ~ /re/`, per POSIX; `x = /re/` assigns 0 or 1, not the pattern text.
 - Typed regexp literals (`@/re/`) are accepted; `--posix` rejects them.
+- Backslashes in the replacement text of `sub()` and `gsub()` follow gawk's rules: `\&` is a literal `&`, `\\&` a literal backslash followed by the matched text, `\\\&` a literal `\&`, `\\\\` a literal `\\`, and any other backslash is kept as is, so the replacement `\\` (written `"\\\\"` in the source) produces `\\` and `\q` produces `\q`. With `--posix`, `\\` is a single literal backslash, as in mawk and One True Awk, and everything else is kept as is. `gensub()` always uses its own gawk rules: `\N` is capture group N, `\\` a backslash, `\&` an ampersand, and any other `\c` a plain `c`.
 - `IGNORECASE` is a built-in special variable managed by the runtime, like `FS` or `RS`: a truthy value makes regexp matching, field and `split()` separators, string comparisons, `index()`, and gawk array sorting case-insensitive.
 - Because the [gawk compatibility extension](extensions.html#gawk) is enabled by default, `gensub`, `typeof`, `isarray`, `asort`, `asorti`, `mkbool`, `patsplit`, `strtonum`, `systime`, `mktime`, `strftime`, `bindtextdomain`, `dcgettext`, and `dcngettext` are function names and can no longer be used as variables. Load an explicit extension list without `GawkExtension` to reclaim those identifiers.
 - Calling a user-defined function with more arguments than it declares is accepted: the extra expressions are evaluated and discarded, and a gawk-style warning is printed to stderr.

src/test/java/io/jawk/AwkEvalTest.java

@@ -255,6 +255,17 @@ public void testEvalHonorsSettingsChangesBetweenCalls() throws Exception {
 		assertEquals("right", awk.eval(expression, "left:right"));
 	}
 
+	@Test
+	public void testEvalAppliesPosixModeOfTheExpression() throws Exception {
+		// the replacement \\ is a single backslash in POSIX mode, \\ in gawk's
+		String expression = "gsub(/x/, \"\\\\\\\\\") ? $0 : $0";
+		assertEquals("\\\\", new Awk().eval(expression, "x"));
+
+		AwkSettings settings = new AwkSettings();
+		settings.setPosix(true);
+		assertEquals("\\", new Awk(settings).eval(expression, "x"));
+	}
+
 	@Test
 	public void testEvalStringInputRepresentsSingleRecord() throws Exception {
 		Awk awk = new Awk();