diff --git a/src/decompiler.rs b/src/decompiler.rs index 33a3465d..0882bb1d 100644 --- a/src/decompiler.rs +++ b/src/decompiler.rs @@ -7540,6 +7540,12 @@ impl<'a> Ctx<'a> { while k2 < self.instrs.len() { let c = &self.instrs[k2]; match c.op { + Op::NOP | Op::NOT_TAKEN | Op::CACHE => { + // Padding does not end the post-POP_EXCEPT return + // trailer. Keep scanning through it so an unnamed + // handler retains its already loaded return value. + k2 += 1; + } Op::LOAD_CONST | Op::STORE_FAST | Op::STORE_NAME | Op::DELETE_FAST | Op::DELETE_NAME | Op::STORE_DEREF | Op::DELETE_DEREF => { @@ -43236,6 +43242,12 @@ impl<'a> Ctx<'a> { cur.target = Some(target); } comp.target_seen = true; + // UNPACK_SEQUENCE consumed the modeled iterator while + // collecting its tuple target. The VM keeps a distinct + // iterator slot until END_FOR/POP_ITER. Restore that slot, + // as the simple-target path already does, instead of + // suppressing cleanup for generators that still own one. + self.push(value); return; } } diff --git a/tests/behavior/cases/b_unpacked_comprehension_cleanup.py b/tests/behavior/cases/b_unpacked_comprehension_cleanup.py new file mode 100644 index 00000000..2b30f9b5 --- /dev/null +++ b/tests/behavior/cases/b_unpacked_comprehension_cleanup.py @@ -0,0 +1,33 @@ +# MIN_VERSION: 3.12 + + +def unpack4(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [a + b + c + d for rows in groups for a, b, c, d in rows] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_list(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a, b in groups for c, d in b] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack6(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c, e) for a, b in groups for c, d in b for e, f in d] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +print(unpack4([[], [(1, 2, 3, 4)], [(5, 6, 7, 8)]])) +print(unpack_list([(1, [(3, 4), (5, 6)]), (7, [])])) +print(unpack6([(1, [(3, [(5, 6), (7, 8)])]), (9, [])])) diff --git a/tests/corpus/3.11.14/handler_return_padding_3.py b/tests/corpus/3.11.14/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.11.14/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.11.14/handler_return_padding_3.pyc b/tests/corpus/3.11.14/handler_return_padding_3.pyc new file mode 100644 index 00000000..ff2d6ead Binary files /dev/null and b/tests/corpus/3.11.14/handler_return_padding_3.pyc differ diff --git a/tests/corpus/3.12.3/handler_return_padding_3.py b/tests/corpus/3.12.3/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.12.3/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.12.3/handler_return_padding_3.pyc b/tests/corpus/3.12.3/handler_return_padding_3.pyc new file mode 100644 index 00000000..aae4bf0e Binary files /dev/null and b/tests/corpus/3.12.3/handler_return_padding_3.pyc differ diff --git a/tests/corpus/3.13.7/handler_return_padding_3.py b/tests/corpus/3.13.7/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.13.7/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.13.7/handler_return_padding_3.pyc b/tests/corpus/3.13.7/handler_return_padding_3.pyc new file mode 100644 index 00000000..0f68db61 Binary files /dev/null and b/tests/corpus/3.13.7/handler_return_padding_3.pyc differ diff --git a/tests/corpus/3.14.2/handler_return_padding_3.py b/tests/corpus/3.14.2/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.14.2/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.14.2/handler_return_padding_3.pyc b/tests/corpus/3.14.2/handler_return_padding_3.pyc new file mode 100644 index 00000000..0f1c0628 Binary files /dev/null and b/tests/corpus/3.14.2/handler_return_padding_3.pyc differ diff --git a/tests/fixtures/handler_return_padding.py b/tests/fixtures/handler_return_padding.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/fixtures/handler_return_padding.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/fixtures/unpacked_comprehension_cleanup.py b/tests/fixtures/unpacked_comprehension_cleanup.py new file mode 100644 index 00000000..948c5ae3 --- /dev/null +++ b/tests/fixtures/unpacked_comprehension_cleanup.py @@ -0,0 +1,75 @@ +# Compile with each named CPython version; the committed .pyc files are +# exercised by the existing Rust integration suite. + +# Python 3.12+ inlines these comprehensions; 3.11 is the legacy control. + +def unpack4(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [a + b + c + d for rows in groups for a, b, c, d in rows] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_list(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a, b in groups for c, d in b] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_set(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = {(a, c) for a, b in groups for c, d in b} + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_dict(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = {a: c for a, b in groups for c, d in b} + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack6(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c, e) for a, b in groups for c, d in b for e, f in d] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def simple(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a in groups for c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def mixed(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for key, a in groups for c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def mixed_inner(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a in groups for key, c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after diff --git a/tests/integration.rs b/tests/integration.rs index f4af5082..1b2ccef1 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -169,3 +169,40 @@ fn py314_hello() { "py3.14 hello", ); } + +#[test] +fn unpacked_targets_keep_iterator_cleanup_and_saved_locals() { + // PEP 709 inlining starts in 3.12; retain the 3.11 compatibility case. + for version in ["3.11", "3.12", "3.13", "3.14"] { + let (src, clean) = decompile_fixture(&format!("unpacked_comprehension_cleanup.{version}.pyc")); + assert!(clean, "Python {version}: output marked incomplete:\n{src}"); + for (name, expression) in [ + ("unpack4", "[a + b + c + d for rows in groups for a, b, c, d in rows]"), + ("unpack_list", "[(a, c) for a, b in groups for c, d in b]"), + ("unpack_set", "{(a, c) for a, b in groups for c, d in b}"), + ("unpack_dict", "{a: c for a, b in groups for c, d in b}"), + ("unpack6", "[(a, c, e) for a, b in groups for c, d in b for e, f in d]"), + ("simple", "[(a, c) for a in groups for c in a]"), + ("mixed", "[(a, c) for key, a in groups for c in a]"), + ("mixed_inner", "[(a, c) for a in groups for key, c in a]"), + ] { + let expected = format!( + "def {name}(groups):\n a, b, c, d, e, f = 10, 20, 30, 40, 50, 60\n result = {expression}\n after = 0\n for number in (1, 2, 3):\n after += number\n return result, (a, b, c, d, e, f), after"); + assert_contains(&src, &[&expected], version); + } + } +} + +#[test] +fn padded_handler_returns_keep_false_before_the_success_tail() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + for padding in [1, 3, 9] { + let (src, clean) = decompile_fixture( + &format!("handler_return_padding_{padding}.{version}.pyc")); + assert!(clean, "Python {version}, {padding} NOPs: incomplete:\n{src}"); + assert_contains(&src, &[ + "async def handled(read, log):\n try:\n await read()\n log('saved')\n except ValueError:\n log('locked')\n return False\n except Exception as error:\n log('failed', error)\n return False\n return True", + ], version); + } + } +} diff --git a/tests/pyc/handler_return_padding_1.3.11.pyc b/tests/pyc/handler_return_padding_1.3.11.pyc new file mode 100644 index 00000000..0f1660ad Binary files /dev/null and b/tests/pyc/handler_return_padding_1.3.11.pyc differ diff --git a/tests/pyc/handler_return_padding_1.3.12.pyc b/tests/pyc/handler_return_padding_1.3.12.pyc new file mode 100644 index 00000000..aae4bf0e Binary files /dev/null and b/tests/pyc/handler_return_padding_1.3.12.pyc differ diff --git a/tests/pyc/handler_return_padding_1.3.13.pyc b/tests/pyc/handler_return_padding_1.3.13.pyc new file mode 100644 index 00000000..0f68db61 Binary files /dev/null and b/tests/pyc/handler_return_padding_1.3.13.pyc differ diff --git a/tests/pyc/handler_return_padding_1.3.14.pyc b/tests/pyc/handler_return_padding_1.3.14.pyc new file mode 100644 index 00000000..be6567e2 Binary files /dev/null and b/tests/pyc/handler_return_padding_1.3.14.pyc differ diff --git a/tests/pyc/handler_return_padding_3.3.11.pyc b/tests/pyc/handler_return_padding_3.3.11.pyc new file mode 100644 index 00000000..ff2d6ead Binary files /dev/null and b/tests/pyc/handler_return_padding_3.3.11.pyc differ diff --git a/tests/pyc/handler_return_padding_3.3.12.pyc b/tests/pyc/handler_return_padding_3.3.12.pyc new file mode 100644 index 00000000..aae4bf0e Binary files /dev/null and b/tests/pyc/handler_return_padding_3.3.12.pyc differ diff --git a/tests/pyc/handler_return_padding_3.3.13.pyc b/tests/pyc/handler_return_padding_3.3.13.pyc new file mode 100644 index 00000000..0f68db61 Binary files /dev/null and b/tests/pyc/handler_return_padding_3.3.13.pyc differ diff --git a/tests/pyc/handler_return_padding_3.3.14.pyc b/tests/pyc/handler_return_padding_3.3.14.pyc new file mode 100644 index 00000000..0f1c0628 Binary files /dev/null and b/tests/pyc/handler_return_padding_3.3.14.pyc differ diff --git a/tests/pyc/handler_return_padding_9.3.11.pyc b/tests/pyc/handler_return_padding_9.3.11.pyc new file mode 100644 index 00000000..9a90c4f1 Binary files /dev/null and b/tests/pyc/handler_return_padding_9.3.11.pyc differ diff --git a/tests/pyc/handler_return_padding_9.3.12.pyc b/tests/pyc/handler_return_padding_9.3.12.pyc new file mode 100644 index 00000000..aae4bf0e Binary files /dev/null and b/tests/pyc/handler_return_padding_9.3.12.pyc differ diff --git a/tests/pyc/handler_return_padding_9.3.13.pyc b/tests/pyc/handler_return_padding_9.3.13.pyc new file mode 100644 index 00000000..0f68db61 Binary files /dev/null and b/tests/pyc/handler_return_padding_9.3.13.pyc differ diff --git a/tests/pyc/handler_return_padding_9.3.14.pyc b/tests/pyc/handler_return_padding_9.3.14.pyc new file mode 100644 index 00000000..57241865 Binary files /dev/null and b/tests/pyc/handler_return_padding_9.3.14.pyc differ diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.11.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.11.pyc new file mode 100644 index 00000000..e5f5cc50 Binary files /dev/null and b/tests/pyc/unpacked_comprehension_cleanup.3.11.pyc differ diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.12.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.12.pyc new file mode 100644 index 00000000..bb9e343f Binary files /dev/null and b/tests/pyc/unpacked_comprehension_cleanup.3.12.pyc differ diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.13.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.13.pyc new file mode 100644 index 00000000..21d84ac7 Binary files /dev/null and b/tests/pyc/unpacked_comprehension_cleanup.3.13.pyc differ diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.14.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.14.pyc new file mode 100644 index 00000000..08964896 Binary files /dev/null and b/tests/pyc/unpacked_comprehension_cleanup.3.14.pyc differ diff --git a/tools/compile_padded_returns.py b/tools/compile_padded_returns.py new file mode 100644 index 00000000..bdb472d9 --- /dev/null +++ b/tools/compile_padded_returns.py @@ -0,0 +1,81 @@ +"""Regenerate the padded handler-return fixtures with CPython 3.11–3.14. + +Run once per interpreter from any directory. This prepares bytecode inputs +for tests/integration.rs; it is not a separate test runner. Only RETURN_VALUE +is padded, leaving versions that emit RETURN_CONST as compatibility controls. +""" +import importlib.util +import marshal +from pathlib import Path +import sys +import types + +def pad_return_values(code, count=3): + """Add NOP padding while retaining jumps and exact handler boundaries.""" + import dis + instructions = list(dis.get_instructions(code)) + # This compact fixture needs no extended operands. NOPs + # reproduce the padded return layout, preserving the return's value. + assert not any(item.opname == 'EXTENDED_ARG' for item in instructions) + starts, sizes, cursor = {}, {}, 0 + for index, item in enumerate(instructions): + starts[item.offset] = cursor + end = instructions[index + 1].offset if index + 1 < len(instructions) else len(code.co_code) + sizes[item.offset] = end - item.offset + cursor += sizes[item.offset] + (2 * count if item.opname == 'RETURN_VALUE' else 0) + starts[len(code.co_code)] = cursor + output = bytearray() + for item in instructions: + if item.opname == 'RETURN_VALUE': + output.extend(bytes((dis.opmap['NOP'], 0)) * count) + actual = len(output) + raw = bytearray(code.co_code[item.offset:item.offset + sizes[item.offset]]) + if item.opcode in getattr(dis, "hasjump", dis.hasjrel + dis.hasjabs): + delta = (starts[item.argval] - (actual + sizes[item.offset])) // 2 + operand = -delta if 'BACKWARD' in item.opname else delta + assert 0 <= operand <= 255 + raw[1] = operand + output.extend(raw) + def varint(value, marker=False): + digits = [value & 63] + while value >> 6: + value >>= 6 + digits.append(value & 63) + digits.reverse() + for index in range(len(digits) - 1): + digits[index] |= 64 + if marker: + digits[0] |= 128 + return bytes(digits) + table = b''.join(varint(starts[row.start] // 2, True) + + varint((starts[row.end] - starts[row.start]) // 2) + + varint(starts[row.target] // 2) + + varint((row.depth << 1) | row.lasti) + for row in dis.Bytecode(code).exception_entries) + padded = code.replace(co_code=bytes(output), co_exceptiontable=table, co_linetable=b'') + return padded + + +if __name__ == "__main__": + if not (3, 11) <= sys.version_info[:2] <= (3, 14): + raise SystemExit("Use CPython 3.11, 3.12, 3.13 or 3.14") + root = Path(__file__).resolve().parents[1] + source_path = Path("tests/fixtures/handler_return_padding.py") + source = (root / source_path).read_bytes() + module = compile(source, source_path.as_posix(), "exec", dont_inherit=True) + version = f"{sys.version_info.major}.{sys.version_info.minor}" + header = (importlib.util.MAGIC_NUMBER + (3).to_bytes(4, "little") + + importlib.util.source_hash(source)) + for count in (1, 3, 9): + padded = module.replace(co_consts=tuple( + pad_return_values(value, count) if isinstance(value, types.CodeType) else value + for value in module.co_consts)) + path = root / f"tests/pyc/handler_return_padding_{count}.{version}.pyc" + path.write_bytes(header + marshal.dumps(padded)) + if count == 3: + # Also exercise a representative padded input in verify_corpus.py. + # Keep the directory named after the interpreter producing the pyc. + corpus = root / "tests/corpus" / ".".join(map(str, sys.version_info[:3])) + corpus.mkdir(parents=True, exist_ok=True) + (corpus / "handler_return_padding_3.py").write_bytes(source) + (corpus / "handler_return_padding_3.pyc").write_bytes(header + marshal.dumps(padded))