Preserve padded handler returns and unpacked comprehension iterators

src/decompiler.rs

@@ -7540,6 +7540,12 @@ impl<'a> Ctx<'a> {
         while k2 < self.instrs.len() {
             let c = &self.instrs[k2];
             match c.op {
+                Op::NOP | Op::NOT_TAKEN | Op::CACHE => {
+                    // Padding does not end the post-POP_EXCEPT return
+                    // trailer. Keep scanning through it so an unnamed
+                    // handler retains its already loaded return value.
+                    k2 += 1;
+                }
                 Op::LOAD_CONST | Op::STORE_FAST | Op::STORE_NAME
                 | Op::DELETE_FAST | Op::DELETE_NAME | Op::STORE_DEREF
                 | Op::DELETE_DEREF => {
@@ -43236,6 +43242,12 @@ impl<'a> Ctx<'a> {
                     cur.target = Some(target);
                 }
                 comp.target_seen = true;
+                // UNPACK_SEQUENCE consumed the modeled iterator while
+                // collecting its tuple target. The VM keeps a distinct
+                // iterator slot until END_FOR/POP_ITER. Restore that slot,
+                // as the simple-target path already does, instead of
+                // suppressing cleanup for generators that still own one.
+                self.push(value);
                 return;
             }
         }

tests/behavior/cases/b_unpacked_comprehension_cleanup.py

@@ -0,0 +1,33 @@
+# MIN_VERSION: 3.12
+
+
+def unpack4(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [a + b + c + d for rows in groups for a, b, c, d in rows]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack_list(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c) for a, b in groups for c, d in b]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack6(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c, e) for a, b in groups for c, d in b for e, f in d]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+print(unpack4([[], [(1, 2, 3, 4)], [(5, 6, 7, 8)]]))
+print(unpack_list([(1, [(3, 4), (5, 6)]), (7, [])]))
+print(unpack6([(1, [(3, [(5, 6), (7, 8)])]), (9, [])]))

tests/corpus/3.11.14/handler_return_padding_3.py

@@ -0,0 +1,15 @@
+# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE.
+# Regenerate with the matching CPython: python tools/compile_padded_returns.py
+# Jump targets and exception-table boundaries are relocated with the padding.
+
+async def handled(read, log):
+    try:
+        await read()
+        log("saved")
+    except ValueError:
+        log("locked")
+        return False
+    except Exception as error:
+        log("failed", error)
+        return False
+    return True

tests/corpus/3.12.3/handler_return_padding_3.py

@@ -0,0 +1,15 @@
+# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE.
+# Regenerate with the matching CPython: python tools/compile_padded_returns.py
+# Jump targets and exception-table boundaries are relocated with the padding.
+
+async def handled(read, log):
+    try:
+        await read()
+        log("saved")
+    except ValueError:
+        log("locked")
+        return False
+    except Exception as error:
+        log("failed", error)
+        return False
+    return True

tests/corpus/3.13.7/handler_return_padding_3.py

@@ -0,0 +1,15 @@
+# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE.
+# Regenerate with the matching CPython: python tools/compile_padded_returns.py
+# Jump targets and exception-table boundaries are relocated with the padding.
+
+async def handled(read, log):
+    try:
+        await read()
+        log("saved")
+    except ValueError:
+        log("locked")
+        return False
+    except Exception as error:
+        log("failed", error)
+        return False
+    return True

tests/corpus/3.14.2/handler_return_padding_3.py

@@ -0,0 +1,15 @@
+# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE.
+# Regenerate with the matching CPython: python tools/compile_padded_returns.py
+# Jump targets and exception-table boundaries are relocated with the padding.
+
+async def handled(read, log):
+    try:
+        await read()
+        log("saved")
+    except ValueError:
+        log("locked")
+        return False
+    except Exception as error:
+        log("failed", error)
+        return False
+    return True

tests/fixtures/handler_return_padding.py

@@ -0,0 +1,15 @@
+# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE.
+# Regenerate with the matching CPython: python tools/compile_padded_returns.py
+# Jump targets and exception-table boundaries are relocated with the padding.
+
+async def handled(read, log):
+    try:
+        await read()
+        log("saved")
+    except ValueError:
+        log("locked")
+        return False
+    except Exception as error:
+        log("failed", error)
+        return False
+    return True

tests/fixtures/unpacked_comprehension_cleanup.py

@@ -0,0 +1,75 @@
+# Compile with each named CPython version; the committed .pyc files are
+# exercised by the existing Rust integration suite.
+
+# Python 3.12+ inlines these comprehensions; 3.11 is the legacy control.
+
+def unpack4(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [a + b + c + d for rows in groups for a, b, c, d in rows]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack_list(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c) for a, b in groups for c, d in b]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack_set(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = {(a, c) for a, b in groups for c, d in b}
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack_dict(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = {a: c for a, b in groups for c, d in b}
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def unpack6(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c, e) for a, b in groups for c, d in b for e, f in d]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def simple(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c) for a in groups for c in a]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def mixed(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c) for key, a in groups for c in a]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after
+
+
+def mixed_inner(groups):
+    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60
+    result = [(a, c) for a in groups for key, c in a]
+    after = 0
+    for number in (1, 2, 3):
+        after += number
+    return result, (a, b, c, d, e, f), after

tests/integration.rs

@@ -169,3 +169,40 @@ fn py314_hello() {
         "py3.14 hello",
     );
 }
+
+#[test]
+fn unpacked_targets_keep_iterator_cleanup_and_saved_locals() {
+    // PEP 709 inlining starts in 3.12; retain the 3.11 compatibility case.
+    for version in ["3.11", "3.12", "3.13", "3.14"] {
+        let (src, clean) = decompile_fixture(&format!("unpacked_comprehension_cleanup.{version}.pyc"));
+        assert!(clean, "Python {version}: output marked incomplete:\n{src}");
+        for (name, expression) in [
+            ("unpack4", "[a + b + c + d for rows in groups for a, b, c, d in rows]"),
+            ("unpack_list", "[(a, c) for a, b in groups for c, d in b]"),
+            ("unpack_set", "{(a, c) for a, b in groups for c, d in b}"),
+            ("unpack_dict", "{a: c for a, b in groups for c, d in b}"),
+            ("unpack6", "[(a, c, e) for a, b in groups for c, d in b for e, f in d]"),
+            ("simple", "[(a, c) for a in groups for c in a]"),
+            ("mixed", "[(a, c) for key, a in groups for c in a]"),
+            ("mixed_inner", "[(a, c) for a in groups for key, c in a]"),
+        ] {
+            let expected = format!(
+                "def {name}(groups):\n    a, b, c, d, e, f = 10, 20, 30, 40, 50, 60\n    result = {expression}\n    after = 0\n    for number in (1, 2, 3):\n        after += number\n    return result, (a, b, c, d, e, f), after");
+            assert_contains(&src, &[&expected], version);
+        }
+    }
+}
+
+#[test]
+fn padded_handler_returns_keep_false_before_the_success_tail() {
+    for version in ["3.11", "3.12", "3.13", "3.14"] {
+        for padding in [1, 3, 9] {
+            let (src, clean) = decompile_fixture(
+                &format!("handler_return_padding_{padding}.{version}.pyc"));
+            assert!(clean, "Python {version}, {padding} NOPs: incomplete:\n{src}");
+            assert_contains(&src, &[
+                "async def handled(read, log):\n    try:\n        await read()\n        log('saved')\n    except ValueError:\n        log('locked')\n        return False\n    except Exception as error:\n        log('failed', error)\n        return False\n    return True",
+            ], version);
+        }
+    }
+}

tools/compile_padded_returns.py

@@ -0,0 +1,81 @@
+"""Regenerate the padded handler-return fixtures with CPython 3.11–3.14.
+
+Run once per interpreter from any directory. This prepares bytecode inputs
+for tests/integration.rs; it is not a separate test runner. Only RETURN_VALUE
+is padded, leaving versions that emit RETURN_CONST as compatibility controls.
+"""
+import importlib.util
+import marshal
+from pathlib import Path
+import sys
+import types
+
+def pad_return_values(code, count=3):
+    """Add NOP padding while retaining jumps and exact handler boundaries."""
+    import dis
+    instructions = list(dis.get_instructions(code))
+    # This compact fixture needs no extended operands. NOPs
+    # reproduce the padded return layout, preserving the return's value.
+    assert not any(item.opname == 'EXTENDED_ARG' for item in instructions)
+    starts, sizes, cursor = {}, {}, 0
+    for index, item in enumerate(instructions):
+        starts[item.offset] = cursor
+        end = instructions[index + 1].offset if index + 1 < len(instructions) else len(code.co_code)
+        sizes[item.offset] = end - item.offset
+        cursor += sizes[item.offset] + (2 * count if item.opname == 'RETURN_VALUE' else 0)
+    starts[len(code.co_code)] = cursor
+    output = bytearray()
+    for item in instructions:
+        if item.opname == 'RETURN_VALUE':
+            output.extend(bytes((dis.opmap['NOP'], 0)) * count)
+        actual = len(output)
+        raw = bytearray(code.co_code[item.offset:item.offset + sizes[item.offset]])
+        if item.opcode in getattr(dis, "hasjump", dis.hasjrel + dis.hasjabs):
+            delta = (starts[item.argval] - (actual + sizes[item.offset])) // 2
+            operand = -delta if 'BACKWARD' in item.opname else delta
+            assert 0 <= operand <= 255
+            raw[1] = operand
+        output.extend(raw)
+    def varint(value, marker=False):
+        digits = [value & 63]
+        while value >> 6:
+            value >>= 6
+            digits.append(value & 63)
+        digits.reverse()
+        for index in range(len(digits) - 1):
+            digits[index] |= 64
+        if marker:
+            digits[0] |= 128
+        return bytes(digits)
+    table = b''.join(varint(starts[row.start] // 2, True)
+                     + varint((starts[row.end] - starts[row.start]) // 2)
+                     + varint(starts[row.target] // 2)
+                     + varint((row.depth << 1) | row.lasti)
+                     for row in dis.Bytecode(code).exception_entries)
+    padded = code.replace(co_code=bytes(output), co_exceptiontable=table, co_linetable=b'')
+    return padded
+
+
+if __name__ == "__main__":
+    if not (3, 11) <= sys.version_info[:2] <= (3, 14):
+        raise SystemExit("Use CPython 3.11, 3.12, 3.13 or 3.14")
+    root = Path(__file__).resolve().parents[1]
+    source_path = Path("tests/fixtures/handler_return_padding.py")
+    source = (root / source_path).read_bytes()
+    module = compile(source, source_path.as_posix(), "exec", dont_inherit=True)
+    version = f"{sys.version_info.major}.{sys.version_info.minor}"
+    header = (importlib.util.MAGIC_NUMBER + (3).to_bytes(4, "little")
+              + importlib.util.source_hash(source))
+    for count in (1, 3, 9):
+        padded = module.replace(co_consts=tuple(
+            pad_return_values(value, count) if isinstance(value, types.CodeType) else value
+            for value in module.co_consts))
+        path = root / f"tests/pyc/handler_return_padding_{count}.{version}.pyc"
+        path.write_bytes(header + marshal.dumps(padded))
+        if count == 3:
+            # Also exercise a representative padded input in verify_corpus.py.
+            # Keep the directory named after the interpreter producing the pyc.
+            corpus = root / "tests/corpus" / ".".join(map(str, sys.version_info[:3]))
+            corpus.mkdir(parents=True, exist_ok=True)
+            (corpus / "handler_return_padding_3.py").write_bytes(source)
+            (corpus / "handler_return_padding_3.pyc").write_bytes(header + marshal.dumps(padded))