From b4c0406bc5dc717c1c86c399254a63d619d38c13 Mon Sep 17 00:00:00 2001 From: Francois Chagnon Date: Tue, 13 Oct 2026 15:05:42 -0400 Subject: [PATCH 1/2] Preserve padded handler returns and unpacked iterator cleanup --- src/decompiler.rs | 12 +++ tests/fixtures/handler_return_padding.py | 15 ++++ .../unpacked_comprehension_cleanup.py | 75 ++++++++++++++++++ tests/integration.rs | 37 +++++++++ tests/pyc/handler_return_padding_1.3.11.pyc | Bin 0 -> 530 bytes tests/pyc/handler_return_padding_1.3.12.pyc | Bin 0 -> 516 bytes tests/pyc/handler_return_padding_1.3.13.pyc | Bin 0 -> 520 bytes tests/pyc/handler_return_padding_1.3.14.pyc | Bin 0 -> 545 bytes tests/pyc/handler_return_padding_3.3.11.pyc | Bin 0 -> 542 bytes tests/pyc/handler_return_padding_3.3.12.pyc | Bin 0 -> 516 bytes tests/pyc/handler_return_padding_3.3.13.pyc | Bin 0 -> 520 bytes tests/pyc/handler_return_padding_3.3.14.pyc | Bin 0 -> 557 bytes tests/pyc/handler_return_padding_9.3.11.pyc | Bin 0 -> 579 bytes tests/pyc/handler_return_padding_9.3.12.pyc | Bin 0 -> 516 bytes tests/pyc/handler_return_padding_9.3.13.pyc | Bin 0 -> 520 bytes tests/pyc/handler_return_padding_9.3.14.pyc | Bin 0 -> 594 bytes .../unpacked_comprehension_cleanup.3.11.pyc | Bin 0 -> 4364 bytes .../unpacked_comprehension_cleanup.3.12.pyc | Bin 0 -> 3386 bytes .../unpacked_comprehension_cleanup.3.13.pyc | Bin 0 -> 3220 bytes .../unpacked_comprehension_cleanup.3.14.pyc | Bin 0 -> 3278 bytes tools/compile_padded_returns.py | 74 +++++++++++++++++ 21 files changed, 213 insertions(+) create mode 100644 tests/fixtures/handler_return_padding.py create mode 100644 tests/fixtures/unpacked_comprehension_cleanup.py create mode 100644 tests/pyc/handler_return_padding_1.3.11.pyc create mode 100644 tests/pyc/handler_return_padding_1.3.12.pyc create mode 100644 tests/pyc/handler_return_padding_1.3.13.pyc create mode 100644 tests/pyc/handler_return_padding_1.3.14.pyc create mode 100644 tests/pyc/handler_return_padding_3.3.11.pyc create mode 100644 tests/pyc/handler_return_padding_3.3.12.pyc create mode 100644 tests/pyc/handler_return_padding_3.3.13.pyc create mode 100644 tests/pyc/handler_return_padding_3.3.14.pyc create mode 100644 tests/pyc/handler_return_padding_9.3.11.pyc create mode 100644 tests/pyc/handler_return_padding_9.3.12.pyc create mode 100644 tests/pyc/handler_return_padding_9.3.13.pyc create mode 100644 tests/pyc/handler_return_padding_9.3.14.pyc create mode 100644 tests/pyc/unpacked_comprehension_cleanup.3.11.pyc create mode 100644 tests/pyc/unpacked_comprehension_cleanup.3.12.pyc create mode 100644 tests/pyc/unpacked_comprehension_cleanup.3.13.pyc create mode 100644 tests/pyc/unpacked_comprehension_cleanup.3.14.pyc create mode 100644 tools/compile_padded_returns.py diff --git a/src/decompiler.rs b/src/decompiler.rs index 33a3465d..0882bb1d 100644 --- a/src/decompiler.rs +++ b/src/decompiler.rs @@ -7540,6 +7540,12 @@ impl<'a> Ctx<'a> { while k2 < self.instrs.len() { let c = &self.instrs[k2]; match c.op { + Op::NOP | Op::NOT_TAKEN | Op::CACHE => { + // Padding does not end the post-POP_EXCEPT return + // trailer. Keep scanning through it so an unnamed + // handler retains its already loaded return value. + k2 += 1; + } Op::LOAD_CONST | Op::STORE_FAST | Op::STORE_NAME | Op::DELETE_FAST | Op::DELETE_NAME | Op::STORE_DEREF | Op::DELETE_DEREF => { @@ -43236,6 +43242,12 @@ impl<'a> Ctx<'a> { cur.target = Some(target); } comp.target_seen = true; + // UNPACK_SEQUENCE consumed the modeled iterator while + // collecting its tuple target. The VM keeps a distinct + // iterator slot until END_FOR/POP_ITER. Restore that slot, + // as the simple-target path already does, instead of + // suppressing cleanup for generators that still own one. + self.push(value); return; } } diff --git a/tests/fixtures/handler_return_padding.py b/tests/fixtures/handler_return_padding.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/fixtures/handler_return_padding.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/fixtures/unpacked_comprehension_cleanup.py b/tests/fixtures/unpacked_comprehension_cleanup.py new file mode 100644 index 00000000..948c5ae3 --- /dev/null +++ b/tests/fixtures/unpacked_comprehension_cleanup.py @@ -0,0 +1,75 @@ +# Compile with each named CPython version; the committed .pyc files are +# exercised by the existing Rust integration suite. + +# Python 3.12+ inlines these comprehensions; 3.11 is the legacy control. + +def unpack4(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [a + b + c + d for rows in groups for a, b, c, d in rows] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_list(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a, b in groups for c, d in b] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_set(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = {(a, c) for a, b in groups for c, d in b} + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_dict(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = {a: c for a, b in groups for c, d in b} + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack6(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c, e) for a, b in groups for c, d in b for e, f in d] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def simple(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a in groups for c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def mixed(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for key, a in groups for c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def mixed_inner(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a in groups for key, c in a] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after diff --git a/tests/integration.rs b/tests/integration.rs index f4af5082..1b2ccef1 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -169,3 +169,40 @@ fn py314_hello() { "py3.14 hello", ); } + +#[test] +fn unpacked_targets_keep_iterator_cleanup_and_saved_locals() { + // PEP 709 inlining starts in 3.12; retain the 3.11 compatibility case. + for version in ["3.11", "3.12", "3.13", "3.14"] { + let (src, clean) = decompile_fixture(&format!("unpacked_comprehension_cleanup.{version}.pyc")); + assert!(clean, "Python {version}: output marked incomplete:\n{src}"); + for (name, expression) in [ + ("unpack4", "[a + b + c + d for rows in groups for a, b, c, d in rows]"), + ("unpack_list", "[(a, c) for a, b in groups for c, d in b]"), + ("unpack_set", "{(a, c) for a, b in groups for c, d in b}"), + ("unpack_dict", "{a: c for a, b in groups for c, d in b}"), + ("unpack6", "[(a, c, e) for a, b in groups for c, d in b for e, f in d]"), + ("simple", "[(a, c) for a in groups for c in a]"), + ("mixed", "[(a, c) for key, a in groups for c in a]"), + ("mixed_inner", "[(a, c) for a in groups for key, c in a]"), + ] { + let expected = format!( + "def {name}(groups):\n a, b, c, d, e, f = 10, 20, 30, 40, 50, 60\n result = {expression}\n after = 0\n for number in (1, 2, 3):\n after += number\n return result, (a, b, c, d, e, f), after"); + assert_contains(&src, &[&expected], version); + } + } +} + +#[test] +fn padded_handler_returns_keep_false_before_the_success_tail() { + for version in ["3.11", "3.12", "3.13", "3.14"] { + for padding in [1, 3, 9] { + let (src, clean) = decompile_fixture( + &format!("handler_return_padding_{padding}.{version}.pyc")); + assert!(clean, "Python {version}, {padding} NOPs: incomplete:\n{src}"); + assert_contains(&src, &[ + "async def handled(read, log):\n try:\n await read()\n log('saved')\n except ValueError:\n log('locked')\n return False\n except Exception as error:\n log('failed', error)\n return False\n return True", + ], version); + } + } +} diff --git a/tests/pyc/handler_return_padding_1.3.11.pyc b/tests/pyc/handler_return_padding_1.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f1660ad70c02470b891672075a51ecfefca979b GIT binary patch literal 530 zcmZWlK}!Nb6n-<~S}R%-ER;bAb;wKpfeJj>r4Ygp>>`UZnzZf8?3zZ}Md>ei?PQ3* zAW!)><`(?{!JDVvx++lv@0;)Oeczkmy~kn^1_0yn-o&`R`uaw`K4i&eQ6zof(1Tmx zBo8TCrfU~~hP;b4`$V_|f?gdcv|)h#2rD>Uobb#F&}Xk9;_HNy0lqO>y@N^n>=3+F z0PiD^@uipk^%j9M9rED)`V0DZJ9IHS)TRSeMr)M-CZI!KXpjqdlQf~liqIrn^QR5N zIi|tB`L7COLdd;nq57qKdFsC;&Kb4{#IvL;3g1y z`$>2+G70vfBTQ&CW3Y*#AY5S`i0CFaZ-KUxg|Elgud8xd0|f?y>=Y{HRUHF)O2-o@nb3sl5Hv@?Zb zVHbjkrIn=iA8hLnP)KLxTs-2z&b~ME<_$adVOb^zfK%&q>g)Y#^^@@6!wB_E0xb}5 z0b3AYj+NO!17Hw7r_+0)um(uVBrrIFP9!>~&O;h<$6Nf;Fi1gF6xfOU5dvc#uB~bC zqYi)>P{Waq{p*SlaDOnHG!w_H?-e^@ohXk`q3-O+{&%U-y* zY&QEd>r2n~^4pcBUsux0@7h)2NL5mjc(qisg{V|_=4ydX{FQ_nq1JG3z3{=_5`{A? zr18Sczzt5uZc`b!4-*fVfyc2zioY;ouS101l7$5CGJ|+GVNjx*;)K`R*mrG%N|T+k eDEtg9?7O1w$VD|yG+cN8SAKIu?3pHx_O-u(Ty8f2 literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_1.3.13.pyc b/tests/pyc/handler_return_padding_1.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f68db615df47527485ea117ad3b75cf5ceaa0f2 GIT binary patch literal 520 zcmYLGJx{|h5WTaTLPA@7+ZU76Ae__RBO_HO<&iqMiY(F^~VAF1oP8OW@)rcX61lYa?)N@i!< zwqEPXs2gp#?W=B2$!MeH1j3iFtY9^nAIi7KkPzAAjdJN5t5&aBi&q0a_ks*#H0l literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_1.3.14.pyc b/tests/pyc/handler_return_padding_1.3.14.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be6567e2a55419f9b7b7c5d153027770ef9b1cac GIT binary patch literal 545 zcmYL`&q^Cn6vn?ZHP*N^*^KddJYWokIX z;s$jrAw`|mu+2?e;UzwsHqa9Bl|*=00QK{bO8*?Ym)^yo$$xgHZus_>+0c0(C; zcNuFa;Aq~*N#+9bkaNHa^9{>8|M?Q?!a7bS@4;-*vK?o{<{C~l>y^PY*Y}*UXRJdC zu_Hp*9@YrI?y`SX8NGlQk^u6+v~#Q zvvPbSk$#MD*%;v@l}-0cXT%7ytkO literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_3.3.11.pyc b/tests/pyc/handler_return_padding_3.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff2d6ead4fb36b4b5b6052154b74ad698692a636 GIT binary patch literal 542 zcmZWlO-lk%6us{~#~RTJ!G#7gP>bB;ABez(Z3-a_!8S7ZUN$t%$eS^Zw2jhVaNEid z|ADRC_Y2-G`U8SHSKaZes0a7Vxtw$E9qwB!7GVG|9w#S>%Ztx12O=fiWJy8mAc8f<Y5S`i0CFaZ-KUxg|Elgud8xd0|f?y>=Y{HRUHF)O2-o@nb3sl5Hv@?Zb zVHbjkrIn=iA8hLnP)KLxTs-2z&b~ME<_$adVOb^zfK%&q>g)Y#^^@@6!wB_E0xb}5 z0b3AYj+NO!17Hw7r_+0)um(uVBrrIFP9!>~&O;h<$6Nf;Fi1gF6xfOU5dvc#uB~bC zqYi)>P{Waq{p*SlaDOnHG!w_H?-e^@ohXk`q3-O+{&%U-y* zY&QEd>r2n~^4pcBUsux0@7h)2NL5mjc(qisg{V|_=4ydX{FQ_nq1JG3z3{=_5`{A? zr18Sczzt5uZc`b!4-*fVfyc2zioY;ouS101l7$5CGJ|+GVNjx*;)K`R*mrG%N|T+k eDEtg9?7O1w$VD|yG+cN8SAKIu?3pHx_O-u(Ty8f2 literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_3.3.13.pyc b/tests/pyc/handler_return_padding_3.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f68db615df47527485ea117ad3b75cf5ceaa0f2 GIT binary patch literal 520 zcmYLGJx{|h5WTaTLPA@7+ZU76Ae__RBO_HO<&iqMiY(F^~VAF1oP8OW@)rcX61lYa?)N@i!< zwqEPXs2gp#?W=B2$!MeH1j3iFtY9^nAIi7KkPzAAjdJN5t5&aBi&q0a_ks*#H0l literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_3.3.14.pyc b/tests/pyc/handler_return_padding_3.3.14.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f1c0628e0c747b19ff2b319b8e85c527ee0669d GIT binary patch literal 557 zcmYL`!AiqG5Qb;6O|(I4)kB50G_4m!&`UijN(J>I9!lv+kkDN<8k@37r5*%P&-wt0 zinl(1Ab6^eu~%O}s1M+5jWw{tKQsR~yG#~R<2D0;x5tP4#nVRdll1VzSoOz9TtXEx z5J44ZVcuvMLl=Na`kGGvgF*=kbCIliF)RWnOW z7G_}tb|xLzi%9?rcp4x_OieS!Ka+CR@Nk)V*tK?*8M%jGV!E0SZ*>%@%i%ls5sgj$WhG<1z(41!~ zA%)8o^MgZW3f&I%W*x^-bCC$6aOuG7L~SX;r6ae=eIfUypuD-?ayj>!hl{OqHS*iy zGDWD~YhN!S+@^4al>*+_CAdW=aW__idy@*!Sb~@7N{+o+l)aCV>LpM|Gg-;7UMflD wUdGgw93#Ek#li3{$O)P%$0?}M)zg5teX%B|DAj+ZZ&UbUi1=;Wm>zh40DbssH~;_u literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_9.3.11.pyc b/tests/pyc/handler_return_padding_9.3.11.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a90c4f1262a32a0385a309d3b69e92d1c8cb12b GIT binary patch literal 579 zcmZuu%}PQ+6h1TKwY+FWAQVDeC?Q+q3q;_;HkA;DU>mtOqe-`WW$rbNw2jg$xNT*K zZ{W_?m|gS&f;(58>mOOoaK1T*@B7XfI3JOS&H!LI+#f}6u0FpoUmdc@CSgpvz@Yh&W|ZcdPlBtU&`+f)rTWg{sMU#V(&#pVh{k;&KbE z{uU+};KmR-F$3PCsS`FLAs_W{D$Yi64f9N&%nW^y+o+Tggz4tlqr+>KIG4Lv`Wu-X SAdDFyWUT9CXAFN~=Kls*cW;LP literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_9.3.12.pyc b/tests/pyc/handler_return_padding_9.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aae4bf0eaa3f1a800afe27d58da3fefe4d7945d8 GIT binary patch literal 516 zcmYLGJxc>Y5S`i0CFaZ-KUxg|Elgud8xd0|f?y>=Y{HRUHF)O2-o@nb3sl5Hv@?Zb zVHbjkrIn=iA8hLnP)KLxTs-2z&b~ME<_$adVOb^zfK%&q>g)Y#^^@@6!wB_E0xb}5 z0b3AYj+NO!17Hw7r_+0)um(uVBrrIFP9!>~&O;h<$6Nf;Fi1gF6xfOU5dvc#uB~bC zqYi)>P{Waq{p*SlaDOnHG!w_H?-e^@ohXk`q3-O+{&%U-y* zY&QEd>r2n~^4pcBUsux0@7h)2NL5mjc(qisg{V|_=4ydX{FQ_nq1JG3z3{=_5`{A? zr18Sczzt5uZc`b!4-*fVfyc2zioY;ouS101l7$5CGJ|+GVNjx*;)K`R*mrG%N|T+k eDEtg9?7O1w$VD|yG+cN8SAKIu?3pHx_O-u(Ty8f2 literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_9.3.13.pyc b/tests/pyc/handler_return_padding_9.3.13.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f68db615df47527485ea117ad3b75cf5ceaa0f2 GIT binary patch literal 520 zcmYLGJx{|h5WTaTLPA@7+ZU76Ae__RBO_HO<&iqMiY(F^~VAF1oP8OW@)rcX61lYa?)N@i!< zwqEPXs2gp#?W=B2$!MeH1j3iFtY9^nAIi7KkPzAAjdJN5t5&aBi&q0a_ks*#H0l literal 0 HcmV?d00001 diff --git a/tests/pyc/handler_return_padding_9.3.14.pyc b/tests/pyc/handler_return_padding_9.3.14.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57241865b96a0e64aa45e4c40a837bc947407292 GIT binary patch literal 594 zcmZvZ%}VP)6vxlWG;NLb8x-``wznWHLW@vWic-PrU5JZPx)LOGMvca%Oj4-}LDaQA zfTBCGZy>npYs~h7YoR`XXKJVjH$(pX&&TgfI9VJWu^0fnJlrSGA2%}p$PXS2T`xiA z0xB>A0aP#xd82L&x&Tb_SM>JYNPa_#dgq`987PHKTf`pGHEq0L+onYwHO$bYQg}lO zD&`z9KTj57usxH8-Eay(6!IZcA)R=}9%T z*{+Lb;CPKZQxPd_u9)i`C{yUNueYC0r&TTxe&84Pop#WY!Y>}$4ekoLCk4fgy{65% z(>N$J&s6NI#N{ZVdiOoOApbAPC6<@)+RDKVE<|nLq^ymN=lj_QVZ|8<+OIne}!ZTw2sN2dwo^GqXGMX5a7k z-n{)mQ33+TaXBU2=YfTEN-yH=#jSDn`0Ug|I+uv2^+jlOZu-E8{wcsVcc?70 zt1iAAX04w-Yt6Lc2cd&lC0+~fXtnrG$_(#hoJXN?29voHgr;zU+?{w3kTv>#&4498 zA-PgsCpU1cA7KC?1Yq%+#V0Kp-l~)?T2ee&)QBZ#OPR+S(UB460z@*16TJ4@kTCSn zNBSnfHg|k;vMmm8$F>WVaAmc|*Jh6C3pHfeWLsH2QkJ)ukNcGE`^SUhryMVhQmDws zKBYFY(_fJ)(s3|Ui`5Ed*j%mi^_jz=d-c*`aN?=?n2T7nhR71*BB#=7K!x1bZ~=GnDBY=+EOE5>@wrADk#s1$19O44}*8q~Gq4VX$s5HKjn7(h?@gRXs$ zci}12)7d6KZwcU5>LUPcW&TK+zpfH^iYe$)34D7&0^}}u?G>`m{R_Hg=Z#yot6{*! zc;_}UMD8JAX#8XT0)H+_K7@&*IQhNKGF>WveaZil2>SrGw#TK=y#rkl=5t5bcS6-) z4Y}j1r*i`fYARuSw-Zn};&QL^z&l`Ap9a8ynq}8+5tCCZ%zDkcjc{WXGIPjz02)`L zqZ3YOmY_Avs+VL`+=%%lKwFtTQf9ABIHA^PMqPx4ol(cS_#Sll#Kv^0Ef~HUvobSH(9XwbYm?gQRaT)16VC8xTZtYi(d$Pn))h9f zH@cF5To$mTx9RQgqF1E!TEa8!1*@tk$096+(d{Dn%}naKruO0(laD@0{eGy-v1@JZ%=X$<{)vV*QY5TyJ5-&2l}{U1^O zAmXE+X|zzMfj!7DFe*mzB*z5oR literal 0 HcmV?d00001 diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.12.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.12.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb9e343fbf9bfb484c1c32461c3c88be6438ccf4 GIT binary patch literal 3386 zcmcImOKclO7@qMyZD$&sq-mj|t!WBP)rSMNkx)UhlsY}2N)cQ@ts>XEZmlLxJZq~D zt%(l^oD#`Sd%%?t%F(_Vi9@*M&c#VViS~rVDK}K}5Iyn#<45AkNvf1u+Ap)QXLjfN zzWM(DXU04dA>`wc%9S6r)MvlLxbvZ{Q6mV0Z;3&^B%c$50#d^T=rSZg$&dkMLjhC_ z6;L%aK<%DB>W;V9z@_@&!iP0#0I(jtrV^5)Iaf}~$vGvb+GLy*wIVILic(RIt2as9 zb^Uetjb>}(IeoJJfKf>Cl8D^VvI=qjJ;A zPG=k?mdKdgQB&EO8z#5BF#qKC=UE_QS{dtNB6&BHg&7yKskvDEwrNboXJ_WPdCN>$ z$=TFYeA zi8>1x5c1I{EPPP4{sOh$Z=hI9M9+_?xhop6q*C>|17rzXd-M zTAhFaKMBx#=ik9M``a{$JI4syX;Z!1rjMjW5114p?A4fb_kFirXC&hpJ^~x^!)5L2 zx)Y~2Sa6L6moj+1YyNc>td3Otl}X47--1zR!9olGT^D>A*M&qxEFFoc-L8Z9Zbrl^ z4EE!`s1((>CSn54Qy2emGCMILLu|l-6!%z+AU13zAuY*^eW1xOTqlJto#UTVUo1f8 zm3b^I+X15GR48|?;<7VDcGsts<6-_bZ1^l())GL^L>M`{!6u%tiO1vRNI6|gud|72 zrjoAtS0`ck)2Azws}#ebH2EM|XA^}Enu?5wWBwH(HqD|*I3}7v^Id&F-wN$76(35yGVuXXp;I6 z2D_4q?UWRk1+7qHpQS7_IN*l5)}(D(@XQ7BGu$T z(>(~Kt~%h}^a}{(c>U>c=P>jSa142hv)oK4LN?ToX8QEAA}X9>Qm|rCc)v-p`Z*ZG zf{@Y{ULKa9B7c+qBGtyQJ{0JqCuM`TqLPnTGen=!xuCh)9}28WevwQo}{j-nm}YV zw~*D$(RfxSnxN#87i6N^pnw0)PaDh1YE~VMC$434&|@^Gq@%IR@_ZzgN~Vc?NmkTE zN{Pf4idkfAI{&8UBvE=(ySmD3!T6^n` z`>wqZEy|^6F;RBjOy27%CaXf;_dxP(woCcRpE~;r!loz{kK8c-&LPK3zu4?0Zz)rD zmgiRRhU3&qZo@XT!4LiOmlbtPVbu&Te6Qcz(cWMMAYZq_C(vkNg>5644I=>QIp8$i z830>=l%jF%2|GL{kROy`@^>5ZTMX%__+njrYq-e77wh6%8pIdt;+wa{kAuPC6G;a^ zwT&F1ZSW!@b_(D+L>v^TE69r!j#4;A0o;pJ>X{u4NiSS#HeHs;LPE_FY9%^sEe6me zjq|lf(!^@_Qdil&?ya7ycng`u_?ovEzulep{UA*|r6hlXByZNJBDeaymbPb8#H}yQ zeHBUkPl}M2VJZ&*k(Hrw? z&SG@w3bmH!8LTzV7W=JUuh3{M$Xk7}f448eL32=0EyjX}&u$X~`l0)*m~ctP4I0%_T0_+AmfnZStr9RH?P}o}vBYY>wt1#`; zRE8Wt(+YQ;ek@IGNK>o6rE_KPdZwDKWWJgE^6k5Gg-mH~F>!CMm{`jc6SsYN-$QBY z5UQ-Wcc<{*u;i!p6fO;Tdpa7N56;zvDL|v8^Es6>1c8W3*%|e)kEGlIj^!@oFe4k% z&~Q6exPwU!z=j*13^Y>A!Ail0qgGH8$@GFu=nnh_9BKrhiP__>2h!Mbylnq0TCta$ zi}7l-7+;F#efOoYgK2Cau?8f%RUY1Pv-_;jKp_yIQ}`h?T2M%98cyX65@Ek!uFVAS zo>W4xGAcz@y-+d_s9tuFxl_H++6Ahg-E>xMhuFR}?y`Knc@JwUMD$I}oJ?Gk=gAPv zo0dOn6rf3hlY##;|85Nk%Yl}B+8CdagrdlVjKDl5_h9w7 z3s|Mb{=!}Dt~gjp0IX7LKNqXVye9GdpvY=PR+X@glWig_!q{qKdb7`CB~cB&5$S#q n&m>dxxdr(w8H4{Y{diiN|HfgApR@~j_(|t}P@{LrDD?Oj9tNNm literal 0 HcmV?d00001 diff --git a/tests/pyc/unpacked_comprehension_cleanup.3.14.pyc b/tests/pyc/unpacked_comprehension_cleanup.3.14.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08964896ceee90da73a53605b07c1205859c6348 GIT binary patch literal 3278 zcmcguO>7%Q6rS;}*XuZ&#&Jm6QWC5rZQbV2BsCJMkQTKLQlv<+@H!_V*IC1=v9q4_ zHto$f?6zsm1=&bQ2@+Dwi8Pn?ShyfgDfEx!01~I%pe`V8ycz$*PMwmJpOyC89ea27 z&G&up%{$#8wQ&fc&ClNY@mA@BPkx5``hlx=`kk1*b%|a(@hm(G;{0#uih?fO~NEewTih?>(*b&BITbK*;rZ9?3 zTW}De)N&^Z9z$rEuRIy)8isdkhr%_D_{Px;H{FMj6ZN6#YQ%VvF_0McaiQ4 z*Zt-b~$ar8)IftkH#*l3-ht1 zgih4=)uf>_}%bl0=se-pKyNb6wL#yVNYj}$v{_}Sy>X^cY9X@&Ulzgnc$tOr&p?Zqd(PYFkVFM3P8(P)*axCELqFNMPZ2KCO4l;j8uEmuXH`9;l0 zlULvuI%7EouuCBCiHA~Pt*6*saBs?`bL(<8buGRj=i+NUnc#O);DECH2C}?frmEaH zB0Jk&OqFq(yU$VO;{U7)f%?&!=C&%$hMFc`c!=sjXzG%FfYz&Dg@0PquDU13=;}f7 z22VP6Jb8he26r<@f*0r&s!^pX{U?sxd+jxgEoiYc8GuDm0*n?G>mPb3O%^YGn9ip* zy`{l*Z#H^$VZ)n?7B5kYO}>D|&alM4aab0cEk<*QCR6!qfW1i$+rxtCV95_UH4qJy zveBf%Ln#2F*+_|a6#E1>!%suiWYj|jvMVgPDnUJ+b>X0eVa`4$z~-=18XUTy*XL_2 z4||JIEye*d1Pea}r(rS%nu_t@sYlY(mNc~%ES@XKo2gQIJ@wV>op)}}W>fjuYueq} zoVJn5X=}ku@PRb74_#LMjWziXNdC7gH90>dAMI%JKX|wz&2<)7i<-=t9FqhpV9E{j z!=94zD;%rBP?Z_sP!p?!W1ZVS(F*Lrk@HG3%^c(n-XFb!p(XT1m7IXsC*U;Z*&>?Q z?Hjx=jThqu_vg`dciwv~UW(@8#ds!oPa6MwDw~Kb1Cf4a@$20(5}J4fCMrA@SP0MX zNP8g8qxf~aUfI2^ z?dnh}`Oj%mI~n z3Q7c+5-Yv3U2LtyN3E5>(2v%X0N*lC*SZU1S9byRzN&)*82mSMfO#5qAU;Z0OHN&k z(|IkKR0$b@$IjD7TQJ)<2+UH4-}0Aw);+9K0A{Jj{}Ho2GB3{fMAk^MW`*sgY)@hJ zjHMmZoQ20ar82~bDxMI}CYBb=MfFWG2ESn%JYB^v9LD%byMRZYbUhEA(tpV)yzv+F C!Ji)h literal 0 HcmV?d00001 diff --git a/tools/compile_padded_returns.py b/tools/compile_padded_returns.py new file mode 100644 index 00000000..6da196c4 --- /dev/null +++ b/tools/compile_padded_returns.py @@ -0,0 +1,74 @@ +"""Regenerate the padded handler-return fixtures with CPython 3.11–3.14. + +Run once per interpreter from any directory. This prepares bytecode inputs +for tests/integration.rs; it is not a separate test runner. Only RETURN_VALUE +is padded, leaving versions that emit RETURN_CONST as compatibility controls. +""" +import importlib.util +import marshal +from pathlib import Path +import sys +import types + +def pad_return_values(code, count=3): + """Add NOP padding while retaining jumps and exact handler boundaries.""" + import dis + instructions = list(dis.get_instructions(code)) + # This compact fixture needs no extended operands. NOPs + # reproduce the padded return layout, preserving the return's value. + assert not any(item.opname == 'EXTENDED_ARG' for item in instructions) + starts, sizes, cursor = {}, {}, 0 + for index, item in enumerate(instructions): + starts[item.offset] = cursor + end = instructions[index + 1].offset if index + 1 < len(instructions) else len(code.co_code) + sizes[item.offset] = end - item.offset + cursor += sizes[item.offset] + (2 * count if item.opname == 'RETURN_VALUE' else 0) + starts[len(code.co_code)] = cursor + output = bytearray() + for item in instructions: + if item.opname == 'RETURN_VALUE': + output.extend(bytes((dis.opmap['NOP'], 0)) * count) + actual = len(output) + raw = bytearray(code.co_code[item.offset:item.offset + sizes[item.offset]]) + if item.opcode in getattr(dis, "hasjump", dis.hasjrel + dis.hasjabs): + delta = (starts[item.argval] - (actual + sizes[item.offset])) // 2 + operand = -delta if 'BACKWARD' in item.opname else delta + assert 0 <= operand <= 255 + raw[1] = operand + output.extend(raw) + def varint(value, marker=False): + digits = [value & 63] + while value >> 6: + value >>= 6 + digits.append(value & 63) + digits.reverse() + for index in range(len(digits) - 1): + digits[index] |= 64 + if marker: + digits[0] |= 128 + return bytes(digits) + table = b''.join(varint(starts[row.start] // 2, True) + + varint((starts[row.end] - starts[row.start]) // 2) + + varint(starts[row.target] // 2) + + varint((row.depth << 1) | row.lasti) + for row in dis.Bytecode(code).exception_entries) + padded = code.replace(co_code=bytes(output), co_exceptiontable=table, co_linetable=b'') + return padded + + +if __name__ == "__main__": + if not (3, 11) <= sys.version_info[:2] <= (3, 14): + raise SystemExit("Use CPython 3.11, 3.12, 3.13 or 3.14") + root = Path(__file__).resolve().parents[1] + source_path = Path("tests/fixtures/handler_return_padding.py") + source = (root / source_path).read_bytes() + module = compile(source, source_path.as_posix(), "exec", dont_inherit=True) + version = f"{sys.version_info.major}.{sys.version_info.minor}" + header = (importlib.util.MAGIC_NUMBER + (3).to_bytes(4, "little") + + importlib.util.source_hash(source)) + for count in (1, 3, 9): + padded = module.replace(co_consts=tuple( + pad_return_values(value, count) if isinstance(value, types.CodeType) else value + for value in module.co_consts)) + path = root / f"tests/pyc/handler_return_padding_{count}.{version}.pyc" + path.write_bytes(header + marshal.dumps(padded)) From 8062ead94e83bcf641cd91f46372d4e750ab5992 Mon Sep 17 00:00:00 2001 From: Francois Chagnon Date: Tue, 13 Oct 2026 18:32:49 -0400 Subject: [PATCH 2/2] Add harness coverage for the reconstruction fix --- .../cases/b_unpacked_comprehension_cleanup.py | 33 ++++++++++++++++++ .../3.11.14/handler_return_padding_3.py | 15 ++++++++ .../3.11.14/handler_return_padding_3.pyc | Bin 0 -> 542 bytes .../corpus/3.12.3/handler_return_padding_3.py | 15 ++++++++ .../3.12.3/handler_return_padding_3.pyc | Bin 0 -> 516 bytes .../corpus/3.13.7/handler_return_padding_3.py | 15 ++++++++ .../3.13.7/handler_return_padding_3.pyc | Bin 0 -> 520 bytes .../corpus/3.14.2/handler_return_padding_3.py | 15 ++++++++ .../3.14.2/handler_return_padding_3.pyc | Bin 0 -> 557 bytes tools/compile_padded_returns.py | 7 ++++ 10 files changed, 100 insertions(+) create mode 100644 tests/behavior/cases/b_unpacked_comprehension_cleanup.py create mode 100644 tests/corpus/3.11.14/handler_return_padding_3.py create mode 100644 tests/corpus/3.11.14/handler_return_padding_3.pyc create mode 100644 tests/corpus/3.12.3/handler_return_padding_3.py create mode 100644 tests/corpus/3.12.3/handler_return_padding_3.pyc create mode 100644 tests/corpus/3.13.7/handler_return_padding_3.py create mode 100644 tests/corpus/3.13.7/handler_return_padding_3.pyc create mode 100644 tests/corpus/3.14.2/handler_return_padding_3.py create mode 100644 tests/corpus/3.14.2/handler_return_padding_3.pyc diff --git a/tests/behavior/cases/b_unpacked_comprehension_cleanup.py b/tests/behavior/cases/b_unpacked_comprehension_cleanup.py new file mode 100644 index 00000000..2b30f9b5 --- /dev/null +++ b/tests/behavior/cases/b_unpacked_comprehension_cleanup.py @@ -0,0 +1,33 @@ +# MIN_VERSION: 3.12 + + +def unpack4(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [a + b + c + d for rows in groups for a, b, c, d in rows] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack_list(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c) for a, b in groups for c, d in b] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +def unpack6(groups): + a, b, c, d, e, f = 10, 20, 30, 40, 50, 60 + result = [(a, c, e) for a, b in groups for c, d in b for e, f in d] + after = 0 + for number in (1, 2, 3): + after += number + return result, (a, b, c, d, e, f), after + + +print(unpack4([[], [(1, 2, 3, 4)], [(5, 6, 7, 8)]])) +print(unpack_list([(1, [(3, 4), (5, 6)]), (7, [])])) +print(unpack6([(1, [(3, [(5, 6), (7, 8)])]), (9, [])])) diff --git a/tests/corpus/3.11.14/handler_return_padding_3.py b/tests/corpus/3.11.14/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.11.14/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.11.14/handler_return_padding_3.pyc b/tests/corpus/3.11.14/handler_return_padding_3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff2d6ead4fb36b4b5b6052154b74ad698692a636 GIT binary patch literal 542 zcmZWlO-lk%6us{~#~RTJ!G#7gP>bB;ABez(Z3-a_!8S7ZUN$t%$eS^Zw2jhVaNEid z|ADRC_Y2-G`U8SHSKaZes0a7Vxtw$E9qwB!7GVG|9w#S>%Ztx12O=fiWJy8mAc8f<Y5S`i0CFaZ-KUxg|Elgud8xd0|f?y>=Y{HRUHF)O2-o@nb3sl5Hv@?Zb zVHbjkrIn=iA8hLnP)KLxTs-2z&b~ME<_$adVOb^zfK%&q>g)Y#^^@@6!wB_E0xb}5 z0b3AYj+NO!17Hw7r_+0)um(uVBrrIFP9!>~&O;h<$6Nf;Fi1gF6xfOU5dvc#uB~bC zqYi)>P{Waq{p*SlaDOnHG!w_H?-e^@ohXk`q3-O+{&%U-y* zY&QEd>r2n~^4pcBUsux0@7h)2NL5mjc(qisg{V|_=4ydX{FQ_nq1JG3z3{=_5`{A? zr18Sczzt5uZc`b!4-*fVfyc2zioY;ouS101l7$5CGJ|+GVNjx*;)K`R*mrG%N|T+k eDEtg9?7O1w$VD|yG+cN8SAKIu?3pHx_O-u(Ty8f2 literal 0 HcmV?d00001 diff --git a/tests/corpus/3.13.7/handler_return_padding_3.py b/tests/corpus/3.13.7/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.13.7/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.13.7/handler_return_padding_3.pyc b/tests/corpus/3.13.7/handler_return_padding_3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f68db615df47527485ea117ad3b75cf5ceaa0f2 GIT binary patch literal 520 zcmYLGJx{|h5WTaTLPA@7+ZU76Ae__RBO_HO<&iqMiY(F^~VAF1oP8OW@)rcX61lYa?)N@i!< zwqEPXs2gp#?W=B2$!MeH1j3iFtY9^nAIi7KkPzAAjdJN5t5&aBi&q0a_ks*#H0l literal 0 HcmV?d00001 diff --git a/tests/corpus/3.14.2/handler_return_padding_3.py b/tests/corpus/3.14.2/handler_return_padding_3.py new file mode 100644 index 00000000..7e7278b7 --- /dev/null +++ b/tests/corpus/3.14.2/handler_return_padding_3.py @@ -0,0 +1,15 @@ +# The .pyc variants insert 1, 3 or 9 NOPs before each RETURN_VALUE. +# Regenerate with the matching CPython: python tools/compile_padded_returns.py +# Jump targets and exception-table boundaries are relocated with the padding. + +async def handled(read, log): + try: + await read() + log("saved") + except ValueError: + log("locked") + return False + except Exception as error: + log("failed", error) + return False + return True diff --git a/tests/corpus/3.14.2/handler_return_padding_3.pyc b/tests/corpus/3.14.2/handler_return_padding_3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f1c0628e0c747b19ff2b319b8e85c527ee0669d GIT binary patch literal 557 zcmYL`!AiqG5Qb;6O|(I4)kB50G_4m!&`UijN(J>I9!lv+kkDN<8k@37r5*%P&-wt0 zinl(1Ab6^eu~%O}s1M+5jWw{tKQsR~yG#~R<2D0;x5tP4#nVRdll1VzSoOz9TtXEx z5J44ZVcuvMLl=Na`kGGvgF*=kbCIliF)RWnOW z7G_}tb|xLzi%9?rcp4x_OieS!Ka+CR@Nk)V*tK?*8M%jGV!E0SZ*>%@%i%ls5sgj$WhG<1z(41!~ zA%)8o^MgZW3f&I%W*x^-bCC$6aOuG7L~SX;r6ae=eIfUypuD-?ayj>!hl{OqHS*iy zGDWD~YhN!S+@^4al>*+_CAdW=aW__idy@*!Sb~@7N{+o+l)aCV>LpM|Gg-;7UMflD wUdGgw93#Ek#li3{$O)P%$0?}M)zg5teX%B|DAj+ZZ&UbUi1=;Wm>zh40DbssH~;_u literal 0 HcmV?d00001 diff --git a/tools/compile_padded_returns.py b/tools/compile_padded_returns.py index 6da196c4..bdb472d9 100644 --- a/tools/compile_padded_returns.py +++ b/tools/compile_padded_returns.py @@ -72,3 +72,10 @@ def varint(value, marker=False): for value in module.co_consts)) path = root / f"tests/pyc/handler_return_padding_{count}.{version}.pyc" path.write_bytes(header + marshal.dumps(padded)) + if count == 3: + # Also exercise a representative padded input in verify_corpus.py. + # Keep the directory named after the interpreter producing the pyc. + corpus = root / "tests/corpus" / ".".join(map(str, sys.version_info[:3])) + corpus.mkdir(parents=True, exist_ok=True) + (corpus / "handler_return_padding_3.py").write_bytes(source) + (corpus / "handler_return_padding_3.pyc").write_bytes(header + marshal.dumps(padded))