Skip to content

Recognize modern loop back edges at EXTENDED_ARG prefixes #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
EiNSTeiN- wants to merge 3 commits into from
Closed

Recognize modern loop back edges at EXTENDED_ARG prefixes #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
85cc105
Recognize modern loop back edges at EXTENDED_ARG prefixes
EiNSTeiN- Oct 13, 2026
eeecb35
Add harness coverage for the reconstruction fix
EiNSTeiN- Oct 13, 2026
ae75257
Preserve prefixed loop back edges in conditional branches
EiNSTeiN- Oct 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 34 additions & 9 deletions src/decompiler.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23192,16 +23192,21 @@ impl<'a> Ctx<'a> {
fn is_loop_top_prefix(&self, t: usize, start: usize) -> bool {
// 3.11+ folds inline CACHE entries into each instruction's size,
// so offsets inside a cached instruction have no idx_of entry
// either - a back edge landing <=8 bytes before the loop top
// would falsely match a CACHE tail (compileall 3.13 main tore
// into an INCOMPLETE stub). The EXTENDED_ARG-prefix shape this
// recognizes is pre-3.11 wordcode only (no CACHE gaps).
if self.version.at_least(3, 11) {
return false;
}
// either. Absence from the instruction map cannot distinguish
// a CACHE tail from an EXTENDED_ARG prefix: modern offsets need
// proof from the original bytes.
if t >= start || start - t > 8 {
return false;
}
if self.version.at_least(3, 11) {
if t % 2 != 0 || start % 2 != 0 {
return false;
}
if !(t..start).step_by(2).all(|offset| self.code.code.get(offset)
.map_or(false, |byte| self.table.op(*byte) == Op::EXTENDED_ARG)) {
return false;
}
}
// no instruction STARTS at t (idx_of also maps END offsets: the
// preceding instruction's end lands on t and must not count as
// an instruction start)
Expand Down Expand Up @@ -33228,14 +33233,21 @@ if split_cond {
// `if c: continue`-style: fall through to the
// arm below that opens an If ending at that back
// edge instead of closing the loop here.
// A prefixed loop top has the same identity here
// as when matching the conditional above. Otherwise
// a later edge to EXTENDED_ARG is missed and a
// conditional continue prematurely closes the loop.
let loop_start0 = self.blocks[i].start;
let has_later_back_edge = self
.idx_of
.get(&self.cur_offset)
.map_or(false, |&ci0| {
self.instrs.iter().skip(ci0 + 1).any(|ins| {
ins.is_backward
&& ins.target == Some(loop_start0)
&& ins.target.map_or(false, |t| {
t == loop_start0
|| self.is_loop_top_prefix(t, loop_start0)
})
&& matches!(
ins.op,
Op::JUMP_ABSOLUTE
Expand Down Expand Up @@ -33415,8 +33427,21 @@ if split_cond {
}
return;
}
// A forward conditional can skip this back
// edge and land on the next one. Keep the
// skipped edge inside its fall-through arm;
// ending before it moves `continue` outside
// the guard and changes the recompiled jumps.
let body_end = if self.version.at_least(3, 11)
&& raw_target == inst.end()
&& self.is_loop_top_prefix(target, loop_start)
{
inst.end()
} else {
inst.start
};
let mut blk =
Block::new(BlockType::If, self.cur_next, inst.start);
Block::new(BlockType::If, self.cur_next, body_end);
let c = if jump_if_true {
negate_cond(cond.clone())
} else {
Expand Down
63 changes: 63 additions & 0 deletions tests/behavior/cases/b_conditional_loop_prefix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# MIN_VERSION: 3.11

# Attribute loads make FOR_ITER require EXTENDED_ARG on Python 3.12+.
# Conditional branches lead to adjacent back edges targeting that prefix.
def process(rows, work):
success = True
for row in rows:
if row.is_file:
if not work(
"file", row.ok,
row.a, row.b, row.c, row.d, row.e, row.f,
row.g, row.h, row.i, row.j, row.k, row.l,
):
success = False
else:
if not work(
"directory", row.ok,
row.a, row.b, row.c, row.d, row.e, row.f,
row.g, row.h, row.i, row.j, row.k, row.l,
):
success = False
return success


class Row:
a = 1
b = 2
c = 3
d = 4
e = 5
f = 6
g = 7
h = 8
i = 9
j = 10
k = 11
l = 12

def __init__(self, is_file, ok):
self.is_file = is_file
self.ok = ok


def work(kind, ok, *values):
calls.append((kind, ok, values))
return ok


for outcomes in [
[],
[(True, True)],
[(True, False)],
[(False, True)],
[(False, False)],
[(True, True), (False, True)],
[(False, True), (True, True)],
[(True, False), (False, True), (True, True)],
[(False, False), (True, True), (False, True)],
]:
calls = []
rows = [Row(is_file, ok) for is_file, ok in outcomes]
print(process(rows, work))
print(calls)
50 changes: 50 additions & 0 deletions tests/behavior/cases/b_extended_loop_prefix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# MIN_VERSION: 3.11


def size_30(rows, work):
result = []
for row in rows:
try:
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
except ValueError:
continue
result.append(row)
return result


calls = []
def work(row):
calls.append(row)
if row == 2:
raise ValueError(row)
print(size_30([1, 2, 3], work))
print(calls)
20 changes: 20 additions & 0 deletions tests/fixtures/conditional_loop_prefix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Attribute loads make FOR_ITER require EXTENDED_ARG on Python 3.12+.
# Conditional branches lead to adjacent back edges targeting that prefix.
def process(rows, work):
success = True
for row in rows:
if row.is_file:
if not work(
"file", row.ok,
row.a, row.b, row.c, row.d, row.e, row.f,
row.g, row.h, row.i, row.j, row.k, row.l,
):
success = False
else:
if not work(
"directory", row.ok,
row.a, row.b, row.c, row.d, row.e, row.f,
row.g, row.h, row.i, row.j, row.k, row.l,
):
success = False
return success
123 changes: 123 additions & 0 deletions tests/fixtures/extended_loop_prefix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
# Versioned .pyc files are consumed by tests/integration.rs.

# The long functions require EXTENDED_ARG before FOR_ITER.

def size_1(rows, work):
result = []
for row in rows:
try:
work(row)
except ValueError:
continue
result.append(row)
return result


def size_30(rows, work):
result = []
for row in rows:
try:
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
except ValueError:
continue
result.append(row)
return result


def size_60(rows, work):
result = []
for row in rows:
try:
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
work(row)
except ValueError:
continue
result.append(row)
return result
42 changes: 42 additions & 0 deletions tests/integration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,3 +169,45 @@ fn py314_hello() {
"py3.14 hello",
);
}

#[test]
fn extended_loop_prefix_preserves_handler_continue() {
for version in ["3.11", "3.12", "3.13", "3.14"] {
let (src, clean) = decompile_fixture(&format!("extended_loop_prefix.{version}.pyc"));
assert!(clean, "Python {version}: output marked incomplete:\n{src}");
for count in [1, 30, 60] {
let body = " work(row)\n".repeat(count);
let expected = format!(
"def size_{count}(rows, work):\n result = []\n for row in rows:\n try:\n{body} except ValueError:\n continue\n result.append(row)");
assert_contains(&src, &[&expected], version);
}
}
}

#[test]
fn conditional_loop_prefix_keeps_both_arms_inside_loop() {
for version in ["3.11", "3.12", "3.13", "3.14"] {
let (src, clean) = decompile_fixture(&format!("conditional_loop_prefix.{version}.pyc"));
assert!(clean, "Python {version}: output marked incomplete:\n{src}");
let args = "row.ok, row.a, row.b, row.c, row.d, row.e, row.f, row.g, row.h, row.i, row.j, row.k, row.l";
let file_condition = if version == "3.12" { "" } else { "not " };
let directory_condition = if version == "3.11" { "elif not" } else { "if" };
assert_contains(
&src,
&[
&format!(" for row in rows:\n if row.is_file:\n if {file_condition}work('file', {args}):"),
&format!(" {directory_condition} work('directory', {args}):"),
" return success",
],
version,
);
assert!(!src.contains("\n success = False"), "Python {version}: failure assignment escaped the loop:\n{src}");
if version == "3.13" || version == "3.14" {
assert_contains(
&src,
&[" success = False\n continue\n continue"],
version,
);
}
}
}
Binary file added tests/pyc/conditional_loop_prefix.3.11.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/conditional_loop_prefix.3.12.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/conditional_loop_prefix.3.13.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/conditional_loop_prefix.3.14.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/extended_loop_prefix.3.11.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/extended_loop_prefix.3.12.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/extended_loop_prefix.3.13.pyc
View file
Open in desktop
Binary file not shown.
Binary file added tests/pyc/extended_loop_prefix.3.14.pyc
View file
Open in desktop
Binary file not shown.