diff --git a/.config/nextest.toml b/.config/nextest.toml index 223bee2865..508a7a42ac 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -7,13 +7,13 @@ serial = { max-threads = 1 } filter = 'binary(file_watching)' test-group = 'serial' -# the runtime tests that need something of their interpreter — an old version -# with `typing_extensions`, a new one, the `regex` package — run on one found to -# have it, which a machine often does not. each says which interpreter it ran -# on, or that it found none and ran nothing, and a passing test's output is -# otherwise hidden, so a skip would read as a pass +# the transpiler's runtime tests transpile for the interpreter they run on. a few +# programs need a newer python than the one found, or more of it — +# `typing_extensions`, the `regex` package — which a machine often does not have. +# each says which interpreter it ran on, or that it found none and ran nothing, and +# a passing test's output is otherwise hidden, so a skip would read as a pass [[profile.default.overrides]] -filter = 'binary(polyfill_runtime) | binary(grapheme_runtime) | binary(conversion_runtime) | binary(lazy_import_runtime) | binary(fstring_debug_runtime) | binary(tstring_expression_runtime) | binary(visibility_runtime) | binary(data_class_runtime) | binary(builtin_names_runtime) | binary(inferred_annotation_runtime) | binary(forward_reference_runtime) | binary(imported_extension_runtime) | binary(imported_conversion_class_runtime)' +filter = 'package(by_transforms) & binary(/_runtime$/)' success-output = "final" [profile.ci] diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d8dc0c44d5..d890739b72 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -368,10 +368,12 @@ jobs: # to answer exactly as that interpreter does, so *which* interpreter is a first-class # variable and belongs in the workflow rather than in whatever a runner image ships. # - # the supported set is python 3.13 and 3.14, each in its gil and its free-threaded - # build. `PYTHON_VERSION` is the primary and `cargo test (linux)` runs the whole suite - # against it; this job covers the other three. 3.12 is deliberately absent — it is - # below the project's floor, and it was only ever reached because an image shipped it. + # the supported set is python 3.11 through 3.14, and the free-threaded builds of 3.13 + # and 3.14 — free threading starts at 3.13, so 3.11 and 3.12 have gil builds only. + # `PYTHON_VERSION` is the primary and `cargo test (linux)` runs the whole suite against + # it; this job covers the other five. 3.11 is the floor `by_build::MINIMUM_PYTHON` + # states, and 3.11 and 3.12 answer differently from 3.13 in places the backend follows + # per version, so leaving them out would leave those branches untested. # # only the version-sensitive targets run here, because only they start an interpreter: # * the six backend crates, which emit and then load real extension modules @@ -394,6 +396,15 @@ jobs: fail-fast: false matrix: include: + # every leg runs every step. a frontend test whose program needs a newer python + # than the leg's says so and skips, by name: a runtime test transpiles for a fixed + # version and runs on an interpreter at least that new, a reified generic is + # refused below 3.12 and its pep 696 defaults below 3.13, and the divergence + # harness lists the sections that need either + - python: "3.11" + backend_filter: "all()" + - python: "3.12" + backend_filter: "all()" - python: "3.13" backend_filter: "all()" # the free-threaded builds run the same tests, minus one that cannot be @@ -478,6 +489,14 @@ jobs: cargo nextest run \ -p by_build -p by_ir -p by_irbuild -p by_opt -p by_codegen_c -p by_rt \ -E "$BACKEND_FILTER" + # python transpiled for a version below the newest takes the typing names that version + # lacks from `typing_extensions` — `override` below 3.12, say — so a program built for + # the leg's version needs it installed, as a project targeting that version would. it + # goes in after the backend has run, which compiles against the interpreter as uv ships + # it. the pin is the one the dev dependencies lock, and uv marks its managed + # interpreters as externally managed, though this one belongs to the job alone + - name: "Install `typing_extensions` into the interpreter" + run: uv pip install --python "$PYTHON" --break-system-packages "typing_extensions==4.16.0" # a separate invocation on purpose: the backend's differential tests fork processes # and compile C, and sharing a run with `by_transforms`' ~1800 other tests starves # them into failures that pass on their own diff --git a/crates/by_build/src/lib.rs b/crates/by_build/src/lib.rs index af4ec152bd..5c5cd11992 100644 --- a/crates/by_build/src/lib.rs +++ b/crates/by_build/src/lib.rs @@ -325,13 +325,37 @@ fn lower( toolchain: Option<&Toolchain>, ) -> Result { finish( - by_irbuild::module_from_source(source, module_name, options.lowering()), + by_irbuild::module_from_source_at( + source, + module_name, + options.lowering(), + fallback_config(options, toolchain).min_version, + ), source, options, toolchain, ) } +/// the config the interpreted fallback is transpiled with, and the version both halves of +/// the module are analysed at +/// +/// the version has to be the *interpreter's*, because the fallback runs inside the +/// extension at import time — emitting syntax the interpreter cannot parse makes the whole +/// module fail to load, taking every function with it. the compiled half is analysed at +/// the same one: a `sys.version_info` test settles which branch binds a name, and analysed +/// at a newer python the module laid a value out as the branch the interpreter never takes +/// binds it, then refused the value the interpreter's own branch bound +fn fallback_config(options: &Options, toolchain: Option<&Toolchain>) -> by_transforms::Config { + let mut config = options.fallback.clone().unwrap_or_default(); + if let Some((major, minor)) = toolchain.and_then(|toolchain| toolchain.version) + && let Ok(parsed) = format!("{major}.{minor}").parse() + { + config.min_version = parsed; + } + config +} + /// apply the gates, optimize, and attach the interpreted fallback /// /// split out from [`lower`] because a whole-project build lowers against a real @@ -395,25 +419,21 @@ fn finish( // `JustFloat = float` locally instead of emitting a // `from ty_extensions import ...` that has no module behind it // - // the version has to be the *interpreter's*, because this python runs inside - // the extension at import time — emitting syntax the interpreter cannot parse - // makes the whole module fail to load, taking every function with it - // a `.py` source is already what runs: it is its own fallback + // the module is analysed at the interpreter's version, as the compiled half is: a + // lowering that asks what a name is, such as the conversion a `frozendict` + // annotation makes on 3.15, has to be answered by the interpreter's builtins rather + // than by ty's newest python's. a `.py` source is already what runs: it is its own + // fallback let twin = if options.language == by_irbuild::Language::Python { source.to_string() } else { - let mut config = options.fallback.clone().unwrap_or_default(); + let mut config = fallback_config(options, toolchain); // the compiled functions read the module's globals by name, and a deferred // `from` import is in them only once something has read it config.defer_from_imports = false; - if let Some((major, minor)) = toolchain.and_then(|toolchain| toolchain.version) - && let Ok(parsed) = format!("{major}.{minor}").parse() - { - config.min_version = parsed; - } - by_transforms::transpile(source, &config).map_err(|error| { - anyhow::anyhow!("could not transpile for the interpreted fallback: {error}") - })? + by_transforms::transpile_analysed_at(source, &config, config.min_version).map_err( + |error| anyhow::anyhow!("could not transpile for the interpreted fallback: {error}"), + )? }; // a decorator module init applies to the native definition would otherwise run here // too, over the twin's — once for each definition rather than once for the name @@ -660,4 +680,26 @@ mod tests { assert!(error.to_string().contains("not well-formed")); assert!(!dir.exists(), "nothing is written for ill-formed IR"); } + + /// `frozendict` is a builtin from 3.15, past ty's newest python, so the conversion its + /// annotation makes is only written where the module is analysed at the interpreter's + /// version. no interpreter is run: the one named is only asked to compile the twin, + /// and an artefact without that runs the source + #[test] + fn the_twin_is_analysed_at_the_interpreter_s_version() { + let toolchain = Toolchain::from_probe( + "by-build-no-such-python", + r#"{"cc": ["cc"], "include": [], "libdir": [], "ext_suffix": ".so", "platform": "linux", "version": "3.15"}"#, + ) + .unwrap(); + let module = lower( + "a: frozendict[str, str] = {}\n", + "twin_version", + &Options::default(), + Some(&toolchain), + ) + .unwrap(); + let twin = module.fallback_source.unwrap_or_default(); + assert!(twin.contains("frozendict()"), "{twin}"); + } } diff --git a/crates/by_build/tests/differential.rs b/crates/by_build/tests/differential.rs index 17c82bc717..4149b34799 100644 --- a/crates/by_build/tests/differential.rs +++ b/crates/by_build/tests/differential.rs @@ -155,6 +155,26 @@ def _warned_into_safely(module, fn, *args): registry = module.__dict__.get('__warningregistry__', {}) return (raised, sorted(key[0] for key in registry if isinstance(key, tuple))) +# what a call left unraisable with every warning turned into an error: a warning issued +# while an object is finalized has nowhere to raise, and is reported to +# `sys.unraisablehook` instead. the object it is reported against and the context python +# writes around that object's repr both name the type, which the two legs spell +# differently, so they are reduced to whether there is one +def _unraisable_from(fn, *args): + import sys + seen = [] + saved = sys.unraisablehook + sys.unraisablehook = lambda u: seen.append(( + type(u.exc_value).__name__, str(u.exc_value), + u.err_msg and u.err_msg.split(' <', 1)[0], u.object is None)) + try: + with warnings.catch_warnings(): + warnings.simplefilter('error') + fn(*args) + finally: + sys.unraisablehook = saved + return seen + # what a warning carried as its `source`, which is the object `tracemalloc` hangs an # allocation traceback off. it reaches the record and nothing else, so it is asked for # separately — and without the file name, since a declined function's frames are named @@ -955,6 +975,30 @@ def _deepest(fn, limit): finally: sys.setrecursionlimit(saved) +# whether `fn(n)` answers `n` at the deepest `n` a python recursion of one frame a level +# reaches from here, asked once. `_deepest` asks at every depth below the limit first, +# so a frame `fn` takes only once on the way in is the whole of what this can see +def _at_the_limit(fn, limit): + import sys + def model(n): + return 0 if n == 0 else model(n - 1) + 1 + saved = sys.getrecursionlimit() + sys.setrecursionlimit(limit) + try: + n = 0 + while True: + try: + model(n) + except RecursionError: + break + n += 1 + try: + return fn(n - 1) == n - 1 + except RecursionError: + return 'RecursionError' + finally: + sys.setrecursionlimit(saved) + # a code object `fn.__code__` can be swapped for, answering `value` however it is called. # python refuses a code object whose free variables do not match the function's, and a # compiled function is published as a forwarder closing over its native entry — so the @@ -1567,8 +1611,9 @@ fn agree_built( // the interpreted leg: for basedpython, the transpiler's own output run by // cpython, under the same config `by_build` uses — including the *target - // version*, which has to be this interpreter's or the two legs are not the - // same program. for python there is nothing to transpile — it already is one + // version*, which has to be this interpreter's and is the version the module is + // analysed at, or the two legs are not the same program. for python there is + // nothing to transpile — it already is one let fallback = config.clone(); let interpreted_source = match language { by_irbuild::Language::BasedPython => { @@ -1578,7 +1623,8 @@ fn agree_built( { config.min_version = parsed; } - by_transforms::transpile(source, &config).expect("the source transpiles") + by_transforms::transpile_analysed_at(source, &config, config.min_version) + .expect("the source transpiles") } by_irbuild::Language::Python => source.to_string(), }; @@ -1630,6 +1676,51 @@ fn agree_built( Some(compiled_dir) } +/// the interpreted fallback is analysed at the version of the interpreter it is built for. +/// `frozendict` is a builtin from 3.15, past ty's newest python, and the module body that +/// binds `a` runs from the fallback: analysed at ty's newest python it took `frozendict` for +/// a name nothing defines and left the display a `dict` +#[test] +fn the_fallback_converts_as_the_interpreter_s_version_does() { + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 15))) { + return; + } + agree( + "fallbackversion", + "\ +a: frozendict[str, str] = {} + +def kind() -> str: + return type(a).__name__ +", + &["m.kind()", "type(m.a).__name__"], + ); +} + +/// the compiled half is analysed at the version of the interpreter it is built for, as the +/// fallback is. analysed at ty's newest python, the test below took the first branch on +/// every interpreter, laid `LIMIT` out as a `float`, and an older interpreter's `int` was +/// refused at the read where python doubled it +#[test] +fn the_compiled_half_is_analysed_at_the_interpreter_s_version() { + agree_python( + "compiledversion", + "\ +import sys + +if sys.version_info >= (3, 14): + LIMIT = 1.5 +else: + LIMIT = 2 + + +def doubled() -> object: + return LIMIT * 2 +", + &["m.doubled()", "type(m.LIMIT).__name__"], + ); +} + #[test] fn integer_arithmetic_agrees() { agree( @@ -2941,6 +3032,204 @@ fn a_native_recursion_follows_the_recursion_limit() { ); } +#[test] +fn a_native_recursion_entered_through_its_forwarder_follows_the_recursion_limit() { + // the forwarder a module function is published as is the python frame python's own + // function would have pushed, and the native call behind it takes no frame of its + // own. python 3.11 counts a call into a builtin against the recursion limit, so there + // that call took a second frame and the recursion stopped a level early + agree_python( + "reclimitentry", + RECURSIONS, + &[ + "_at_the_limit(m.depth, 100)", + "_at_the_limit(m.ping, 100)", + "_at_the_limit(m.entered, 100)", + ], + ); +} + +#[test] +fn a_published_new_or_init_subclass_follows_the_recursion_limit() { + // a written `__new__` and the `__init_subclass__` a class extended in its module carries + // are published as builtins over the compiled entries. python 3.11 counts a call into a + // builtin against the recursion limit, and the body behind it counts the frame python's + // own would, so there a construction or a subclass made near the limit raised a level + // early + let Some(compiled) = agree_python_built( + "reclimitnew", + "\ +class Made: + def __new__(cls, v: int) -> 'Made': + return object.__new__(cls) + + def __init__(self, v: int) -> None: + self.v = v + + +class MadeLeaf(Made): + pass + + +class Root: + def __init__(self, n: int) -> None: + self.count = n + + +class Leaf(Root): + pass +", + &[ + "(lambda nest: (_deepest(lambda n: nest(n, lambda: m.Made(1)), 100), \ + _deepest(lambda n: nest(n, lambda: type('Sub', (m.Made,), {})(1)), 100), \ + _deepest(lambda n: nest(n, lambda: type('Sub', (m.Root,), {})), 100))) \ + (lambda n, act: (lambda f: f(f, n))(lambda f, k: f(f, k - 1) if k else act()))", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_reclimitnew as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Made)['__new__']).__name__, type(vars(m.Root)['__init_subclass__']).__name__)\n", + ), + "staticmethod classmethod" + ); +} + +/// what a function defined in the module under test would be, for a recursion that has to +/// take exactly one python frame a level, as `_at_the_limit` counts them +const DEFINED_AGAINST: &str = + "(lambda source: (g := dict(globals(), m=m), exec(source, g), g['result'])[2])"; + +#[test] +fn a_construction_from_python_counts_the_frame_of_the_init_it_runs() { + // python runs a class's `__init__` in a frame of its own, the one `@dataclass` generates + // included, and a compiled class's `tp_init` counted none: a recursion that went round + // through a construction reached a level deeper for every one, and a construction at + // the limit succeeded where python's raises. a keyword argument keeps 3.13 and later + // from specialising python's construction, which takes a second frame — see + // runtime.md#how-deep-a-recursion-goes + let at_the_limit = |construct: &str| { + format!( + "{DEFINED_AGAINST}(\"\"\" +def d(n): + return d(n - 1) + 1 if n else ({construct}, 0)[1] +result = _at_the_limit(d, 100) +\"\"\")" + ) + }; + let Some(compiled) = agree_in( + "recinit", + "\ +from collections.abc import Callable + + +class Node: + def __init__(self, f: Callable[[int], int], n: int) -> None: + self.v = f(n) + + +class Pair: + def __init__(self, a: int) -> None: + self.a = a + + +data class Point: + x: int + y: int +", + &[ + "(lambda f: _deepest(lambda n: f(f, n), 100))(lambda f, k: 0 if k == 0 else m.Node(lambda j: f(f, j), n=k - 1).v + 1)", + &at_the_limit("m.Node(lambda j: j, 1)"), + &at_the_limit("m.Pair(1)"), + &at_the_limit("m.Point(1, 2)"), + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_recinit as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(*(type(vars(c)['__init__']).__name__ for c in (m.Node, m.Pair, m.Point)))\n", + ), + "wrapper_descriptor wrapper_descriptor wrapper_descriptor" + ); +} + +#[test] +fn a_construction_compiled_code_makes_counts_as_python_s_does() { + // a compiled `__init__` that builds another instance of its class goes round a cycle + // of compiled calls, and each round is python's call of the type and then the frame of + // the `__init__` that call runs. 3.11 counts both against the limit, and the compiled + // construction counted only the frame, so the recursion went twice as deep there. + // + // from 3.13 python specialises the construction inside `__init__`, which takes a second + // frame a level once the call site is warm — see runtime.md#how-deep-a-recursion-goes. + // a metaclass is one thing that keeps it from doing so, and a compiled class with one + // declines, so the depth is held to the same class written with one in python: a + // recursion of the construction python makes when it does not specialise + let source = "\ +class Node: + def __init__(self, n: int) -> None: + self.child = Node(n - 1) if n else None +"; + let Some(compiled) = agree_python_built( + "recconstruct", + source, + &[ + "(lambda node: (node.child.child.child.child is None, type(node.child).__name__))(m.Node(3))", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + &format!( + "import sys\n\ + import by_diff_recconstruct as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Node)['__init__']).__name__)\n\ + class Unspecialised(type):\n\ + \x20 pass\n\ + namespace = {{'Unspecialised': Unspecialised}}\n\ + exec({source:?}.replace('class Node:', 'class Node(metaclass=Unspecialised):'), \ + namespace)\n\ + def deepest(construct):\n\ + \x20 sys.setrecursionlimit(100)\n\ + \x20 n = 0\n\ + \x20 while True:\n\ + \x20 try:\n\ + \x20 construct(n=n)\n\ + \x20 except RecursionError:\n\ + \x20 return n - 1\n\ + \x20 n += 1\n\ + print(deepest(m.Node) == deepest(namespace['Node']))\n" + ), + ), + "wrapper_descriptor\nTrue" + ); +} + #[test] fn a_recursive_body_keeps_the_copy_it_is_versioned_against_in_a_function_of_its_own() { // a function versioned at its entry holds two copies of itself: the one the test says @@ -2987,11 +3276,17 @@ def outer(n: int) -> int: let c = std::fs::read_to_string(compiled.join("by_diff_recsplit.c")) .expect("the emitted C is kept"); assert!( - c.contains("__attribute__((noinline)) static ByTagged byu_by_diff_recsplit_fib(ByDepth by_depth, ByTagged r0) {"), + c.contains(&format!( + "__attribute__((noinline)) static ByTagged {}(ByDepth by_depth, ByTagged r0) {{", + symbol("byu", &["by_diff_recsplit", "fib"], "") + )), "the copy the test says no to is not a C function of its own:\n{c}" ); assert!( - c.contains("else return byu_by_diff_recsplit_fib(by_depth, r0);"), + c.contains(&format!( + "else return {}(by_depth, r0);", + symbol("byu", &["by_diff_recsplit", "fib"], "") + )), "the test does not reach it by an uncounted call:\n{c}" ); } @@ -7439,6 +7734,99 @@ def describe() -> str: ); } +#[test] +fn a_function_the_module_body_writes_over_through_its_namespace_keeps_the_write() { + // the module body runs before init publishes anything, and a write it makes through + // the namespace object rather than by name is not something the source says: a + // subscript or an `update` on `globals()`, a helper handed `globals()`, an `exec`. + // each leaves the name holding what was written, as python does, and a compiled + // caller calls through it. `kept` is written over by nothing, and is the compiled one + let Some(compiled) = agree_python_built( + "nswrite", + "\ +def by_subscript(n: int) -> int: + return n + 1 + + +def by_update(n: int) -> int: + return n + 1 + + +def by_helper(n: int) -> int: + return n + 1 + + +def by_exec(n: int) -> int: + return n + 1 + + +def kept(n: int) -> int: + return n + 1 + + +def call_all(n: int) -> list[int]: + return [by_subscript(n), by_update(n), by_helper(n), by_exec(n), kept(n)] + + +def rebind(namespace: dict[str, object]) -> None: + namespace['by_helper'] = lambda n: n - 300 + + +globals()['by_subscript'] = lambda n: n - 100 +globals().update(by_update=lambda n: n - 200) +rebind(globals()) +exec('by_exec = lambda n: n - 400') +", + &[ + "m.call_all(1)", + "[f(1) for f in (m.by_subscript, m.by_update, m.by_helper, m.by_exec, m.kept)]", + "[f.__name__ for f in (m.by_subscript, m.by_update, m.by_helper, m.by_exec, m.kept)]", + ], + ) else { + return; + }; + // a forwarder's frame names the file every compiled function's does, so this is what + // says the function that was not written over is still the compiled one + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_nswrite as m\n\ + print(m.kept.__code__.co_filename, m.call_all.__code__.co_filename)\n", + ); + assert_eq!(out, " "); +} + +#[test] +fn a_class_the_module_body_writes_over_through_its_namespace_keeps_the_write() { + // compiled code reaches a class without going through its name, so a write over the + // name that the source does not spell out leaves the whole module as its interpreted + // definition built it, where `make` calls what the name holds + agree_python( + "nsclasswrite", + "\ +class Kept: + def get(self) -> int: + return 1 + + +class Other: + def get(self) -> int: + return 2 + + +def make() -> object: + return Kept().get() + + +globals()['Kept'] = Other +", + &["m.make()", "m.Kept is m.Other", "m.Kept.__name__"], + ); +} + #[test] fn a_rebound_global_is_observed() { // resolving a global per call rather than caching it: early binding is a @@ -7886,6 +8274,10 @@ def total(shapes: list[Shape]) -> int: /// a method whose decorators only mark it answers as python's does, marks included #[test] fn a_method_whose_decorators_only_mark_it_agrees() { + // the source is python 3.12's: `typing.override` is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } agree_python( "markingdeco", MARKING_METHODS, @@ -7914,6 +8306,10 @@ fn a_method_whose_decorators_only_mark_it_agrees() { /// calling the decorator a second time. `type` is what tells the two apart #[test] fn a_method_whose_decorators_only_mark_it_is_the_compiled_one() { + // the source is python 3.12's: `typing.override` is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((python, toolchain)) = environment() else { return; }; @@ -8239,6 +8635,10 @@ def same(a: Money, b: Money) -> bool: #[test] fn a_marked_dunder_agrees() { + // the source is python 3.12's: `typing.override` is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } agree_python( "markeddunder", MARKED_DUNDERS, @@ -8255,6 +8655,10 @@ fn a_marked_dunder_agrees() { /// …and the name answers with the compiled method #[test] fn a_marked_dunder_is_the_compiled_method() { + // the source is python 3.12's: `typing.override` is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((_dir, answers)) = compiled_answers( "markeddunder_t", MARKED_DUNDERS, @@ -8303,6 +8707,11 @@ def total(shapes: list[Shape]) -> int: #[test] fn an_override_modifier_agrees() { + // below 3.12 the transpiler imports `override` from `typing_extensions`, which an + // interpreter does not ship, so neither leg can import the module there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } agree( "overridemodifier", MARKING_MODIFIERS, @@ -8316,6 +8725,11 @@ fn an_override_modifier_agrees() { /// …and the method it marks is the compiled one #[test] fn an_override_modifier_is_the_compiled_method() { + // below 3.12 the transpiler imports `override` from `typing_extensions`, which an + // interpreter does not ship, so neither leg can import the module there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((_dir, answers)) = compiled_answers( "overridemodifier_t", MARKING_MODIFIERS, @@ -10790,6 +11204,81 @@ def took(obj, name): ); } +#[test] +fn a_decorated_class_declaring_slots_throughout_keeps_the_bare_layout() { + // a class decorator is handed the class the `class` statement made, and with + // `__slots__` declared on every rung that class has no dict in python. so neither has + // the emitted one, whatever the decorators go on to do with it + let Some(compiled) = agree_python_built( + "slotsdecorated", + "\ +import functools + + +def tag(cls): + cls.tagged = True + return cls + + +def took(obj, name): + try: + setattr(obj, name, 7) + except AttributeError: + return 'AttributeError' + return getattr(obj, name) + + +class Base: + __slots__ = ('v',) + + def __init__(self, v: int) -> None: + self.v = v + + def __eq__(self, other: object) -> bool: + return isinstance(other, Base) and self.v == other.v + + def __hash__(self) -> int: + return self.v + + def get(self) -> int: + return self.v + + +@tag +@functools.total_ordering +class Leaf(Base): + __slots__ = () + + def __lt__(self, other: 'Leaf') -> bool: + return self.v < other.v + + def twice(self) -> int: + return self.v * 2 +", + &[ + "m.took(m.Leaf(1), 'brand_new')", + "m.took(m.Base(1), 'brand_new')", + "m.took(m.Leaf(1), 'v')", + "(hasattr(m.Leaf(1), '__dict__'), hasattr(m.Base(1), '__dict__'))", + "(m.Leaf(1) <= m.Leaf(2), m.Leaf(3) >= m.Leaf(2), m.Leaf(1).twice(), m.Leaf.tagged)", + ], + ) else { + return; + }; + // the interpreted leg holds functions, so the answers above only say something where + // the compiled leg holds its own methods + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_slotsdecorated as m\n\ + print(type(m.Base.get).__name__, type(m.Leaf.twice).__name__)\n", + ); + assert_eq!(out, "method_descriptor method_descriptor"); +} + #[test] fn a_slotted_class_over_a_base_that_declares_none_still_takes_a_dict() { // python asks the whole chain: a `__slots__` on one class does not take the dict its @@ -11906,9 +12395,10 @@ frozen data class Fixed: \x20 except (TypeError, AttributeError) as e:\n print(type(e).__name__)\n\ \x20 else:\n print('accepted')\n", ); - // the frozen write is *not* part of that delta: `FrozenInstanceError` is what the - // interpreted twin raises, and the emitted type's `__setattr__` raises the same - assert_eq!(out, "5\nTypeError\nAttributeError\nFrozenInstanceError"); + // the delete and the frozen write are *not* part of that delta: python deletes a + // field its slot holds, and `FrozenInstanceError` is what the interpreted twin raises, + // which the emitted type's `__setattr__` raises the same + assert_eq!(out, "5\nTypeError\naccepted\nFrozenInstanceError"); } #[test] @@ -13086,6 +13576,10 @@ class Sealed: #[test] fn a_method_answers_for_the_annotations_the_class_statement_evaluated() { + // the source is python 3.12's: a type parameter list is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // a method of a class answers `__annotations__` with what python evaluated for the // interpreted definition — on 3.14 when first asked for — whether it is an instance // method, a class method or a static method @@ -13254,6 +13748,10 @@ class Counter: #[test] fn a_nested_function_answers_for_its_annotations() { + // the source is python 3.12's: a type parameter list is new there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // python evaluates a nested function's annotations over the frames it is written in: // where the `def` stands on 3.13, and when they are first asked for from 3.14. either way // they, the signature `inspect` reads and the hints `typing` resolves are what the @@ -13448,6 +13946,9 @@ class Later: "_annotation_report(m.Box().make(3))", "_annotation_report(m.generic())", "_annotation_report(m.forward())", + // a name a function does not have is refused in python's words, whichever + // names the running version gives a function + "[(str(e), e.name, e.obj is f) for f in [m.forward()] for e in [_capture(lambda: f.nothing_here)]]", ], ); } @@ -13665,7 +14166,7 @@ fn a_data_class_is_slotted_as_its_twin_is() { out, "wrapper_descriptor wrapper_descriptor wrapper_descriptor wrapper_descriptor \ wrapper_descriptor\n\ - attribute 'n' of 'by_diff_dcslots.Fixed' objects is not writable 1" + attribute 'n' of 'Fixed' objects is not writable 1" ); } @@ -13760,6 +14261,11 @@ class Mixed(Plain): #[test] fn reading_a_class_s_annotations_leaves_the_module_compiled() { + // below 3.12 the transpiler imports `override` from `typing_extensions`, which an + // interpreter does not ship, so neither leg can import the module there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // from python 3.14 a class's annotations are worked out when they are first read, and // the read writes its answer back onto the class: the mapping, and `None` under // `__annotate_func__` where the class has no function computing one. `@dataclass` @@ -13866,6 +14372,11 @@ class Tagged: #[test] fn an_inherited_default_binds_the_same_value_in_both_halves() { + // below 3.12 the transpiler imports `override` from `typing_extensions`, which an + // interpreter does not ship, so neither leg can import the module there + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // basedpython carries a method's defaults to its overrides, so `Loud.say` may be // called with no argument even though nothing in its own header says so. the // compiled half reads its parameter list out of the source, where the default is @@ -14493,6 +15004,49 @@ fn a_nested_function_is_the_compiled_one() { assert_eq!(out, "by function False"); } +/// a nested function that captures nothing answers `__closure__` with `None`, as python's +/// does. one that captures a name refuses the attribute: python's answer is a tuple of +/// cells, and a compiled closure keeps the names in an environment its siblings share, +/// where cells made for the asking would go on holding the values they were made with — +/// see "a nested function's closure" in runtime.md +#[test] +fn a_nested_function_answers_closure_where_it_captures_nothing() { + let source = "\ +def plain(): + def inner(x: int) -> int: + return x + + return inner + + +def outer(k: int): + def inner(x: int) -> int: + return x + k + + return inner +"; + agree_python( + "closurenone", + source, + &["m.plain().__closure__", "m.plain()(3)", "m.outer(1)(2)"], + ); + let Some((_dir, answers)) = compiled_answers( + "closurecaptured", + source, + Options { + language: by_irbuild::Language::Python, + ..Options::default() + }, + &[ + "(type(m.plain()).__module__, type(m.outer(1)).__module__)", + "hasattr(m.outer(1), '__closure__')", + ], + ) else { + return; + }; + assert_eq!(answers, ["('by', 'by')", "False"]); +} + #[test] fn a_raise_out_of_a_try_body_does_not_leak_what_it_wrote() { // the exception edge is a CFG edge, and the refcount pass used not to follow it @@ -16693,44 +17247,51 @@ fn a_for_over_a_compiled_generator_is_emitted_as_its_type_s_step() { } let emitted = std::fs::read_to_string(dir.join("by_diff_stepped_pin.c")) .expect("the generated C is written beside the extension"); - let step = - "By_by_diff_stepped_pin_counting_gen_Type_next((By_by_diff_stepped_pin_counting_gen *)"; + let module = "by_diff_stepped_pin"; + let step = format!( + "{}(({} *)", + symbol("By", &[module, "counting$gen"], "_Type_next"), + symbol("By", &[module, "counting$gen"], ""), + ); for symbol in [ - "by_by_diff_stepped_pin_total", - "by_by_diff_stepped_pin_collected", - "by_by_diff_stepped_pin_left_and_resumed", - "by_by_diff_stepped_pin_relayed_gen__resume", + symbol("by", &[module, "total"], ""), + symbol("by", &[module, "collected"], ""), + symbol("by", &[module, "left_and_resumed"], ""), + symbol("by", &[module, "relayed$gen", "$resume"], ""), ] { - let body = emitted_function(&emitted, symbol); - assert!(body.contains(step), "{symbol}: {body}"); + let body = emitted_function(&emitted, &symbol); + assert!(body.contains(&step), "{symbol}: {body}"); assert!(body.contains(", by_depth, 1)"), "{symbol}: {body}"); } // a frame handed no depth looks it up once, and a generator's body is handed the one // its step counted assert!( - emitted_function(&emitted, "by_by_diff_stepped_pin_total") + emitted_function(&emitted, &symbol("by", &[module, "total"], "")) .contains("ByDepth by_depth = By_DepthHere();"), "{emitted}" ); assert!( - !emitted_function(&emitted, "by_by_diff_stepped_pin_relayed_gen__resume") - .contains("By_DepthHere"), + !emitted_function( + &emitted, + &symbol("by", &[module, "relayed$gen", "$resume"], "") + ) + .contains("By_DepthHere"), "{emitted}" ); // a generator expression is stepped the same way, and a loop whose iterable is not // known to be one is left to the protocol with no depth looked up for it assert!( - emitted_function(&emitted, "by_by_diff_stepped_pin_over_an_expression") - .contains("_genexpr0_gen_Type_next(("), + emitted_function(&emitted, &symbol("by", &[module, "over_an_expression"], "")) + .contains("genexpr0Z24_gen_Type_next(("), "{emitted}" ); // `tp_iternext` finishes quietly through the same step assert!( emitted_function( &emitted, - "By_by_diff_stepped_pin_counting_gen_Type_iternext" + &symbol("By", &[module, "counting$gen"], "_Type_iternext") ) - .contains("By_by_diff_stepped_pin_counting_gen_Type_next("), + .contains(&symbol("By", &[module, "counting$gen"], "_Type_next(")), "{emitted}" ); } @@ -17107,8 +17668,11 @@ async def streamed() -> Any: ("awaited", "BY_FRAME_COROUTINE"), ("streamed", "BY_FRAME_ASYNC_GENERATOR"), ] { - let symbol = format!("By_by_diff_framekind_pin_{function}_gen_Type_send"); - let send = emitted_function(&emitted, &symbol); + let state = format!("{function}$gen"); + let send = emitted_function( + &emitted, + &symbol("By", &["by_diff_framekind_pin", &state], "_Type_send"), + ); assert!(send.contains(kind), "{function}: {send}"); for other in [ "BY_FRAME_GENERATOR", @@ -17128,23 +17692,153 @@ async def streamed() -> Any: // `__anext__` hands back, which is its `send` with nothing sent. a coroutine has no // such step of its own — it is not an iterator for symbol in [ - "By_by_diff_framekind_pin_plain_gen_Type_next", - "By_by_diff_framekind_pin_streamed_gen_Type_asend_send", + symbol("By", &["by_diff_framekind_pin", "plain$gen"], "_Type_next"), + symbol( + "By", + &["by_diff_framekind_pin", "streamed$gen"], + "_Type_asend_send", + ), ] { - let step = emitted_function(&emitted, symbol); + let step = emitted_function(&emitted, &symbol); assert!(step.contains("Py_None"), "{symbol}: {step}"); } assert!( - emitted_function(&emitted, "By_by_diff_framekind_pin_plain_gen_Type_iternext") - .contains("By_by_diff_framekind_pin_plain_gen_Type_next("), + emitted_function( + &emitted, + &symbol( + "By", + &["by_diff_framekind_pin", "plain$gen"], + "_Type_iternext" + ) + ) + .contains(&symbol( + "By", + &["by_diff_framekind_pin", "plain$gen"], + "_Type_next(" + )), "{emitted}" ); assert!( - !emitted.contains("By_by_diff_framekind_pin_awaited_gen_Type_iternext"), + !emitted.contains(&symbol( + "By", + &["by_diff_framekind_pin", "awaited$gen"], + "_Type_iternext" + )), "{emitted}" ); } +/// an emitted symbol: `prefix`, then each name [mangled](by_ir::mangle::mangle) and joined +/// with `_`, then `suffix` +fn symbol(prefix: &str, names: &[&str], suffix: &str) -> String { + let names: Vec = names + .iter() + .map(|name| by_ir::mangle::mangle(name)) + .collect(); + format!("{prefix}_{}{suffix}", names.join("_")) +} + +/// names python tells apart are emitted under symbols C tells apart +/// +/// each name used to have every character a C identifier cannot hold replaced with `_`, +/// and its parts joined with `_`: so `é1` and `_1` were one symbol, a function `A_b` and +/// the method `b` of a class `A` were one, the class `Foo_Type` was the type object of the +/// class `Foo`, and the module failed to build. a property's setter, named `value$set`, +/// declined rather than meet a method `value_set` +#[test] +fn names_python_tells_apart_build_under_their_own_symbols() { + agree_python( + "manglednames", + "\ +from collections.abc import Iterator + + +def A_b() -> int: + return 1 + + +def é1() -> int: + return 2 + + +def _1() -> int: + return 3 + + +class A: + def b(self) -> int: + return 4 + + +class Foo_Type: + def kind(self) -> str: + return \"Foo_Type\" + + +class Foo: + def kind(self) -> str: + return \"Foo\" + + +class Point_Type_get_x: + pass + + +class Point: + def __init__(self, x: int, é1: int, _1: int) -> None: + self.x = x + self.é1 = é1 + self._1 = _1 + + def total(self) -> int: + return self.x + self.é1 + self._1 + + +class Settable: + def __init__(self) -> None: + self.held = 0 + + @property + def value(self) -> int: + return self.held + + @value.setter + def value(self, v: int) -> None: + self.held = v + + def value_set(self) -> str: + return \"method\" + + +def counted_é(n: int) -> Iterator[int]: + for i in range(n): + yield i + + +def counted__(n: int) -> Iterator[int]: + for i in range(n): + yield i + 1 + + +def outer(n: int) -> int: + def é(k: int) -> int: + return k + n + + def _(k: int) -> int: + return k * n + + return é(1) + _(2) +", + &[ + "(m.A_b(), m.é1(), m._1(), m.A().b(), m.Foo_Type().kind(), m.Foo().kind())", + "[(p.total(), p.é1, p._1, vars(p)) for p in [m.Point(1, 2, 3)]]", + "[(setattr(s, 'value', 5), s.value, s.value_set()) for s in [m.Settable()]]", + "(list(m.counted_é(3)), list(m.counted__(3)), m.outer(5))", + "type(m.Point_Type_get_x()).__name__", + ], + ); +} + /// one emitted C function's body, from its definition to the closing brace /// /// a whole-module `contains` is not an assertion about the function it was written @@ -17218,7 +17912,11 @@ def written(n: int) -> object: let resume = |name: &str| { emitted_function( &emitted, - &format!("by_by_diff_finish_pin_{name}_gen__resume"), + &symbol( + "by", + &["by_diff_finish_pin", &format!("{name}$gen"), "$resume"], + "", + ), ) }; let finishes = |body: &str| body.matches("->by_returned = by_t;").count(); @@ -18451,13 +19149,20 @@ def through_the_base(a: A) -> int: } #[test] -fn a_base_named_override_reading_underscore_declines() { - // an override that took its names from the base binds `_` to the first of them at the - // top of its body, and a read of `_` finds that. the compiled build declines any body - // that reads a repeated `_`, so this one runs from its interpreted definition and the - // two builds answer alike - let Some(compiled) = agree_in( - "underscorebaseread", +fn a_base_named_override_reading_underscore_is_refused() { + // an override's `_`s take the base's names, but its body still spells them all `_`, so a + // read of `_` there cannot say which parameter it means. the transpiler refuses it, and + // the build with it, rather than leave the method to an interpreted definition that + // reads the first of them + let Some((python, toolchain)) = environment() else { + return; + }; + let dir = Scratch::new("by_diff_underscorebaseread_c".to_string()); + let options = Options { + fallback: Some(Config::default()), + ..Options::default() + }; + let error = build_source( "\ class A: def m(self, x: int, y: int) -> int: @@ -18466,12 +19171,134 @@ class A: class C(A): def m(self, _: int, _: int) -> int: return _ + 1 +", + "underscorebaseread", + &toolchain, + &dir, + &options, + ) + .expect_err("a read of a repeated `_` is not compiled"); + assert!( + format!("{error:#}").contains("reading a repeated `_` parameter of `m`"), + "expected the refusal, got {error:#}" + ); + + // the same override ignoring its `_`s takes the base's names, and compiles + let Some(compiled) = agree_in( + "underscorebaseignored", + "\ +class A: + def m(self, x: int, y: int) -> int: + return x * 100 + y + +class C(A): + def m(self, _: int, _: int) -> int: + return 1 ", &[ "str(__import__('inspect').signature(m.C().m))", "(m.C().m(3, 4), m.C().m(x=3, y=4), m.C().m(3, y=4))", ], - true, + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorebaseignored as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(m.C.__dict__['m']).__name__)\n", + ), + "method_descriptor" + ); +} + +#[test] +fn a_repeated_underscore_bound_again_before_its_read_compiles() { + // a write gives `_` a local of its own, so a read that only the write reaches reads + // what was written rather than any of the parameters. every way of binding a name is + // such a write, and each body compiles + let Some(compiled) = agree_in( + "underscorerebound", + "\ +import tempfile + +def assigns(_: int, _: str) -> int: + _ = 7 + return _ + +def retypes(_: int, _: str) -> str: + _ = 'own' + return _ + '!' + +def starred(_: int, *_: str) -> int: + _ = 5 + return _ + +def unpacks(_: int, _: str) -> int: + a, _ = 1, 2 + return a + _ + +def loops(_: int, _: str) -> int: + total = 0 + for _ in range(3): + total += _ + return total + +def walrus(_: int, _: str) -> int: + if (_ := 3) > 0: + return _ + return 0 + +def with_(_: int, _: str) -> bool: + with tempfile.TemporaryFile() as _: + return _.closed +", + &[ + "(m.assigns(1, 'a'), m.retypes(1, 'a'), m.starred(1, 'a', 'b'), m.unpacks(1, 'a'))", + "(m.loops(1, 'a'), m.walrus(1, 'a'), m.with_(1, 'a'))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorerebound as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print({f.__code__.co_filename for f in (m.assigns, m.retypes, m.starred, m.unpacks, m.loops, m.walrus, m.with_)})\n", + ), + "{''}" + ); +} + +#[test] +fn deleting_a_repeated_underscore_compiles() { + // `del _` takes no value, so it reads none of the parameters. it unbinds the name the + // first of them has in the python, and a read after it finds nothing + let Some(compiled) = agree_in( + "underscoredeleted", + "\ +def deletes(_: int, _: str) -> int: + del _ + return 1 + +def deletes_then_binds(_: int, _: str) -> int: + del _ + _ = 4 + return _ +", + &["(m.deletes(1, 'a'), m.deletes_then_binds(1, 'a'))"], + false, by_irbuild::Language::BasedPython, ) else { return; @@ -18483,11 +19310,132 @@ class C(A): run( &python, &compiled, - "import by_diff_underscorebaseread as m\n\ + "import by_diff_underscoredeleted as m\n\ assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ - print(type(m.C.__dict__['m']).__name__, m.C.m.__code__.co_filename)\n", + print({f.__code__.co_filename for f in (m.deletes, m.deletes_then_binds)})\n", ), - "function " + "{''}" + ); +} + +#[test] +fn a_nested_scope_beside_a_repeated_underscore_compiles() { + // a nested function that binds `_` on every path before its read, one that declares it + // `global`, and a comprehension or function binding a `_` of its own each read + // something other than the parameters, so each compiles + let Some(compiled) = agree_in( + "underscorenested", + "\ +_ = 5 + +def rebinds_nonlocal(_: int, _: str) -> int: + def inner() -> int: + nonlocal _ + _ = 9 + return _ + return inner() + +def declares_global(_: int, _: str) -> int: + def module() -> int: + global _ + return _ + return module() + +def binds_its_own(_: int, _: str) -> list[int]: + def g(_: bytes) -> bytes: + return _ + assert g(b'a') == b'a' + return [_ * 2 for _ in range(3)] +", + &["(m.rebinds_nonlocal(1, 'a'), m.declares_global(1, 'a'), m.binds_its_own(1, 'a'), m._)"], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorenested as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print({f.__code__.co_filename for f in (m.rebinds_nonlocal, m.declares_global, m.binds_its_own)})\n", + ), + "{''}" + ); +} + +#[test] +fn an_override_binding_or_deleting_underscore_compiles() { + // an override's `_`s take the base's names in the python, which binds `_` to the first + // of them at the top of a body that mentions it. a body that binds `_` before reading + // it, or deletes it, reads none of the parameters, and compiles — and `del _` finds the + // compiled method's `_` bound on entry too, as the python's is + let Some(compiled) = agree_in( + "underscorebaseown", + "\ +from typing import Iterator + +class A: + def m(self, x: int, y: str) -> str: + return y + +class Deletes(A): + def m(self, _: int, _: str) -> str: + del _ + return 'deleted' + +class Binds(A): + def m(self, _: int, _: str) -> str: + _ = 'own' + return _ + +class DeletesThenBinds(A): + def m(self, _: int, _: str) -> str: + del _ + _ = 'again' + return _ + +class Shares(A): + def m(self, _: int, _: str) -> str: + _ = 'outer' + def inner() -> str: + return _ + return inner() + +class Each: + def g(self, x: int, y: int) -> Iterator[int]: + yield x + +class Generates(Each): + def g(self, _: int, _: int) -> Iterator[int]: + _ = 3 + yield _ +", + &[ + "(m.Deletes().m(1, 'a'), m.Deletes().m(x=1, y='a'), m.Binds().m(1, 'a'), m.Binds().m(x=1, y='a'))", + "(m.DeletesThenBinds().m(1, 'a'), m.Shares().m(1, 'a'), list(m.Generates().g(1, 2)))", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_underscorebaseown as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print({type(c.__dict__[name]).__name__ for c, name in ((m.Deletes, 'm'), (m.Binds, 'm'), (m.DeletesThenBinds, 'm'), (m.Shares, 'm'), (m.Generates, 'g'))})\n", + ), + "{'method_descriptor'}" ); } @@ -20730,8 +21678,9 @@ def guarded() -> str: "m.submodule('a b/c')", "m.dotted('a b/c')", "m.guarded()", - // the type, the message and every attribute an `except` clause reads - "(lambda e: (type(e).__name__, str(e), e.name, e.path, e.name_from))(_capture(m.absent))", + // the type, the message and every attribute an `except` clause reads, of which + // `name_from` is 3.12's + "(lambda e: (type(e).__name__, str(e), e.name, e.path, getattr(e, 'name_from', None)))(_capture(m.absent))", // a thousand imports must not accumulate references to the module. a // leaked module is one object with a climbing refcount, which no // object-count check can see @@ -21660,6 +22609,12 @@ fn a_subclass_that_appends_nothing_past_a_base_agrees() { #[test] fn a_subclass_that_appends_nothing_is_the_compiled_type() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // the behaviour above is answered identically by a class that fell back to its // interpreted definition, so it cannot say which build answered. // `method_descriptor` can: a real type of ours holds one where the interpreted @@ -21773,6 +22728,12 @@ fn a_subclass_write_to_a_base_s_field_agrees() { #[test] fn a_subclass_write_to_a_base_s_field_keeps_the_rest_of_the_module() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // the behaviour above is the same either way — the base publishes a descriptor over // the field, so the dynamic write reached it too. what moved is which build answers, // and only `type(...).__name__` says: `Refused` falls back on its own, and `Held` and @@ -21946,14 +22907,21 @@ fn only_the_class_no_spec_can_build_is_left_interpreted() { \x20 type(m.Kept.__dict__['note']).__name__,\n\ \x20 type(m.Deeper.__dict__['down']).__name__,\n\ \x20 _leg(m.add))\n\ - # a spec has no code object to write one from, so its absence is the emitted type\n\ - print('__firstlineno__' in vars(m.Held), '__firstlineno__' in vars(m.Kept))\n", - ); - assert_eq!( - out, - "function method_descriptor method_descriptor native\n\ - True False" + # a spec has no code object to write one from, so its absence is the emitted type.\n\ + # `__firstlineno__` is 3.13's, and before it the type of a method is what says so\n\ + import sys\n\ + _twin = lambda c, method: '__firstlineno__' in vars(c) if sys.version_info >= (3, 13) else type(vars(c)[method]).__name__ == 'function'\n\ + print(_twin(m.Held, 'read'), _twin(m.Kept, 'note'))\n", ); + // below 3.12 there is no way to append storage to a base's instance, so no class here + // has a construction and the whole module stands down: every class and the module's + // own function are the interpreted definitions + let expected = if supports(&toolchain, (3, 12)) { + "function method_descriptor method_descriptor native\nTrue False" + } else { + "function function function function\nTrue True" + }; + assert_eq!(out, expected); } /// and the classes that stand beside the refused one still lay out, collect and free @@ -21966,6 +22934,12 @@ fn only_the_class_no_spec_can_build_is_left_interpreted() { /// what it finds — which is the failure that segfaulted 24 of the `encodings` modules #[test] fn the_classes_beside_a_refused_one_lay_out_and_deallocate() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((python, toolchain)) = environment() else { return; }; @@ -22085,12 +23059,19 @@ fn a_family_no_spec_can_build_agrees_beside_a_module_function() { /// which build answered, which no comparison of the two legs can say /// /// the family is interpreted either way — `__firstlineno__` is in both class dicts, since -/// a spec has no code object to write one from — so the only thing that moves is whether +/// a spec has no code object to write one from, and below 3.13, which has no +/// `__firstlineno__`, their methods are python functions — so the only thing that moves is whether /// the module's own function was installed. before the family could stand down together, /// `by_exec` gave up before `PyModule_AddFunctions` ran and `count` was an ordinary python /// function #[test] fn a_module_function_stands_where_the_whole_family_stood_down() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((python, toolchain)) = environment() else { return; }; @@ -22119,7 +23100,10 @@ fn a_module_function_stands_where_the_whole_family_stood_down() { "import by_diff_refusedfamily_t as m\n\ _leg = lambda f: 'native' if f.__code__.co_filename == '' else type(f).__name__\n\ print(_leg(m.count), m.count([1, 2, 3]))\n\ - print('__firstlineno__' in vars(m.Base), '__firstlineno__' in vars(m.Rotating))\n\ + # `__firstlineno__` is 3.13's, and before it the type of a method is what says so\n\ + import sys\n\ + _twin = lambda c, method: '__firstlineno__' in vars(c) if sys.version_info >= (3, 13) else type(vars(c)[method]).__name__ == 'function'\n\ + print(_twin(m.Base, 'label'), _twin(m.Rotating, 'label'))\n\ # the pair kept the definitions the module body built, so they still agree on\n\ # what a `Rotating` is an instance of\n\ print(m.Rotating.__base__ is m.Base, isinstance(m.Rotating('r', 2), m.Base))\n", @@ -22209,6 +23193,12 @@ fn a_chain_of_appended_storage_agrees() { /// released the wrong region would free somebody else's object #[test] fn a_chain_of_appended_storage_deallocates_without_growing() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((python, toolchain)) = environment() else { return; }; @@ -22363,6 +23353,12 @@ fn a_chain_over_a_base_that_holds_nothing_agrees() { /// and call it straight back until the stack ran out #[test] fn a_chain_over_a_base_that_holds_nothing_deallocates_without_growing() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } let Some((python, toolchain)) = environment() else { return; }; @@ -25260,11 +26256,11 @@ Bag.second = later /// fields has nothing to lose and moves. the alternative — turning every immutable class /// down up front — would cost the second case for nothing. #[test] -fn a_frozen_instance_moves_only_where_it_has_no_field_to_fill() { - // `Fixed.origin` is absent because `n` has no setter to write it through, and absent is - // the right answer: an emitted instance with `n` unwritten would answer `0`, which is - // the quiet wrong answer the whole move is arranged to avoid. `Blank.nothing` moves, - // because a frozen class with no fields is entirely its type +fn a_frozen_instance_moves_onto_the_compiled_class() { + // a `frozen data class` refuses every assignment in `__setattr__`, and a moved instance + // is filled underneath it, the way `object.__setattr__` fills one: each field takes its + // one write there. so `Fixed.origin` moves with its field, as `Blank.nothing` does with + // none, and each is the compiled class's own instance let Some((python, toolchain)) = environment() else { return; }; @@ -25303,16 +26299,168 @@ Blank.nothing = nothing &python, &dir, "import by_diff_twinfrozen as m\n\ - print(hasattr(m.Fixed, 'origin'), type(m.origin) is m.Fixed)\n\ + print(type(m.Fixed.origin) is m.Fixed, m.origin is m.Fixed.origin, m.origin.n)\n\ print(type(m.Blank.nothing) is m.Blank, m.nothing is m.Blank.nothing)\n", ); assert_eq!( out, - "False False\n\ + "True True 0\n\ True True" ); } +#[test] +fn a_module_level_instance_of_a_frozen_data_class_is_an_instance_of_it() { + // the module body builds `ORIGIN` and the key of `TABLE` with the interpreted class, + // and each is moved onto the compiled one, so it compares, hashes and is checked as + // what the class makes now + agree( + "frozenglobal", + "\ +from typing import Final + + +frozen data class Pair: + x: int + y: int + + +ORIGIN: Final = Pair(0, 0) +TABLE = {Pair(1, 2): \"a\"} + + +def origin() -> Pair: + return ORIGIN + + +def look(key: Pair) -> str: + return TABLE[key] +", + &[ + "(type(m.ORIGIN) is m.Pair, isinstance(m.ORIGIN, m.Pair))", + "(m.ORIGIN == m.Pair(0, 0), m.ORIGIN)", + "m.TABLE[m.Pair(1, 2)]", + "m.look(m.Pair(1, 2))", + "m.origin() is m.ORIGIN", + "(lambda ns: (exec('try:\\n o.x = 1\\nexcept Exception as e:\\n r = type(e).__name__', ns), \ + ns['r'])[1])({'o': m.ORIGIN})", + ], + ); +} + +/// a set the module body built holds the instances its members were built as, and each is +/// moved onto the compiled class, so the set answers for what the class makes now. a member +/// that hashes as the instance it replaces takes that instance's own entry, so the set +/// iterates in the order it did; one that does not — a class hashed on its identity — has +/// the set built again. a `frozenset` is written in place where every hash is kept, and is +/// settled as a copy where one is not, as a tuple is +#[test] +fn a_set_the_module_body_built_holds_the_compiled_class_s_instances() { + let Some(compiled) = agree_in( + "twinsets", + "\ +frozen data class Pair: + x: int + y: int + + +class Cell: + def __init__(self, v: int) -> None: + self.v = v + + +POINTS = {Pair(1, 2), Pair(3, 4), Pair(5, 6)} +FROZEN = frozenset({Pair(1, 2), Pair(7, 8)}) +CELLS = {Cell(1), Cell(2)} +HELD = {\"cells\": frozenset({Cell(3)})} +NESTED = {FROZEN: 1} + + +def has(p: Pair) -> bool: + return p in POINTS +", + &[ + "(m.Pair(1, 2) in m.POINTS, m.has(m.Pair(3, 4)), m.Pair(9, 9) in m.POINTS)", + "list(m.POINTS)", + "(m.Pair(7, 8) in m.FROZEN, list(m.FROZEN))", + "hash(m.FROZEN) == hash(frozenset({m.Pair(1, 2), m.Pair(7, 8)}))", + "(len(m.CELLS), all(isinstance(c, m.Cell) for c in m.CELLS), sorted(c.v for c in m.CELLS))", + "[isinstance(c, m.Cell) for c in m.HELD['cells']]", + "m.NESTED[frozenset({m.Pair(1, 2), m.Pair(7, 8)})]", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_twinsets as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Cell)['__init__']).__name__, type(vars(m.Pair)['__init__']).__name__)\n", + ), + "wrapper_descriptor wrapper_descriptor" + ); +} + +/// a subclass of `dict` or `set` whose every write is its base's own keeps no account of its +/// members beside the table, so its keys and members move as an exact one's do. a subclass +/// that writes one of its own — `OrderedDict` keeps its order in a list of its own — is left +/// as the body built it, see runtime.md +#[test] +fn a_dict_or_set_subclass_that_writes_as_its_base_moves_its_members() { + let Some(compiled) = agree_python_built( + "twinsubsets", + "\ +class Key: + def __init__(self, x: int) -> None: + self.x = x + + def __eq__(self, other: object) -> bool: + return type(other) is type(self) and other.x == self.x + + def __hash__(self) -> int: + return hash(self.x) + + +class Table(dict): + pass + + +class Bag(set): + pass + + +TABLE = Table({Key(1): 'a', Key(2): 'b'}) +BAG = Bag({Key(1), Key(2)}) +", + &[ + "(type(m.TABLE).__name__, m.TABLE.get(m.Key(1)), [k.x for k in m.TABLE])", + "(type(m.BAG).__name__, m.Key(2) in m.BAG, sorted(k.x for k in m.BAG))", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_twinsubsets as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Key)['__eq__']).__name__)\n", + ), + "wrapper_descriptor" + ); +} + /// a move that fails partway takes the moves begun inside it down as well /// /// a move registers itself before its fields are filled, which is what lets a cyclic graph @@ -25837,13 +26985,13 @@ def alias_answers() -> int: ); } -/// the three places a module-level name still keeps what the body put there +/// the place a module-level name still keeps what the body put there, and two it used to /// -/// a set's member and a dict's key are hashed on, so replacing one is a removal and an -/// insertion in a container somebody else is already holding — the settling walk refuses -/// both rather than moving them. +/// a set's member and a dict's key are hashed on, so each used to be refused rather than +/// moved. both move now: the dict is keyed again in the order it held its keys, and the set +/// is built again from its members, each the same object as before. /// -/// a module-level *tuple* is the third, and it is a different reason. a tuple cannot be +/// a module-level *tuple* is the one left, and it is a different reason. a tuple cannot be /// written, so settling one builds a copy; the module name is only ever rebound to a /// value that *is* something replaced, and a copy is not — putting one where the original /// stood would break every other holder's `is` against it, and a second holder walked @@ -25890,12 +27038,14 @@ PAIR = (proxy,) print(m.proxy in m.MEMBERS, m.proxy in m.KEYED, m.PAIR[0] is m.proxy)\n\ print(next(iter(m.MEMBERS))(), m.KEYED[next(iter(m.KEYED))], m.PAIR[0]())\n", ); - // python answers `True True True` on the first line. the second is what the + // python answers `True True True` on the first line. a set's member and a dict's key + // move, and the set and the dict are built again, so `MEMBERS` and `KEYED` agree; a + // tuple is the definition's own, so `PAIR` does not. the second line is what the // divergence still cannot reach: what is held there computes what the forwarder // computes, so nothing but identity sees it assert_eq!( out, - "False False False\n\ + "True True False\n\ 7 held 7" ); } @@ -26099,6 +27249,81 @@ def bare(cls: type) -> object: ); } +/// a class whose `int` fields start unset publishes a `__new__` that hands an interpreted +/// subclass its allocator, where python's class has none and `C.__new__` is +/// `object.__new__`. so it answers as `object.__new__` does: every refusal in python's words, +/// decided for the class python would have built — whose `tp_new` is `object`'s rather than +/// the dispatcher the published entry gives it — and an abstract subclass refused, which the +/// entry used to allocate +#[test] +fn the_new_a_class_with_unset_fields_publishes_answers_as_object_new_does() { + let Some(compiled) = agree_python_built( + "objectnewentry", + "\ +import abc + + +class Box: + def __init__(self, v: int) -> None: + self.v = v + + +class Lone: + def __init__(self, v: int) -> None: + self.v = v + + +class Sub(Box): + def __init__(self, v: int) -> None: + Box.__init__(self, v) + self.w = 1 + + +class Written: + def __new__(cls) -> 'Written': + return object.__new__(cls) +", + &[ + "str(_capture(lambda: m.Box.__new__()))", + "str(_capture(lambda: m.Box.__new__(1)))", + "str(_capture(lambda: m.Box.__new__(int)))", + "str(_capture(lambda: m.Box.__new__(type('Q', (dict,), {}))))", + "type(m.Box.__new__(m.Box)).__name__", + "type(m.Sub.__new__(m.Box)).__name__", + "type(m.Box.__new__(m.Lone)).__name__", + "type(m.Box.__new__(m.Written)).__name__", + "str(_capture(lambda: m.Box.__new__(m.Written, 1)))", + "type(m.Box.__new__(m.Box, 1, x=2)).__name__", + "type(m.Sub.__new__(m.Sub, 1)).__name__", + "type('S', (m.Box,), {})(1).v", + "str(_capture(lambda: m.Box.__new__(type('S', (m.Box,), {'__init__': object.__init__}), 1)))", + "str(_capture(lambda: m.Box.__new__(type('S', (m.Box,), {'__new__': lambda c: object.__new__(c)}), 1)))", + "str(_capture(lambda: m.abc.ABCMeta('A', (m.Box,), {'f': m.abc.abstractmethod(lambda s: 1)})(1)))", + "type(m.abc.ABCMeta('A', (m.Box,), {'f': lambda s: 1})(1)).__name__", + "str(_capture(lambda: m.Box.__new__(m.abc.ABCMeta('A', (m.Box,), {'f': m.abc.abstractmethod(lambda s: 1)}))))", + "str(_capture(lambda: object.__new__(type('S', (m.Box,), {})).v))", + "(m.Box.__new__.__name__, m.Box.__new__.__qualname__, m.Box.__new__.__self__)", + "(m.Box.__new__.__doc__, m.Box.__new__.__text_signature__)", + "repr(m.Box.__new__) == repr(object.__new__)", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_objectnewentry as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Box)['__new__']).__name__, type(vars(m.Box)['__init__']).__name__)\n", + ), + "builtin_function_or_method wrapper_descriptor" + ); +} + /// run a class statement's worth of python against the module and hand back what it bound /// to `result` const SUBCLASS_AGAINST: &str = @@ -26856,6 +28081,67 @@ class Sub(Written): ); } +#[test] +fn a_class_answers_the_annotate_function_its_body_wrote() { + // 3.14 keeps the function a class body's annotations are computed by, and + // `__annotate__` answers with it — `None` only for a class that wrote none. it + // evaluates the annotations afresh at each call: a name the class body bound is read + // out of the namespace the body ran in, where `Kind` was bound to the interpreted + // `Node`, and every other name out of the module, where the compiled type stands. + // `many` is one `__annotations__` refuses, because the mapping settled at import + // holds the interpreted `Node`; a call evaluates it again and has no such trouble. + // below 3.14 there is no `__annotate__` and each of these answers its default + let Some(compiled) = agree_python_built( + "annannotate", + "\ +class Node: + def tag(self) -> str: + return \"node\" + + +class Written: + Kind = Node + count: int + label: str = \"x\" + peer: Node + kind: Kind + many: list[Node] + + def tag(self) -> str: + return \"written\" + + +class Bare: + def tag(self) -> str: + return \"bare\" +", + &[ + "(lambda a: a if a is None else (type(a).__name__, a.__qualname__, a(1))) \ + (getattr(m.Written, '__annotate__', None))", + "(lambda a: a if a is None else \ + (a(1)['peer'] is m.Node, a(1)['kind'] is m.Node, a(1)['many'] == list[m.Node])) \ + (getattr(m.Written, '__annotate__', None))", + "getattr(m.Bare, '__annotate__', 'absent')", + "getattr(m.Written, '__annotate__', 1) is getattr(m.Written, '__annotate__', 2)", + "(lambda found: found and __import__('annotationlib').get_annotations( \ + m.Written, format=__import__('annotationlib').Format.STRING)) \ + (__import__('importlib.util').util.find_spec('annotationlib') is not None)", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + let out = run( + &python, + &compiled, + "import by_diff_annannotate as m\n\ + print(type(m.Written.tag).__name__, type(m.Bare.tag).__name__)\n", + ); + assert_eq!(out, "method_descriptor method_descriptor"); +} + #[test] fn a_deferred_annotation_is_carried_as_the_string_the_body_wrote() { // `from __future__ import annotations` makes every annotation the text of itself, so @@ -26967,7 +28253,7 @@ class Deep: out, "{'peer': , 'plain': list[int]} True\n\ False\n\ - refused type object 'by_diff_annreach.Deep' has no attribute '__annotations__'\n\ + refused type object 'Deep' has no attribute '__annotations__'\n\ sealed\n\ no base\n\ method_descriptor method_descriptor" @@ -27038,7 +28324,7 @@ class Fine: ); assert_eq!( out, - "refused type object 'by_diff_annlost.Node' has no attribute '__annotations__'\n\ + "refused type object 'Node' has no attribute '__annotations__'\n\ {'plain': }\n\ method_descriptor method_descriptor" ); @@ -27614,6 +28900,12 @@ EARLY = make_held().kind #[test] fn fields_past_a_python_base_leave_that_class_to_its_interpreted_definition() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // `Wrapped` keeps its fields past a `codecs.IncrementalDecoder` instance, so it // supplies `tp_dealloc`, `tp_traverse` and `tp_clear` and each calls the base's. // that base is a class statement's type, whose three are python's own dispatchers: @@ -30520,7 +31812,99 @@ async def echoed() -> AsyncGenerator[object, object]: lambda: t.send(None), lambda: next(g.aclose())])(g.aclose()))", "_against(m.counted, lambda g: (lambda t: [lambda: next(g.__anext__()), \ lambda: t.throw(KeyError('k')), lambda: t.send(None)])(g.athrow(ValueError('v'))))", + // 3.11's `throw` is not refused while another awaitable steps the generator + "_against(m.counted, lambda g: (lambda t: [lambda: next(g.__anext__()), \ + lambda: t.throw(KeyError('k')), lambda: next(g.__anext__())])(g.asend(None)))", + // from 3.13 closing an awaitable throws `GeneratorExit` into the generator it is + // stepping; before it the awaitable is only finished, and the generator is left + // running + "_against(m.counted, lambda g: (lambda a: [lambda: a.send(None), lambda: a.close(), \ + lambda: next(g.__anext__()), lambda: next(g.aclose())])(g.__anext__()))", + ], + ); +} + +#[test] +fn a_coroutine_dropped_unawaited_warns() { + // python reports a coroutine dropped before it ever ran, and one the filters turn into + // an error is reported as unraisable, which 3.14 says it was finalizing the coroutine + let Some(compiled) = agree_python_built( + "corounawaited", + "\ +async def answer() -> int: + return 1 +", + &[ + "_warnings_from(lambda: m.answer())", + "_unraisable_from(lambda: m.answer())", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_corounawaited as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + c = m.answer()\n\ + print(type(c).__name__)\n\ + c.close()\n", + ), + "answer$gen" + ); +} + +#[test] +fn an_async_generator_s_awaitable_dropped_unstepped_warns() { + // from 3.13 python reports an awaitable of an async generator that is dropped before it + // was ever stepped: `coroutine method 'asend' of 'numbers' was never awaited`, naming + // `asend` for `__anext__()` too. one that was stepped, or closed, is not reported, and + // before 3.13 none is + let Some(compiled) = agree_python_built( + "agenunawaited", + "\ +from collections.abc import AsyncIterator + + +async def numbers() -> AsyncIterator[int]: + yield 1 + + +class Box: + async def items(self) -> AsyncIterator[int]: + yield 2 +", + &[ + "_warnings_from(lambda: m.numbers().asend(None))", + "_warnings_from(lambda: m.numbers().__anext__())", + "_warnings_from(lambda: m.numbers().athrow(ValueError))", + "_warnings_from(lambda: m.numbers().aclose())", + "_warnings_from(lambda: m.Box().items().asend(None))", + "_warnings_from(lambda: m.numbers().asend(None).close())", + "_warnings_from(lambda: [next(m.numbers().asend(None), None)])", + // a warning the filters turn into an error is reported as unraisable, which 3.14 + // says it was finalizing the generator + "_unraisable_from(lambda: m.numbers().asend(None))", ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_agenunawaited as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(m.numbers()).__name__, type(m.Box().items()).__name__)\n", + ), + "numbers$gen items$gen" ); } @@ -30605,12 +31989,13 @@ async def drains(source: Any) -> list[int]: ); } +// a resumable frame's `return` *is* a `StopIteration`, so raising it under the enclosing +// `with`'s error target handed that exception to `__exit__` as though the block had failed. +// the cleanups run first now, and the raise leaves without a target of its own. the three +// tests below are one program split by the frame that leaves, so each compiles a third of it + #[test] fn leaving_a_context_early_runs_its_exit_with_no_exception() { - // a resumable frame's `return` *is* a `StopIteration`, so raising it under the - // enclosing `with`'s error target handed that exception to `__exit__` as though - // the block had failed. the cleanups run first now, and the raise leaves without - // a target of its own agree_python( "earlyexit", "\ @@ -30630,6 +32015,21 @@ def bare(manager: Any, log: list[str]) -> Any: with manager as name: yield str(name) return +", + &[ + "_logged(lambda t, log: list(m.returned(t, log, 2)))", + "_logged(lambda t, log: _value(m.returned(t, log, 2)))", + "_logged(lambda t, log: _value(m.bare(t, log)))", + ], + ); +} + +#[test] +fn leaving_an_async_context_runs_its_exit_with_no_exception() { + agree_python( + "earlyexit_async", + "\ +from typing import Any async def returns(manager: Any, log: list[str]) -> str: @@ -30648,6 +32048,21 @@ async def raises(manager: Any, log: list[str]) -> str: async with manager as name: log.append('body ' + str(name)) raise ValueError('boom') +", + &[ + "_logged(lambda t, log: _run(m.returns(t, log)))", + "_logged(lambda t, log: _run(m.falls(t, log)))", + "_logged(lambda t, log: _run(m.raises(t, log)))", + ], + ); +} + +#[test] +fn leaving_a_loop_or_a_nested_async_context_runs_each_exit_with_no_exception() { + agree_python( + "earlyexit_async_nested", + "\ +from typing import Any async def breaks(manager: Any, log: list[str], n: int) -> str: @@ -30674,12 +32089,6 @@ async def pair(manager: Any, log: list[str]) -> str: return 'pair' ", &[ - "_logged(lambda t, log: list(m.returned(t, log, 2)))", - "_logged(lambda t, log: _value(m.returned(t, log, 2)))", - "_logged(lambda t, log: _value(m.bare(t, log)))", - "_logged(lambda t, log: _run(m.returns(t, log)))", - "_logged(lambda t, log: _run(m.falls(t, log)))", - "_logged(lambda t, log: _run(m.raises(t, log)))", "_logged(lambda t, log: _run(m.breaks(t, log, 4)))", "_logged(lambda t, log: _run(m.pair(t, log)))", "_nested(lambda a, b, log: _run(m.nested(a, b, log)))", @@ -31889,10 +33298,16 @@ def counter() -> Callable[[], int]: \x20 print(str(e).replace(kind.__module__ + '.', '').replace(kind.__name__, 'T'))\n\ print(env[0].__init__(), step())\n", ); - assert_eq!( - out, + // the refusal is python's own for a type it may not instantiate, and python reworded it + // in 3.13: before it `object.__new__` names the type it was handed as unsafe, exactly + // as it does for a `list_iterator` + let expected = if supports(&toolchain, (3, 13)) { "1\ncannot create 'T' instances\ncannot create 'T' instances\ncannot create 'T' instances\nNone 2" - ); + } else { + "1\nobject.__new__(T) is not safe, use T.__new__()\ncannot create 'T' instances\n\ + object.__new__(T) is not safe, use T.__new__()\nNone 2" + }; + assert_eq!(out, expected); } #[test] @@ -32742,7 +34157,7 @@ def classify(v: object) -> int: }; let c = std::fs::read_to_string(compiled.join("by_diff_onelayouttest.c")) .expect("the emitted C is kept"); - let body = one_function(&c, "_classify("); + let body = one_function(&c, &format!("_{}(", by_ir::mangle::mangle("classify"))); assert!( body.contains("PyObject_TypeCheck("), "the layout test is gone:\n{body}" @@ -36121,12 +37536,14 @@ fn the_members_a_data_class_generates_agree() { /// the generated `__eq__` is `self.a==other.a and self.b==other.b`, term by term /// -/// not a comparison of two tuples, which is what it looks like and what it used to be. -/// the difference shows in three places at once: `and` hands back the *term* rather than -/// a bool, the chain stops at the first falsy one, and a plain `==` has none of the -/// identity shortcut a tuple comparison applies to each pair — so two instances sharing -/// one NaN are not equal, while an instance is equal to itself because the generated body -/// tests that outright before it looks at a field +/// from 3.13, that is. before it the generated method compared two tuples, and the +/// difference shows in three places at once: `and` hands back the *term* rather than a +/// bool, the chain stops at the first falsy one, and a plain `==` has none of the identity +/// shortcut a tuple comparison applies to each pair — so from 3.13 two instances sharing one +/// NaN are not equal, while an instance is equal to itself because the generated body tests +/// that outright before it looks at a field. a tuple is built whole before it is compared, +/// so below 3.13 every field of the left operand is read before any of the right one's. +/// both legs run the one interpreter, so each of these asks that interpreter's rule #[test] fn the_generated_equality_answers_term_by_term() { agree( @@ -36136,6 +37553,10 @@ frozen data class Holder: a: object b: object +data class Open: + a: object + b: object + frozen data class Blank: pass @@ -36171,9 +37592,16 @@ class Falsy: "[(p := m.Floats(float('nan'))), p == p][-1]", "[(p := m.Floats(float('nan'))), p != p][-1]", "[(p := m.Holder(float('nan'), 1)), p == p][-1]", - // …and two instances sharing the one NaN object are not - "[(n := float('nan')), m.Floats(n) == m.Floats(n)][-1]", + // …and from 3.13 two instances sharing the one NaN object are not. before it + // python finds the one object in both and calls them equal, which a `float` + // field kept unboxed cannot: runtime.md lists it, and `object` fields agree + "[(n := float('nan')), m.Floats(n) == m.Floats(n)][-1] \ + if __import__('sys').version_info >= (3, 13) else None", "[(n := float('nan')), m.Holder(n, 1) == m.Holder(n, 1)][-1]", + // which field is read first: the left operand lacks `b`, the right one `a` + "repr(_capture(lambda: [(l := m.Open.__new__(m.Open)), setattr(l, 'a', 1), l == m.Open.__new__(m.Open)]))", + // an instance compared with itself, holding no fields at all + "repr(_capture(lambda: [(p := m.Open.__new__(m.Open)), p == p][-1]))", ], ); } @@ -36191,7 +37619,10 @@ fn the_dataclass_bookkeeping_of_a_data_class_agrees() { "__import__('dataclasses').astuple(m.Pair(1, 2))", "__import__('dataclasses').asdict(m.Pair(1, 2))", "repr(__import__('dataclasses').replace(m.Pair(1, 2), x=9))", - "repr(__import__('copy').replace(m.Pair(1, 2), y=9))", + // `copy.replace` and the `__replace__` it reaches for are 3.13's; before it + // the twin's class has no `__replace__`, and the emitted type has none either + "repr(__import__('copy').replace(m.Pair(1, 2), y=9)) \ + if __import__('sys').version_info >= (3, 13) else hasattr(m.Pair, '__replace__')", ], ); } @@ -36269,9 +37700,12 @@ frozen data class Pair: &[ "repr(_capture(setattr, m.Pair(1, 2), 'x', 5))", "repr(_capture(delattr, m.Pair(1, 2), 'x'))", - // not only the fields: python's generated `__setattr__` refuses every name - // on an instance of the class that declared it + // not only the fields: from 3.13 python's generated `__setattr__` refuses every + // name on an instance of the class that declared it. before 3.13 it holds the + // class `slots=True` replaced, so a name that is not a field reaches `super` + // with that class and is refused there with a `TypeError` "repr(_capture(setattr, m.Pair(1, 2), 'extra', 5))", + "repr(_capture(delattr, m.Pair(1, 2), 'extra'))", "isinstance(_capture(setattr, m.Pair(1, 2), 'x', 5), AttributeError)", ], ); @@ -36306,14 +37740,23 @@ fn the_generated_dataclass_members_are_slots_of_the_emitted_type() { print('__hash__ of a mutable one', m.Point.__hash__)\n", ); // a `wrapper_descriptor` is what `PyType_Ready` builds over a filled slot: the - // interpreted twin's would be a plain `function` + // interpreted twin's would be a plain `function`. below 3.13 a frozen one answers a + // write with the twin's own `__setattr__`, which is a function — see + // `By_CarryDataclassMembers` + let setattr = if supports(&toolchain, (3, 13)) { + "wrapper_descriptor" + } else { + "function" + }; assert_eq!( out, - "__repr__ wrapper_descriptor\n\ - __eq__ wrapper_descriptor\n\ - __hash__ wrapper_descriptor\n\ - __setattr__ wrapper_descriptor\n\ - __hash__ of a mutable one None" + format!( + "__repr__ wrapper_descriptor\n\ + __eq__ wrapper_descriptor\n\ + __hash__ wrapper_descriptor\n\ + __setattr__ {setattr}\n\ + __hash__ of a mutable one None" + ) ); } @@ -36727,7 +38170,7 @@ def area_of(shape: Shape) -> int: }; assert!(built.declined.is_empty(), "declined: {:?}", built.declined); let c = std::fs::read_to_string(&built.artifact.source).expect("the generated C is read"); - let body = one_function(&c, "_area_of("); + let body = one_function(&c, &format!("_{}(", by_ir::mangle::mangle("area_of"))); assert!( body.contains("By_MethodStands("), "the licence test is gone:\n{body}" @@ -37056,7 +38499,11 @@ class Quiet(Base): let generated = std::fs::read_to_string(built.artifact.source).expect("the generated c is there"); let (_, tail) = generated - .split_once("By_by_diff_inheritedinitslot_Quiet_Type_slots[]") + .split_once(&symbol( + "By", + &["by_diff_inheritedinitslot", "Quiet"], + "_Type_slots[]", + )) .expect("the subclass has a slot table"); let (slots, _) = tail.split_once("};").expect("the slot table ends"); assert!(!slots.contains("Py_tp_init"), "the subclass filled tp_init"); @@ -37202,11 +38649,12 @@ fn a_slots_declaration_is_storage_the_emitted_type_owns() { let generated = std::fs::read_to_string(built.artifact.source).expect("the generated c is there"); let (_, tail) = generated - .split_once("struct By_by_diff_slotsowned_Link {") + .split_once(&symbol("struct By", &["by_diff_slotsowned", "Link"], " {")) .expect("`Link` has an instance struct"); let (fields, _) = tail.split_once('}').expect("the struct ends"); assert!( - fields.contains("by_f_prev") && fields.contains("by_p_prev"), + fields.contains(&format!("by_f_{}", by_ir::mangle::mangle("prev"))) + && fields.contains(&format!("by_p_{}", by_ir::mangle::mangle("prev"))), "`Link` kept no room for `prev`: {fields}" ); } @@ -37679,26 +39127,35 @@ def through(f: object, x: str) -> object: fn a_keyword_given_twice_through_a_merge_is_refused_as_python_refuses_it() { // a call's `**` merges a mapping into the keywords, and unlike a dict display it // refuses a keyword that is already there — from a name written before it, from an - // earlier `**`, or from a name written after it. both refusals, and the one for an - // operand that is no mapping, name the callee the way python does. python rewords - // any single-argument `KeyError` the merge raises the same way, one a mapping's own - // `__getitem__` raised included + // earlier `**`, or from a name written after it — before it asks the mapping for that + // keyword's value. both refusals, and the one for an operand that is no mapping, name + // the callee the way python does. python rewords every `AttributeError` the merge + // raises, and from 3.12 any single-argument `KeyError` the same way, one a mapping's + // own `__getitem__` raised included. 3.11 rewords only the merge's own `KeyError`, and + // not even that one while an exception is being handled agree_python( "kwmerge", "\ class Keys: def __init__(self, keys: list[str]) -> None: self.names = keys + self.asked: list[str] = [] def keys(self) -> list[str]: return self.names def __getitem__(self, key: str) -> int: + self.asked.append(key) if key == \"missing\": raise KeyError(key) return len(key) +class Unkeyed: + def keys(self) -> list[str]: + raise AttributeError + + class Counter: def m(self, a: int, **extra: int) -> int: return a + len(extra) @@ -37728,6 +39185,13 @@ def not_a_mapping(x: object) -> int: return f(1, **x) # type: ignore +def merged_while_handling(d: dict[str, int]) -> int: + try: + raise ValueError + except ValueError: + return f(1, step=2, **d) + + def method(c: Counter, d: dict[str, int]) -> int: return c.m(1, b=2, **d) @@ -37744,6 +39208,11 @@ def display(d: dict[str, int], e: dict[str, int]) -> dict[str, int]: "m.merged_mapping(m.Keys(['x', 'yy']))", "repr(_capture(m.merged_mapping, m.Keys(['x', 'step'])))", "repr(_capture(m.merged_mapping, m.Keys(['missing'])))", + // the value of a keyword given twice is never asked for + "[(repr(_capture(m.merged_mapping, k)), k.asked) for k in [m.Keys(['x', 'step', 'y'])]]", + "repr(_capture(m.not_a_mapping, m.Unkeyed()))", + "repr(_capture(m.merged_while_handling, {'step': 7}))", + "repr(_capture(m.merged_while_handling, m.Keys(['step'])))", "repr(_capture(m.not_a_mapping, [1]))", "repr(_capture(m.not_a_mapping, 3))", // a dict subclass is walked as a dict, whatever its `keys` says @@ -40278,6 +41747,238 @@ fn the_class_that_deletes_an_attribute_is_the_compiled_one() { assert_eq!(out, "method_descriptor method_descriptor"); } +const REASSIGNS_CLASS: &str = "\ +class Base: + def __init__(self) -> None: + self.x = 1 + + def name(self) -> str: + return \"base\" + + def both(self) -> str: + return self.name() + self.name() + + +class Circle(Base): + def name(self) -> str: + return \"circle\" + + +class Shape(Base): + def name(self) -> str: + return \"shape\" + + +class Wide(Base): + def __init__(self) -> None: + super().__init__() + self.y = 2 + + +class Lone: + def __init__(self) -> None: + self.x = 1 + + +class Other: + def __init__(self) -> None: + self.x = 1 + + +def describe(b: Base) -> str: + return b.name() + b.both() +"; + +/// an instance moved to a class laid out as its own one is reaches that class through +/// compiled code, and one python refuses to move is refused in python's words +/// +/// two classes the module extends that share a base and add no field to it lay their +/// instances out alike, so python takes the move. a call compiled against the base asks the +/// instance's type before it reaches a method directly, so the move is seen — see +/// runtime.md#an-instances-__class__-reassigned +#[test] +fn an_instance_moved_to_a_class_laid_out_alike_answers_as_that_class() { + agree_python( + "classmove", + REASSIGNS_CLASS, + &[ + "[(setattr(o, '__class__', m.Shape), type(o).__name__, m.describe(o), o.both(), o.x) \ + for o in [m.Circle()]]", + "repr(_capture(setattr, m.Circle(), '__class__', int))", + "repr(_capture(setattr, m.Lone(), '__class__', int))", + ], + ); +} + +/// and every other move between two compiled classes is refused, where python's classes take +/// it +/// +/// the refusal is python's own, made against the layouts the module emitted: a class nothing +/// extends is an immutable type, and a class and its base, or a subclass that adds a field, +/// lay their instances out differently. it is what keeps a method reached directly working +/// on the layout its class was compiled with. these are the documented differences, held +/// here so a change to either is seen +#[test] +fn an_instance_moved_between_compiled_classes_laid_out_apart_is_refused() { + let Some(compiled) = agree_python_built( + "classmoveapart", + REASSIGNS_CLASS, + &["m.describe(m.Circle())"], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_classmoveapart as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + for source, target in ((m.Circle, m.Base), (m.Base, m.Circle), (m.Circle, m.Wide), \ + (m.Lone, m.Other)):\n\ + \x20 try:\n\ + \x20 source().__class__ = target\n\ + \x20 except TypeError as error:\n\ + \x20 print(error)\n", + ), + "__class__ assignment: 'Base' object layout differs from 'Circle'\n\ + __class__ assignment: 'Circle' object layout differs from 'Base'\n\ + __class__ assignment: 'Wide' object layout differs from 'Circle'\n\ + __class__ assignment only supported for mutable types or ModuleType subclasses" + ); +} + +/// a field the module never deletes is deleted from outside it as python deletes one +/// +/// every field python can name has an absent state, the one an instance `__init__` never +/// ran on is in, so a delete from another module puts it back there. it used to refuse with +/// `cannot delete an attribute`, whatever the field held, where python deletes. a compiled +/// read that follows finds it gone, a second delete raises python's own words — which name +/// only the attribute where the class keeps it in a slot — and a write sets it again +#[test] +fn a_field_deleted_from_outside_the_module_is_gone() { + agree_python( + "outsidedelete", + "\ +from collections.abc import Callable + + +class Point: + def __init__(self, x: int, y: float, label: str, flag: bool) -> None: + self.x = x + self.y = y + self.label = label + self.flag = flag + + def total(self) -> float: + return self.x + self.y + + def describe(self) -> str: + return self.label + str(self.flag) + + +class Slotted: + __slots__ = (\"a\",) + + def __init__(self, a: int) -> None: + self.a = a + + def read(self) -> int: + return self.a + + +def around(p: Point, hook: Callable[[Point], object]) -> int: + first = p.x + hook(p) + return first + p.x +", + &[ + "[(n, delattr(p, n), hasattr(p, n), repr(_capture(delattr, p, n))) \ + for p in [m.Point(1, 2.0, 's', True)] for n in ('x', 'y', 'label', 'flag')]", + "[(delattr(p, 'x'), repr(_capture(p.total)), setattr(p, 'x', 5), p.total()) \ + for p in [m.Point(1, 2.0, 's', True)]]", + "[(delattr(p, 'flag'), repr(_capture(p.describe)), vars(p)) \ + for p in [m.Point(1, 2.0, 's', True)]]", + "[(delattr(s, 'a'), repr(_capture(s.read)), repr(_capture(delattr, s, 'a'))) \ + for s in [m.Slotted(1)]]", + "repr(_capture(m.around, m.Point(1, 2.0, 's', True), lambda p: delattr(p, 'x')))", + "m.around(m.Point(1, 2.0, 's', True), lambda p: setattr(p, 'x', 10))", + ], + ); +} + +/// a store lets go of the value it replaces once the new one is in place, as python's own +/// store does +/// +/// letting go of the old value can run its `__del__`, and one that deletes or reads the +/// field it was held in has to find the new value there. the store used to let go first, +/// and the finalizer found the value being let go of still in the field — where deleting +/// it would have let go of it a second time +#[test] +fn a_finalizer_a_store_runs_finds_the_new_value_in_place() { + agree_python( + "storethenrelease", + "\ +class Holder: + def __init__(self, held: object) -> None: + self.held = held + self.seen = \"\" + + def replace(self, value: object) -> None: + self.held = value +", + &[ + "(lambda h: (setattr(h, 'held', type('Fin', (), {'__del__': lambda self: delattr(h, 'held')})()), \ + h.replace(2), hasattr(h, 'held')))(m.Holder(1))[2]", + "(lambda h: (setattr(h, 'held', type('Fin', (), {'__del__': lambda self: setattr(h, 'seen', repr(h.held))})()), \ + h.replace(3), h.seen, h.held))(m.Holder(1))[2:]", + "(lambda h: (setattr(h, 'held', type('Fin', (), {'__del__': lambda self: setattr(h, 'seen', repr(h.held))})()), \ + setattr(h, 'held', 4), h.seen, h.held))(m.Holder(1))[2:]", + ], + ); +} + +/// `obj.__dict__.clear()` removes every attribute, as python's clears an instance's dict +/// +/// it went through each name in turn and stopped at the first field that refused to be +/// deleted, leaving the names before it gone and the rest in place. it lets go of what it +/// removed only once every name is gone, as python's clear does, so a `__del__` that +/// letting go runs finds the object already cleared. `obj.__dict__ = {...}` removes what +/// the new mapping does not hold the same way +#[test] +fn clearing_an_instance_dict_removes_every_attribute() { + agree_python( + "dictclear", + "\ +class Record: + def __init__(self, a: int, b: str) -> None: + self.a = a + self.b = b + + def both(self) -> str: + return str(self.a) + self.b + + +class Box: + def __init__(self, held: object, tag: str) -> None: + self.held = held + self.tag = tag +", + &[ + "[(d.__setitem__('extra', 3), d.clear(), vars(r), [hasattr(r, n) for n in ('a', 'b', 'extra')], \ + repr(_capture(r.both))) for r in [m.Record(1, 'x')] for d in [r.__dict__]]", + "[(setattr(r, 'extra', 3), setattr(r, '__dict__', {'b': 'y'}), vars(r), \ + [hasattr(r, n) for n in ('a', 'b', 'extra')]) for r in [m.Record(1, 'x')]]", + "[(d.clear(), setattr(r, 'a', 2), setattr(r, 'b', 'z'), r.both(), vars(r)) \ + for r in [m.Record(1, 'x')] for d in [r.__dict__]]", + "[(setattr(b, 'held', type('Fin', (), {'__del__': lambda self: setattr(b, 'seen', sorted(vars(b)))})()), \ + b.__dict__.clear(), vars(b)) for b in [m.Box(None, 'x')]][0][2]", + ], + ); +} + /// and it releases what the field held exactly once /// /// a delete that forgot to release leaks the value; one that cleared the byte without @@ -42134,18 +43835,21 @@ def keys(n: int) -> str: } let emitted = std::fs::read_to_string(dir.join("by_diff_strofint_shape.c")) .expect("the generated C is written beside the extension"); - let body = emitted_function(&emitted, "by_by_diff_strofint_shape_keys"); + let body = emitted_function( + &emitted, + &symbol("by", &["by_diff_strofint_shape", "keys"], ""), + ); // the conversion is fused, the argument's `PyLong` is gone with it, and the // resolution the fusion is guarded on is still made — through the memo, which // re-derives it whenever a namespace has been written to since the last answer assert!(body.contains("By_StrOfInt("), "{body}"); assert!(!body.contains("By_BoxInt("), "{body}"); assert!( - body.contains("By_LookupGlobalSite(&by_gs_str, by_module_dict, by_g_str)"), + body.contains("By_LookupGlobalSite(&by_gs_3str, by_module_dict, by_g_3str)"), "{body}" ); assert!( - body.contains("static ByGlobalSite by_gs_str = BY_GLOBAL_SITE_INIT;"), + body.contains("static ByGlobalSite by_gs_3str = BY_GLOBAL_SITE_INIT;"), "{body}" ); } @@ -42238,13 +43942,16 @@ def joined(n: int) -> str: // string the digits used to be built into is gone — and with it the second // allocation and the copy back out of it. the resolution the fast path is // guarded on is still made, through the same memo the unfused shape used - let body = emitted_function(&emitted, "by_by_diff_strconcatint_shape_keys"); + let body = emitted_function( + &emitted, + &symbol("by", &["by_diff_strconcatint_shape", "keys"], ""), + ); assert!(body.contains("By_StrConcatInt("), "{body}"); assert!(!body.contains("By_StrOfInt("), "{body}"); assert!(!body.contains("By_StrConcat("), "{body}"); assert!(!body.contains("By_UnboxStr("), "{body}"); assert!( - body.contains("By_LookupGlobalSite(&by_gs_str, by_module_dict, by_g_str)"), + body.contains("By_LookupGlobalSite(&by_gs_3str, by_module_dict, by_g_3str)"), "{body}" ); @@ -42252,7 +43959,10 @@ def joined(n: int) -> str: // resize in place, and a chain of those is linear where a chain of copies would // be quadratic — which is worth more than the one small allocation fusing it // would save - let body = emitted_function(&emitted, "by_by_diff_strconcatint_shape_joined"); + let body = emitted_function( + &emitted, + &symbol("by", &["by_diff_strconcatint_shape", "joined"], ""), + ); assert!(body.contains("By_StrAddAppend("), "{body}"); assert!(!body.contains("By_StrConcatInt("), "{body}"); } @@ -42299,7 +44009,10 @@ def once(i: int) -> str: let emitted = std::fs::read_to_string(dir.join("by_diff_strconcatint_shared.c")) .expect("the generated C is written beside the extension"); for name in ["twice", "once"] { - let body = emitted_function(&emitted, &format!("by_by_diff_strconcatint_shared_{name}")); + let body = emitted_function( + &emitted, + &symbol("by", &["by_diff_strconcatint_shared", name], ""), + ); assert!(body.contains("By_StrOfInt("), "{name}: {body}"); assert!(!body.contains("By_StrConcatInt("), "{name}: {body}"); } @@ -43632,6 +45345,62 @@ def discarded(f: Callable[[int], int], log: list[str]) -> None: ); } +#[test] +fn a_discarded_tuple_of_ints_from_a_rebound_function_lets_go_of_them_last_to_first() { + // once `ints` is rebound the call goes through python and the pair it answers is taken + // apart into two `int` slots, so nothing holds them together but the tuple the slots + // are put back into. an `int` subclass with a `__del__` sees when that tuple goes: + // python lets go of the second element first + agree_python( + "intrebounddrop", + "\ +from collections.abc import Callable + + +def ints(f: Callable[[int], int]) -> tuple[int, int]: + return f(1), f(2) + + +def discarded(f: Callable[[int], int], log: list[str]) -> None: + ints(f) + log.append('after') +", + &[ + "(lambda log, ints: (setattr(m, 'ints', lambda f: ints(f)), \ + m.discarded(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[2])([], m.ints)", + "(lambda log: (m.discarded(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[1])([])", + ], + ); +} + +#[test] +fn a_named_int_taken_out_of_a_tuple_lives_as_long_as_its_name() { + // `part` holds what the pair held, and an `int` subclass with a `__del__` sees when + // that goes: python lets go of it with the frame, after the last line has run, not + // once the last read of `part` is done + agree_python( + "intnamedelement", + "\ +from collections.abc import Callable + + +def split(f: Callable[[int], int]) -> tuple[str, int]: + return 'a', f(2) + + +def kept(f: Callable[[int], int], log: list[str]) -> int: + head, part = split(f) + total = part + 1 + log.append('kept') + return total +", + &["(lambda log: (m.kept(lambda i: type('B', (int,), \ + {'__del__': lambda s: log.append('del ' + str(int(s) >> 70))})(i << 70), log), log)[1])([])"], + ); +} + #[test] fn a_length_past_the_short_range_agrees() { // a length is a `Py_ssize_t`, and a tagged `int` holds one bit fewer, so a length an @@ -44500,6 +46269,74 @@ fn the_docstrings_are_read_off_compiled_definitions() { ); } +/// a class answers `__doc__` with the string its body opens with, and `None` where it opens +/// with none, and a written `__new__` with its own docstring. an emitted type is built from +/// tables, so both used to answer `None` — only a data class carried its docstring across. +/// `__new__` returns `Self`: a soundness check naming the class it is written in reads a name +/// the class statement has not bound yet, and leaves the method to its interpreted definition +#[test] +fn a_class_and_its_written_new_answer_their_docstrings() { + let Some(compiled) = agree_in( + "classdoc", + "\ +from typing import Self + + +class Box: + \"\"\"a box\"\"\" + + def __init__(self, v: int) -> None: + self.v = v + + def get(self) -> int: + return self.v + + +class Made: + \"\"\"made by its `__new__`\"\"\" + + def __new__(cls, v: int) -> Self: + \"\"\"the `__new__` that makes it\"\"\" + return object.__new__(cls) + + +class Plain: + def get(self) -> int: + return 1 + + +class Sub(Box): + pass +", + &[ + "m.Box.__doc__", + "m.Box(1).__doc__", + "m.Made.__doc__", + "m.Made.__new__.__doc__", + "m.Plain.__doc__", + "m.Sub.__doc__", + "type('Below', (m.Box,), {}).__doc__", + ], + false, + by_irbuild::Language::BasedPython, + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_classdoc as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.Box)['get']).__name__, type(vars(m.Made)['__new__'].__func__).__name__)\n", + ), + "method_descriptor builtin_function_or_method" + ); +} + /// a module-level function is installed on a class and called through an instance /// /// python's `function` is a descriptor, so a lookup through the class binds the @@ -45366,6 +47203,12 @@ fn install_census(python: &str, dir: &Path, module: &str, also: &str) -> String #[test] fn a_class_the_report_counts_says_whether_it_installed() { + // below 3.12 there is no way to append storage to a base's instance, so the module this + // builds stands down whole and none of its classes is the compiled type — + // `only_the_class_no_spec_can_build_is_left_interpreted` says so below 3.12 + if environment().is_some_and(|(_, toolchain)| !supports(&toolchain, (3, 12))) { + return; + } // `Rebound` puts a class-level value over a field whose presence the base's layout // left no room to record, and there is no second construction to try — so the pair // stands down together and both keep their interpreted definitions. `Held`'s own @@ -45457,84 +47300,93 @@ class Derived(Base): ); } -/// build `source` as python, import it, and hand back the directory it was built into and -/// the message the import refused with -fn refused_import(tag: &str, source: &str) -> Option<(Scratch, String)> { - let (python, toolchain) = environment()?; - let dir = Scratch::new(format!("by_diff_{tag}_r")); - let module = format!("by_diff_{tag}"); - let options = Options { - language: by_irbuild::Language::Python, - ..Options::default() - }; - if let Err(error) = build_source(source, module.as_str(), &toolchain, &dir, &options) { - assert!( - missing_toolchain(&error), - "{tag} failed to build: {error:#}" - ); - eprintln!("skipping {tag}: no working C toolchain ({error})"); - return None; - } - let message = run( +/// which build answers for `class` in the module built into `dir`: `method_descriptor` for +/// the compiled one, `function` where the module stood down to its interpreted definition +fn answering_build(dir: &Path, module: &str, class: &str, method: &str) -> Option { + let (python, _) = environment()?; + Some(run( &python, - &dir, + dir, &format!( - "try:\n\ - \x20 import {module}\n\ - except ImportError as error:\n\ - \x20 print(error)\n\ - else:\n\ - \x20 print('the import was not refused')\n" + "import {module} as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print(type(vars(m.{class})['{method}']).__name__)\n" ), - ); - Some((dir, message)) + )) } #[test] -fn a_mixture_left_on_a_base_the_module_replaced_refuses_the_import() { - // `dict` owns its instances and the emitted `Ours` owns its own, so the two cannot - // be mixed and the construction hands back the interpreted definition to stand as - // `AsDict` — which the module body built on the interpreted `Ours`, while the name - // `Ours` now means the emitted type. `isinstance(m.AsDict(), m.Ours)` is then False - // where python answers True. +fn a_mixture_left_on_a_base_the_module_replaced_stands_the_module_down() { + // `dict` owns its instances and the emitted `Ours` owns its own, so from 3.12 the two + // cannot be laid out as one and the construction hands back the interpreted definition + // to stand as `AsDict` — which the module body built on the interpreted `Ours`, while + // the name `Ours` already means the emitted type. `isinstance(m.AsDict(), m.Ours)` would + // then be False where python answers True: a wrong answer, and invisible to every other + // comparison of the two legs. // - // that is a wrong answer, not a slow one, and it is invisible to every comparison of - // the two legs: both classes still answer every call the same way, and both spell - // their `__mro__` with the same names. so the import refuses, and the module is - // unusable rather than quietly wrong until the compiler stands the whole family down - // instead. `logging/config.py` is the shape in the standard library, where - // `ConvertingDict(dict, ConvertingMixin)` is the same three lines - let Some((_dir, message)) = refused_import( - "orphanmix", - "\ + // whether the two can be laid out as one is the interpreter's own rule over whatever the + // bases' names hold at import, so only the import can tell. it used to refuse, which + // left the module unusable; it stands the whole module down to its interpreted + // definition instead, which answers as python does. `logging/config.py` is the shape in + // the standard library, where `ConvertingDict(dict, ConvertingMixin)` is the same three + // lines + // + // below 3.12 python lays out a class over a base whose instances keep nothing past a + // dict and weak references as if that base kept nothing at all, so `AsDict` stands on + // the emitted `Ours` and the module is the compiled one + let source = "\ class Ours: def side(self) -> str: return \"ours\" + def twice(self) -> str: + return self.side() + self.side() + class AsDict(dict, Ours): def label(self) -> str: return \"dict\" -", + + +def make() -> Ours: + return Ours() +"; + let Some(dir) = agree_python_built( + "orphanmix", + source, + &[ + "(isinstance(m.AsDict(), m.Ours), [c.__name__ for c in m.AsDict.__mro__])", + "[(d.label(), d.side(), d.twice(), d) for d in [m.AsDict(a=1, b=2, c=3)]]", + "[(setattr(d, 'extra', 4), d.extra, d.twice()) for d in [m.AsDict(a=1)]]", + "(isinstance(m.make(), m.Ours), m.make().twice())", + ], ) else { return; }; + let Some((_, toolchain)) = environment() else { + return; + }; + let expected = if supports(&toolchain, (3, 12)) { + "function" + } else { + "method_descriptor" + }; assert_eq!( - message, - "by_diff_orphanmix.AsDict stands on an orphaned base: by_diff_orphanmix.Ours is \ - what this module's own base of that name means and is not among its __bases__, \ - so isinstance answers False where python answers True" + answering_build(&dir, "by_diff_orphanmix", "Ours", "twice").as_deref(), + Some(expected) ); } #[test] -fn a_class_on_two_bases_of_this_modules_own_refuses_the_import() { +fn a_class_on_two_bases_of_this_modules_own_stands_the_module_down() { // the same failure without an outside base in sight: two emitted classes each own - // their instances, so `type(\"Mixin\", (Root, Extra), ...)` cannot lay one out either, - // and `Mixin` is left as the definition the body built on the two interpreted ones - let Some((_dir, message)) = refused_import( - "orphanpair", - "\ + // their instances, so `type(\"Mixin\", (Root, Extra), ...)` cannot lay one out either + // from 3.12, and `Mixin` would be left as the definition the body built on the two + // interpreted ones + // + // below 3.12 the two lay out as one, as they do for `AsDict` above, so `Mixin` stands on + // both emitted bases + let source = "\ class Root: def kind(self) -> str: return \"root\" @@ -45544,19 +47396,37 @@ class Extra: def extra(self) -> str: return \"extra\" + def both(self) -> str: + return self.extra() + self.extra() + class Mixin(Root, Extra): def tag(self) -> int: return 1 -", +"; + let Some(dir) = agree_python_built( + "orphanpair", + source, + &[ + "(isinstance(m.Mixin(), m.Root), isinstance(m.Mixin(), m.Extra))", + "[c.__name__ for c in m.Mixin.__mro__]", + "[(x.kind(), x.extra(), x.both(), x.tag()) for x in [m.Mixin()]]", + "[(setattr(x, 'extra_field', 4), x.extra_field, x.both()) for x in [m.Mixin()]]", + ], ) else { return; }; + let Some((_, toolchain)) = environment() else { + return; + }; + let expected = if supports(&toolchain, (3, 12)) { + "function" + } else { + "method_descriptor" + }; assert_eq!( - message, - "by_diff_orphanpair.Mixin stands on an orphaned base: by_diff_orphanpair.Root is \ - what this module's own base of that name means and is not among its __bases__, \ - so isinstance answers False where python answers True" + answering_build(&dir, "by_diff_orphanpair", "Extra", "both").as_deref(), + Some(expected) ); } @@ -45732,3 +47602,46 @@ def name_of(c: Color) -> str: ], ); } + +#[test] +fn a_compiled_class_is_named_bare_in_python_s_messages() { + // python's own messages name a type by its `tp_name`, which a `class` statement leaves + // the bare name: `'Box' object is not callable`. a compiled class answers the same, + // while its `repr`, `__module__` and pickling still carry the module + let Some(compiled) = agree_python_built( + "barename", + "\ +class Box: + def __init__(self, v: int) -> None: + self.v = v + + +class Small(Box): + pass +", + &[ + "[str(_capture(f)) for f in (lambda: m.Box(1)(), lambda: m.Small(1)(), \ + lambda: [*m.Box(1)], lambda: (lambda a, b: 0)(*m.Box(1)), \ + lambda: [x for x, y in [m.Box(1)]], lambda: len(m.Box(1)), lambda: iter(m.Box(1)), \ + lambda: m.Box(1)[0], lambda: m.Box(1) + 1)]", + "(repr(m.Box), repr(m.Small), m.Box.__module__, m.Box.__name__, m.Box.__qualname__)", + "(repr(m.Box(1)).split(' at ')[0], type(m.Small(1)).__name__)", + "(lambda p: p.loads(p.dumps(m.Small(3))).v)(__import__('pickle'))", + ], + ) else { + return; + }; + let Some((python, _)) = environment() else { + return; + }; + assert_eq!( + run( + &python, + &compiled, + "import by_diff_barename as m\n\ + assert m.__file__.endswith(('.so', '.pyd')), m.__file__\n\ + print([type(c.__dict__['__init__']).__name__ for c in (m.Box,)], type(m.Small.__init__).__name__)\n", + ), + "['wrapper_descriptor'] wrapper_descriptor" + ); +} diff --git a/crates/by_codegen_c/src/lib.rs b/crates/by_codegen_c/src/lib.rs index 50f69bfc23..b74638fc90 100644 --- a/crates/by_codegen_c/src/lib.rs +++ b/crates/by_codegen_c/src/lib.rs @@ -30,6 +30,7 @@ use by_ir::function::{ Binding, ClassBase, ClassIr, Function, KeywordValue, ModuleIr, PropertyIr, RegisterDecl, SlotAlias, Surface, cleaned_doc, }; +use by_ir::mangle::mangle; use by_ir::ops::{ BinOp, BlockId, CmpOp, Concatenation, LicenceKind, Mutation, Op, RegisterId, SoundTarget, Terminator, UnaryOp, Value, @@ -1109,11 +1110,8 @@ fn emit_absent_values_new(module: &ModuleIr, class: &ClassIr) -> String { format!( "static PyObject *{owner_name}_alloc(PyTypeObject *by_type, Py_ssize_t by_items);\n\ static PyObject *{type_name}_new(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {{\n\ - \x20 (void)self; (void)kwnames;\n\ - \x20 PyTypeObject *by_type = By_NewTarget({type_name}_OBJ, args, PyVectorcall_NARGS(nargs));\n\ - \x20 if (by_type == NULL) return NULL;\n\ - \x20 By_AdoptAllocator(by_type, {type_name}_OBJ, {owner_name}_alloc);\n\ - \x20 return PyType_GenericNew(by_type, NULL, NULL);\n}}\n\n\ + \x20 (void)self;\n\ + \x20 return By_ObjectNew({type_name}_OBJ, {owner_name}_alloc, args, nargs, kwnames);\n}}\n\n\ static PyMethodDef {type_name}_new_def =\n\ \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){type_name}_new, METH_FASTCALL | METH_KEYWORDS, NULL}};\n\n" ) @@ -1538,6 +1536,18 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { }), i32::from(class.inherited_init) ); + // python runs the `__init__` its class was given — the one `@dataclass` generates, or + // the one the transpiler writes for an accessor block's storage — in a frame of its own, + // and pushing that frame is where a construction at the recursion limit raises. nothing + // here calls back into python, so the frame is taken and given straight back. a class + // with no `__init__` of its own reaches `object.__init__`, which pushes none + if !class.inherited_init { + out.push_str( + " { ByDepth by_depth = By_DepthHere();\n\ + \x20 if (BY_UNLIKELY(By_DepthEnter(by_depth))) return -1;\n\ + \x20 By_DepthLeave(by_depth); }\n", + ); + } // a field the constructor takes no parameter for but the class body gave a value still // has to be written, and an accessor block's storage is that: the twin writes it at // the top of an `__init__` the transpiler injected for exactly this @@ -1594,9 +1604,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { error_check(&field.ty, "by_v") ); let storage = storage_name(module, class, &field.name); - let release = release_old(field, &storage); - if !release.is_empty() { - let _ = writeln!(out, " {release}"); + let (take, release) = old_value(module, field, &storage); + if !take.is_empty() { + let _ = writeln!(out, " {take}"); } let _ = writeln!(out, " {storage}->{} = by_v;", field.member()); // the byte beside a field is what a later read and the deallocation both ask, @@ -1613,6 +1623,9 @@ fn emit_class_type(module: &ModuleIr, class: &ClassIr) -> String { field, &format!("{storage}->{}", field.member()), )); + if !release.is_empty() { + let _ = writeln!(out, " {release}"); + } out.push_str(" }\n"); } out.push_str(" return 0;\n}\n\n"); @@ -1806,11 +1819,23 @@ fn emit_written_init(module: &ModuleIr, init: &Function) -> String { .params() .first() .map_or_else(|| "PyObject *".to_string(), |decl| ctype(module, &decl.ty)); + // the frame python pushes for the `__init__` a construction runs, counted as the + // method's own wrapper counts it — see `emit_wrapper`. a body on a cycle is handed the + // depth, so the calls round the cycle count against the same one + let (body, depth) = if is_recursive(init) { + (depth_symbol(module, init), "by_depth, ") + } else { + (init.native_symbol(module.name.dotted()), "") + }; + out.push_str( + " ByDepth by_depth = By_DepthHere();\n\ + \x20 if (BY_UNLIKELY(By_DepthEnter(by_depth))) goto by_init_error;\n", + ); let _ = writeln!( out, - " {{ {} by_result = {}(({receiver})selfobj{arguments});", + " {{ {} by_result = {body}({depth}({receiver})selfobj{arguments});\n\ + \x20 By_DepthLeave(by_depth);", ctype(module, &init.ret), - init.native_symbol(module.name.dotted()) ); if init.convention.can_fail() { let _ = writeln!( @@ -1854,19 +1879,34 @@ fn emit_class_getitem(module: &ModuleIr, class: &ClassIr, type_name: &str) -> St ) } -/// releasing what a field held before a setter overwrites it +/// what a field held before a store overwrites it, taken aside, and the release of it once +/// the new value is in place — empty strings for a field that holds no reference /// -/// an optional one may hold nothing at all, and the byte beside it is what says so — -/// releasing a value that was never written would be releasing whatever `tp_alloc` -/// left there -fn release_old(field: &by_ir::function::FieldDecl, storage: &str) -> String { - let Some(release) = dec_ref(&field.ty, &format!("{storage}->{}", field.member())) else { - return String::new(); +/// the release comes after the store, as python's own store does it: letting go of the +/// old value can run a `__del__`, and one that reads or deletes this same field has to +/// find the new value there rather than the one being let go of. an optional field may +/// hold nothing at all, and the byte beside it is what says so — releasing a value that +/// was never written would be releasing whatever `tp_alloc` left there +fn old_value( + module: &ModuleIr, + field: &by_ir::function::FieldDecl, + storage: &str, +) -> (String, String) { + let Some(release) = dec_ref(&field.ty, "by_old") else { + return (String::new(), String::new()); }; + let member = format!("{storage}->{}", field.member()); + let ty = ctype(module, &field.ty); if field.optional { - format!("if ({storage}->{}) {{ {release} }}", field.presence()) + ( + format!( + "{ty} by_old = {member}; char by_held = {storage}->{};", + field.presence() + ), + format!("if (by_held) {{ {release} }}"), + ) } else { - release + (format!("{ty} by_old = {member};"), release) } } @@ -1913,11 +1953,15 @@ fn mark_present(field: &by_ir::function::FieldDecl, storage: &str) -> String { /// `del obj.field`, which python reaches through the setter with no value /// -/// the presence byte is what makes this expressible at all: clearing it puts the field -/// back into the state `tp_alloc` left it in, which a read already answers with -/// `AttributeError` and a later write already sets again. so the delete is the write -/// read backwards, and a field with no byte — one `__init__` assigns on every path and -/// nothing deletes — has no absent state to return to and keeps refusing. +/// every field python can name has an absent state, because `C.__new__(C)` makes an +/// instance `__init__` never ran on: an optional field's byte clear, an object's `NULL`, a +/// tagged `int`'s error value. a read already answers that state with `AttributeError` and +/// a later write already sets the field again, so the delete is the write read backwards. +/// a field python cannot name — one the compiler made up, a closure's shared cell — has no +/// such state and keeps refusing. +/// +/// deleting an attribute the instance does not have raises python's own `AttributeError`, +/// which names only the attribute where the class keeps it in a slot. /// /// the order is the one `PyObject_GenericSetAttr` uses on a slot, and it is not /// cosmetic: releasing the old value can run a `__del__` that reaches back into this @@ -1932,29 +1976,50 @@ fn delete_field( field: &by_ir::function::FieldDecl, storage: &str, ) -> String { - if !field.optional { + let Some(absent) = field_absent(field, storage) else { return "\x20 if (by_value == NULL) {\n\ \x20 PyErr_SetString(PyExc_AttributeError, \"cannot delete an attribute\");\n\ \x20 return -1;\n\ \x20 }\n" .to_string(); - } - let member = field.member(); + }; + let member = format!("{storage}->{}", field.member()); + // the state `field_absent` reads as absent, which is the state `tp_alloc` and the + // instance's own unset left the field in + let clear = if field.has_presence_byte() { + format!( + "\x20 memset(&{member}, 0, sizeof({member}));\n\ + \x20 {storage}->{} = 0;\n", + field.presence() + ) + } else if field.ty == RType::INT { + format!("\x20 {member} = BY_INT_ERROR;\n") + } else { + format!("\x20 {member} = NULL;\n") + }; + let missing = if owner.slots.is_some() && !instance_dict(module, owner) { + format!( + "PyErr_SetString(PyExc_AttributeError, {});", + c_string(&field.name) + ) + } else { + format!( + "PyErr_Format(PyExc_AttributeError, \"'%s' object has no attribute '%s'\", \ + By_TypeName(selfobj), {});", + c_string(&field.name) + ) + }; let release = dec_ref(&field.ty, "by_old") .map_or_else(String::new, |release| format!("\x20 {release}\n")); format!( "\x20 if (by_value == NULL) {{\n\ - \x20 if (!{storage}->{presence}) {{ PyErr_Format(PyExc_AttributeError, \ - \"'%s' object has no attribute '%s'\", By_TypeName(selfobj), {name}); return -1; }}\n\ - \x20 {ty} by_old = {storage}->{member};\n\ - \x20 memset(&{storage}->{member}, 0, sizeof({storage}->{member}));\n\ - \x20 {storage}->{presence} = 0;\n\ + \x20 if ({absent}) {{ {missing} return -1; }}\n\ + \x20 {ty} by_old = {member};\n\ + {clear}\ {unpublish}\ {release}\ \x20 return 0;\n\ \x20 }}\n", - presence = field.presence(), - name = c_string(&field.name), ty = ctype(module, &field.ty), // before the release, for the reason the order above is what it is: a `__del__` // the release runs can read this instance, and it must not find the mapping @@ -1975,7 +2040,7 @@ fn delete_field( /// the C identifier the two halves of a property are reached through fn property_symbol(type_name: &str, name: &str) -> String { - format!("{type_name}_prop_{name}") + format!("{type_name}_prop_{}", mangle(name)) } /// the three halves of a property, each as the C identifier its `PyMethodDef` goes under @@ -2055,7 +2120,7 @@ fn emit_initializing_setter( "static int {type_name}_set_{name}(PyObject *selfobj, PyObject *by_value, void *closure) {{\n\ \x20 (void)by_value; (void)closure;\n\ \x20 return By_FieldNotWritable(selfobj, {quoted});\n}}\n", - name = field.name, + name = mangle(&field.name), quoted = c_string(&field.name), ); }; @@ -2070,7 +2135,7 @@ fn emit_initializing_setter( {present}\ {publish}\ \x20 return 0;\n}}\n", - name = field.name, + name = mangle(&field.name), quoted = c_string(&field.name), bind = bind_self(module, owner, "selfobj"), ty = ctype(module, &field.ty), @@ -2108,7 +2173,7 @@ fn emit_dataclass_members(class: &ClassIr, type_name: &str) -> String { out, " {{{}, {type_name}_get_{}}},", c_string(&field.name), - field.name + mangle(&field.name) ); } out.push_str(" {NULL, NULL}\n};\n"); @@ -2161,7 +2226,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!( out, "static PyObject *{type_name}_get_{}(PyObject *selfobj, void *closure) {{\n (void)closure;\n {}\n{absent} return {};\n}}", - field.name, + mangle(&field.name), bind_self(module, field_owner(module, class, &field.name), "selfobj"), box_borrowed(&field.ty, &format!("self->{}", field.member())) ); @@ -2179,7 +2244,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { "static int {type_name}_has_{}(PyObject *selfobj) {{\n\ \x20 {}\n\ \x20 return self->{} != 0;\n}}", - field.name, + mangle(&field.name), bind_self(module, field_owner(module, class, &field.name), "selfobj"), field.presence() ); @@ -2192,6 +2257,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out.push_str(&emit_initializing_setter(module, class, &type_name, field)); continue; } + let (take, release) = old_value(module, field, "self"); let _ = writeln!( out, "static int {type_name}_set_{}(PyObject *selfobj, PyObject *by_value, void *closure) {{\n\ @@ -2200,12 +2266,13 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { {}\ \x20 {} by_v = {};\n\ \x20 if ({}) return -1;\n\ - \x20 {}\n\ + \x20 {take}\n\ \x20 self->{} = by_v;\n\ {}\ {}\ + \x20 {release}\n\ \x20 return 0;\n}}\n", - field.name, + mangle(&field.name), bind_self(module, field_owner(module, class, &field.name), "selfobj"), delete_field( module, @@ -2216,7 +2283,6 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ctype(module, &field.ty), unbox_checked(module, &field.ty, "by_value"), error_check(&field.ty, "by_v"), - release_old(field, "self"), field.member(), mark_present(field, "self"), publish_field( @@ -2266,14 +2332,14 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { ); for field in dict_fields(module, class) { let present = if field_default(module, class, field).is_some() { - format!("{type_name}_has_{}", field.name) + format!("{type_name}_has_{}", mangle(&field.name)) } else { "NULL".to_string() }; // the same setter the field's descriptor publishes, and the one condition it // is emitted under let set = if class.exported { - format!("{type_name}_set_{}", field.name) + format!("{type_name}_set_{}", mangle(&field.name)) } else { "NULL".to_string() }; @@ -2329,14 +2395,15 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { continue; } let setter = if class.exported { - format!("{type_name}_set_{}", field.name) + format!("{type_name}_set_{}", mangle(&field.name)) } else { "NULL".to_string() }; let _ = writeln!( out, - " {{\"{}\", {type_name}_get_{}, {setter}, NULL, NULL}},", - field.name, field.name + " {{{}, {type_name}_get_{}, {setter}, NULL, NULL}},", + c_string(&field.name), + mangle(&field.name) ); } // a class whose whole state is the dict answers `__dict__` with the dict itself, @@ -2676,7 +2743,8 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { let _ = writeln!( out, "static PyMethodDef {type_name}_new_def =\n\ - \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){symbol}, METH_FASTCALL | METH_KEYWORDS, NULL}};\n" + \x20 {{\"__new__\", (PyCFunction)(void(*)(void)){symbol}, METH_FASTCALL | METH_KEYWORDS, {}}};\n", + method_doc(new) ); } @@ -2759,8 +2827,20 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 PyObject *by_value;\n\ }} {type_name}_asend;\n\ static PyTypeObject {type_name}_asend_type;\n\ + /* one dropped before it was ever stepped is reported, as python reports it from 3.13.\n\ + \x20* it is untracked first: the warning runs python, and a collection it sets off must not\n\ + \x20* find an object already being freed */\n\ static void {type_name}_asend_dealloc(PyObject *self) {{\n\ \x20 PyObject_GC_UnTrack(self);\n\ + #if PY_VERSION_HEX >= 0x030D0000\n\ + \x20 {type_name}_asend *by_self = ({type_name}_asend *)self;\n\ + \x20 if (by_self->by_state == 0)\n\ + \x20 By_WarnUnawaitedAgenMethod((PyObject *)by_self->by_gen,\n\ + \x20 by_self->by_mode == 1 ? \"aclose\"\n\ + \x20 : by_self->by_mode == 3 ? \"athrow\"\n\ + \x20 : \"asend\",\n\ + \x20 {qualname});\n\ + #endif\n\ \x20 Py_XDECREF((PyObject *)(({type_name}_asend *)self)->by_gen);\n\ \x20 Py_XDECREF((({type_name}_asend *)self)->by_value);\n\ \x20 Py_TYPE(self)->tp_free(self);\n\ @@ -2793,9 +2873,14 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 by_gen->by_running_async = 0;\n\ \x20 return NULL;\n\ }}\n\ - /* the message a second awaitable is refused with while one is stepping the generator */\n\ + /* the message a second awaitable is refused with while one is stepping the generator.\n\ + \x20* 3.11 leaves an `__anext__()` or `asend()` it refused as it was, to be tried again */\n\ static void {type_name}_refuse_running({type_name}_asend *by_self) {{\n\ + #if PY_VERSION_HEX >= 0x030C0000\n\ \x20 by_self->by_state = 2;\n\ + #else\n\ + \x20 if (by_self->by_mode == 1 || by_self->by_mode == 3) by_self->by_state = 2;\n\ + #endif\n\ \x20 PyErr_Format(PyExc_RuntimeError, \"%s(): asynchronous generator is already running\",\n\ \x20 by_self->by_mode == 1 ? \"aclose\" : by_self->by_mode == 3 ? \"athrow\" : \"anext\");\n\ }}\n\ @@ -2919,6 +3004,8 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { static PyObject *{type_name}_asend_next(PyObject *self) {{\n\ \x20 return {type_name}_asend_send(self, NULL);\n\ }}\n\ + /* 3.11's throw neither refuses a generator another awaitable is stepping nor marks this\n\ + \x20* one as being stepped, and an `athrow()` or `aclose()` it ended is not finished by it */\n\ static PyObject *{type_name}_asend_throw(PyObject *self, PyObject *const *args, Py_ssize_t nargs) {{\n\ \x20 {type_name}_asend *by_self = ({type_name}_asend *)self;\n\ \x20 {struct_name} *by_gen = by_self->by_gen;\n\ @@ -2927,6 +3014,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 {type_name}_refuse_reuse(by_self);\n\ \x20 return NULL;\n\ \x20 }}\n\ + #if PY_VERSION_HEX >= 0x030C0000\n\ \x20 if (by_self->by_state == 0) {{\n\ \x20 if (by_gen->by_running_async) {{\n\ \x20 {type_name}_refuse_running(by_self);\n\ @@ -2935,13 +3023,18 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 by_self->by_state = 1;\n\ \x20 by_gen->by_running_async = 1;\n\ \x20 }}\n\ + #endif\n\ \x20 PyObject *by_step = {type_name}_gen_throw(by_gen, args, nargs);\n\ \x20 if (!by_closing) {{\n\ \x20 by_step = {type_name}_unwrap(by_gen, by_step);\n\ + #if PY_VERSION_HEX >= 0x030C0000\n\ \x20 if (by_step == NULL) {{\n\ \x20 by_gen->by_running_async = 0;\n\ \x20 by_self->by_state = 2;\n\ \x20 }}\n\ + #else\n\ + \x20 if (by_step == NULL && by_self->by_mode != 3) by_self->by_state = 2;\n\ + #endif\n\ \x20 return by_step;\n\ \x20 }}\n\ \x20 if (by_step != NULL && by_gen->{kind} == By_ShortFrom(1)) {{\n\ @@ -2952,8 +3045,10 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 return NULL;\n\ \x20 }}\n\ \x20 if (by_step != NULL) return by_step;\n\ + #if PY_VERSION_HEX >= 0x030C0000\n\ \x20 by_gen->by_running_async = 0;\n\ \x20 by_self->by_state = 2;\n\ + #endif\n\ \x20 By_EndAsyncIteration();\n\ \x20 if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)\n\ \x20 || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {{\n\ @@ -2962,8 +3057,14 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 }}\n\ \x20 return NULL;\n\ }}\n\ + /* before 3.13 closing one of these awaitables only finishes it, and throws nothing into\n\ + \x20* the generator */\n\ static PyObject *{type_name}_asend_close(PyObject *self, PyObject *by_unused) {{\n\ \x20 (void)by_unused;\n\ + #if PY_VERSION_HEX < 0x030D0000\n\ + \x20 (({type_name}_asend *)self)->by_state = 2;\n\ + \x20 Py_RETURN_NONE;\n\ + #else\n\ \x20 if ((({type_name}_asend *)self)->by_state == 2) Py_RETURN_NONE;\n\ \x20 PyObject *by_exit = PyExc_GeneratorExit;\n\ \x20 PyObject *by_step = {type_name}_asend_throw(self, &by_exit, 1);\n\ @@ -2979,6 +3080,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { \x20 Py_RETURN_NONE;\n\ \x20 }}\n\ \x20 return NULL;\n\ + #endif\n\ }}\n\ static PyMethodDef {type_name}_asend_methods[] = {{\n\ \x20 {{\"send\", (PyCFunction){type_name}_asend_send, METH_O, NULL}},\n\ @@ -3053,6 +3155,7 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { thrown = mangle_member(crate::GENERATOR_THROWN), state = mangle_member(crate::GENERATOR_STATE), dotted = dotted, + qualname = c_string(&resume.qualname), ); format!( " .tp_as_async = &{type_name}_async,\n .tp_finalize = {type_name}_finalize,\n" @@ -3294,34 +3397,6 @@ fn emit_class_members(module: &ModuleIr, class: &ClassIr) -> String { out } -/// whether this class or an in-module base of it carries a class decorator -/// -/// the base chain, because an instance discipline is not a per-class answer: a subclass -/// allocates and frees instances of a shape its base decided. -/// -/// a `data class` rung does not count. `@dataclass(slots=True)` is the twin's innermost -/// decorator, so every decorator written above it is handed a class whose instances have -/// no dict in python either -fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { - let mut current = class; - // bounded by the class count, for the reason `inherits_layout` gives - for _ in 0..=module.classes.len() { - if !current.decorators.is_empty() && !current.dataclass { - return true; - } - match current - .base - .as_ref() - .and_then(ClassBase::in_module) - .and_then(|name| class_named(module, name)) - { - Some(next) => current = next, - None => return false, - } - } - false -} - /// whether an emitted instance keeps a `__dict__` of its own /// /// python gives an instance somewhere to put a name its class never mentioned, and an @@ -3336,11 +3411,11 @@ fn decorated_chain(module: &ModuleIr, class: &ClassIr) -> bool { /// right — a compiled class that accepted what its twin refuses would be a divergence in /// the other direction. /// -/// a class decorator is a second reason, and an older one. what a decorator hands back -/// is often code it *generated* from what it read — `@dataclass` writes an `__init__` -/// that assigns one attribute per annotation — and that code is ordinary python which -/// assumes an ordinary instance, so on a bare layout every assignment fell off and -/// `E(3)` raised. +/// a class decorator is handed the class the `class` statement made, so it is no reason +/// of its own. what one hands back is often code it *generated* from what it read — +/// `@dataclass` writes an `__init__` that assigns one attribute per annotation — and that +/// code assumes an ordinary instance, which is one with a dict unless every rung declares +/// `__slots__`. one that does has no dict in python either, whatever decorates it. /// /// the dict is a word in the class's own struct, at an offset the type names through a /// `__dictoffset__` member. that is what lets a call site ask whether an instance shadows @@ -3359,7 +3434,7 @@ fn instance_dict(module: &ModuleIr, class: &ClassIr) -> bool { class.exported && heap_type(class) && !inherits_layout(module, class) - && (decorated_chain(module, class) || !slots_declared_throughout(module, class)) + && !slots_declared_throughout(module, class) } /// whether this class's spec tells the type where an instance keeps its dict or its @@ -4274,6 +4349,15 @@ fn class_named<'a>(module: &'a ModuleIr, name: &str) -> Option<&'a ClassIr> { module.classes.iter().find(|class| class.name == name) } +/// what an operation naming a class this module does not emit is written as +/// +/// the verifier refuses such an operation, so nothing should reach this. emitting nothing +/// would leave the destination unwritten and every later read of it a read of garbage, +/// so the C refuses to preprocess instead +fn unemitted_class(name: &str, what: &str) -> String { + format!("#error \"{what} names `{name}`, which this module does not emit\"\n") +} + /// whether `name` is a dunder, which is the shape of every name an emitted type can /// answer through a slot rather than through its method table fn is_dunder(name: &str) -> bool { @@ -6160,6 +6244,20 @@ thread_local! { std::cell::RefCell::new(HashSet::new()); } +/// whether a call to a compiled `__init__` is the second half of a construction: its +/// receiver is an instance this frame allocated itself, which is the `tp_alloc` python's +/// call of the type makes before it runs `__init__`. an `__init__` called on any other +/// receiver — a base's, reached through `super()` — is an ordinary method call +fn constructs(function: &Function, owner: Option<&str>, callee: &str, args: &[Value]) -> bool { + let (Some(owner), Some(Value::Register(receiver))) = (owner, args.first()) else { + return false; + }; + callee == "__init__" + && function.blocks.iter().flat_map(|block| &block.ops).any(|op| { + matches!(op, Op::NewInstance { dest, class, .. } if dest == receiver && class == owner) + }) +} + fn is_recursive(function: &Function) -> bool { RECURSIVE.with_borrow(|recursive| recursive.contains(&function.qualified_name())) } @@ -7550,12 +7648,14 @@ fn commit_checked(function: &Function, dest: RegisterId, error_target: ErrorEdge /// owes `by_fn` a release fn resolve_str(error_target: ErrorEdge) -> String { let label = error_label(error_target); + let slot = global_name_slot("str"); + let site = global_site("str"); format!( - " {{ static PyObject *by_g_str = NULL; PyObject *by_fn = NULL;\n \ - if (by_g_str == NULL) by_g_str = By_InternedStr(\"str\", 3);\n \ - if (by_g_str == NULL) goto {label};\n \ - static ByGlobalSite by_gs_str = BY_GLOBAL_SITE_INIT;\n \ - by_fn = By_LookupGlobalSite(&by_gs_str, by_module_dict, by_g_str);\n \ + " {{ static PyObject *{slot} = NULL; PyObject *by_fn = NULL;\n \ + if ({slot} == NULL) {slot} = By_InternedStr(\"str\", 3);\n \ + if ({slot} == NULL) goto {label};\n \ + static ByGlobalSite {site} = BY_GLOBAL_SITE_INIT;\n \ + by_fn = By_LookupGlobalSite(&{site}, by_module_dict, {slot});\n \ if (by_fn == NULL) goto {label};\n" ) } @@ -8095,11 +8195,12 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE return refuse("no class in the chain publishes the field"); }; format!( - " By_RecheckField((PyObject *){}, {type_name}_OBJ, {}, {}, {}_get_{member});\n", + " By_RecheckField((PyObject *){}, {type_name}_OBJ, {}, {}, {}_get_{});\n", value_expr(src), c_string(class), c_string(member), publisher.type_name(module.name.dotted()), + mangle(member), ) } } @@ -8486,6 +8587,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE args, } => { let target = native_callee(module, owner.as_deref(), callee); + let construction = constructs(function, owner.as_deref(), callee, args); // an argument may be a *subclass* of what the callee declares — the // layouts line up, so the pointer is valid, but C wants to be told let args = args @@ -8514,6 +8616,13 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE Some(target) => target.native_symbol(module.name.dotted()), None => format!("by_{}_{}", mangle(module.name.dotted()), mangle(callee)), }; + // a construction is the type's call as well as the frame of the `__init__` it + // runs, and 3.11 counts that call against the same limit + let (enter, leave) = if construction { + ("By_DepthEnterConstruction", "By_DepthLeaveConstruction") + } else { + ("By_DepthEnter", "By_DepthLeave") + }; let call = match (counted, target) { (true, Some(target)) => { let args = if args.is_empty() { @@ -8522,7 +8631,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE format!("by_depth, {args}") }; format!( - "({{ {ret} by_called = {symbol}({args}); By_DepthLeave(by_depth); by_called; }})", + "({{ {ret} by_called = {symbol}({args}); {leave}(by_depth); by_called; }})", ret = ctype(module, &target.ret) ) } @@ -8530,7 +8639,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }; let enter = if counted { format!( - " if (BY_UNLIKELY(By_DepthEnter(by_depth))) goto {};\n", + " if (BY_UNLIKELY({enter}(by_depth))) goto {};\n", error_label(error_target) ) } else { @@ -9011,12 +9120,8 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE class, fields, } => { - let Some(owner) = module - .classes - .iter() - .find(|candidate| candidate.name == *class) - else { - return String::new(); + let Some(owner) = class_named(module, class) else { + return unemitted_class(class, "an allocation"); }; let struct_name = owner.struct_name(module.name.dotted()); let type_name = owner.type_name(module.name.dotted()); @@ -9238,10 +9343,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE method, env, } => { - let Some(nested) = module - .classes - .iter() - .find(|candidate| candidate.name == *class) + let Some(nested) = class_named(module, class) .and_then(|owner| { owner .methods @@ -9250,7 +9352,7 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE }) .filter(|nested| nested.nested.is_some()) else { - return String::new(); + return unemitted_class(&format!("{class}.{method}"), "a nested function"); }; let made = format!( "By_MakeFunction(&{}_spec, (PyObject *)({}))", @@ -9371,12 +9473,8 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE out } Op::LoadClass { dest, class } => { - let Some(owner) = module - .classes - .iter() - .find(|candidate| candidate.name == *class) - else { - return String::new(); + let Some(owner) = class_named(module, class) else { + return unemitted_class(class, "a class read"); }; let type_name = owner.type_name(module.name.dotted()); assign_owned( @@ -9734,22 +9832,24 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE if !*moves && let Some(retain) = inc_ref(&ty, &value_expr(value)) { let _ = writeln!(out, " {retain}"); } - // the release reads the old value, so an optional field that was never - // written must not be released: the byte says whether there is one - if let Some(release) = dec_ref(&ty, &target) { - match field_decl(module, class, field) { - Some(decl) if decl.optional => { - let _ = writeln!( - out, - " if ({}->{}) {{ {release} }}", - fields, - decl.presence() - ); - } - _ => { - let _ = writeln!(out, " {release}"); - } - } + // the value it replaces is let go of only once the new one is in place, as + // python's own store does: letting go can run a `__del__`, and one that reads + // or deletes this field has to find the new value. an optional field that was + // never written has nothing to let go of, and its byte says whether there is one + let release = + dec_ref(&ty, "by_old").map(|release| match field_decl(module, class, field) { + Some(decl) if decl.optional => ( + format!(" char by_held = {fields}->{};", decl.presence()), + format!("if (by_held) {{ {release} }}"), + ), + _ => (String::new(), release), + }); + if let Some((held, _)) = &release { + let _ = writeln!( + out, + " {{ {} by_old = {target};{held}", + ctype(module, &ty) + ); } let _ = writeln!(out, " {target} = {};", value_expr(value)); // and the register is left empty, so the release every later exit makes of it @@ -9768,6 +9868,9 @@ fn emit_op(module: &ModuleIr, function: &Function, op: &Op, error_target: ErrorE { out.push_str(&publish_field(module, owner, &fields, decl, &target)); } + if let Some((_, release)) = &release { + let _ = writeln!(out, " {release} }}"); + } out } Op::GetAttr { @@ -11515,6 +11618,92 @@ fn twin_decorators(module: &ModuleIr) -> String { out } +/// what an exit from module init that leaves the whole module interpreted has to undo, +/// beyond the names it installed +struct ModuleUnwind<'a> { + /// how many interpreted definitions `by_twin` holds + twin_count: usize, + record_interpreted: &'a str, + drop_installer: &'a str, + release_bodies: &'a str, +} + +/// the test after `class`'s construction that stands the whole module down where the +/// construction handed back the interpreted definition while a base of ours is the +/// emitted type, or nothing where no base of ours has a twin slot to ask +/// +/// that definition stands on the interpreted copy of the base, and the name of the base +/// already means the emitted type, so `isinstance` would answer False where python +/// answers True. a class on a base of ours beside one from outside is built by calling its +/// metaclass, which hands the definition back where the interpreter cannot lay the bases +/// out as one — a `dict` beside a class of ours, from 3.12. only the import can know: the +/// bases are whatever their names hold by then, and whether two of them can be laid out as +/// one is the interpreter's own rule. nothing has been decorated or adopted yet, so every +/// name installed so far is given back its interpreted definition and the module is left +/// as the interpreted one +fn standing_down_over_an_orphaned_base( + module: &ModuleIr, + class: &ClassIr, + twins: &[&ClassIr], + slot: usize, + unwind: &ModuleUnwind<'_>, +) -> String { + // a base that was itself handed back its interpreted definition is what this one's + // stands on already + let emitted_bases = class + .base + .iter() + .flat_map(ClassBase::plain_names) + .filter_map(|name| { + twins + .iter() + .take(slot) + .rposition(|candidate| candidate.name == name) + }) + .map(|base| format!("(by_type[{base}] != NULL && by_type[{base}] != by_twin[{base}])")) + .collect::>(); + if emitted_bases.is_empty() { + return String::new(); + } + let mut restore = String::new(); + for (earlier, installed) in twins.iter().enumerate().take(slot) { + let name = c_string(&installed.name); + let _ = writeln!( + restore, + "\x20 if (by_type[{earlier}] != NULL) {{\n\ + \x20 if (by_twin[{earlier}] != NULL) {{\n\ + \x20 if (PyDict_SetItemString(dict, {name}, by_twin[{earlier}]) < 0) return -1;\n\ + \x20 }} else {{\n\ + \x20 By_DropName(dict, {name});\n\ + \x20 }}\n\ + \x20 }}" + ); + } + let release_forwarders = if module.shims.is_some() { + "\x20 Py_DECREF(by_built);\n" + } else { + "" + }; + let type_name = class.type_name(module.name.dotted()); + format!( + " if ({type_name} == by_twin[{slot}] && ({})) {{\n\ + {restore}\ + \x20 Py_CLEAR({type_name});\n\ + {}{}{}\ + \x20 for (Py_ssize_t by_at = 0; by_at < {}; by_at++) Py_XDECREF(by_twin[by_at]);\n\ + \x20 Py_CLEAR(by_twins.moved);\n\ + {release_forwarders}{}\ + \x20 return 0;\n\ + \x20 }}\n", + emitted_bases.join(" || "), + unwind.record_interpreted, + twin_decorators(module), + unwind.drop_installer, + unwind.twin_count, + unwind.release_bodies, + ) +} + /// the classes module init publishes, in the order the twin arrays hold them /// /// this is the same walk `emit_module_init` uses to lay those arrays out, so a class's @@ -11656,16 +11845,18 @@ fn emit_install_verification(module: &ModuleIr) -> String { /// whether init has an exit that leaves the *whole* module interpreted /// -/// two things can refuse in a way init cannot go on from — see the layout guard in -/// [`emit_module_init`]: a class built ahead of everything else, and a class the module body -/// changed after its statement in a way the type replacing it cannot answer for. a module -/// with neither installs its classes or fails the import, and never quietly stands down +/// four things can refuse in a way init cannot go on from — see the layout guard in +/// [`emit_module_init`]: a class built ahead of everything else, a class the module body +/// changed after its statement in a way the type replacing it cannot answer for, a class +/// whose name the module body wrote over, and a class the interpreter could not build over +/// a base of ours — see [`standing_down_over_an_orphaned_base`]. a module with no class in +/// the namespace installs its functions or fails the import, and never quietly stands down /// entire fn stands_the_module_down(module: &ModuleIr) -> bool { module .classes .iter() - .any(|class| built_ahead(module, class) || class.checked_after_the_body()) + .any(|class| built_ahead(module, class) || class.exported) } /// what compiled code reaches on `class` with no lookup at all, by the name the class @@ -11854,6 +12045,18 @@ fn emit_module_init(module: &ModuleIr) -> String { i32::from(class.records_what_it_bound()) )); } + // and every class the module body wrote its name over after the statement: compiled + // code reaches the class without going through the name, so installing the type over + // the write would throw the write away — see `By_ClassNameMoved`. a write the source + // spells out is declined at build time, and this is the one for the shapes no reading + // of it can see: `globals()[name] = v`, a helper handed `globals()`, a star import + for class in module.classes.iter().filter(|class| class.exported) { + conditions.push(format!( + "By_ClassNameMoved(by_statements, by_bound, dict, {}, {})", + c_string(&class.name), + i32::from(class.records_what_it_bound()) + )); + } // whether the fallback source has to be run with its class bodies captured. the // capture costs a dict copy per class the body writes, so a module with nothing to // take out of one runs its body the plain way. a decorated method is taken out of a @@ -11862,9 +12065,9 @@ fn emit_module_init(module: &ModuleIr) -> String { // named by the one list, so this asks that rather than repeating it let captures_bodies = module.classes.iter().any(|class| class.exported); let release_bodies = if captures_bodies { - " Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n" + " Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n Py_CLEAR(by_ran);\n" } else { - "" + " Py_CLEAR(by_ran);\n" }; // an exit that leaves the module interpreted never installs a native, so the // installer the twin defined has nothing to do — and it must not be left standing @@ -12044,21 +12247,24 @@ fn emit_module_init(module: &ModuleIr) -> String { ); } // the definition still standing under the name, paired with the forwarder built - // for that name. a name the body went on to rebind to something that is not a - // function is not a definition this replaced, and is left unpaired rather than - // paired against whatever is there + // for that name. a name the body went on to write over is not a definition this + // replaced, and is left unpaired rather than paired against whatever is there // // both halves are borrowed: `dict` holds the definition until the install at the // end of init, and `by_built` holds the forwarder for at least as long + let published: Vec<&str> = module + .shims + .as_ref() + .map(|shims| shims.functions.iter().map(String::as_str).collect()) + .unwrap_or_default(); for (at, name) in forwarder_twins.iter().enumerate() { let slot = classes + at; + let Some(stands) = published.iter().position(|published| published == name) else { + continue; + }; let _ = writeln!( forwarder_record, - " by_twin[{slot}] = PyDict_GetItemString(dict, {name});\n\ - \x20 if (by_twin[{slot}] == NULL) PyErr_Clear();\n\ - \x20 if (by_twin[{slot}] != NULL && !PyFunction_Check(by_twin[{slot}])) {{\n\ - \x20 by_twin[{slot}] = NULL;\n\ - \x20 }}\n\ + " by_twin[{slot}] = by_stands[{stands}] ? PyDict_GetItemString(dict, {name}) : NULL;\n\ \x20 Py_XINCREF(by_twin[{slot}]);\n\ \x20 if (by_twin[{slot}] != NULL) {{\n\ \x20 by_type[{slot}] = By_BuiltForwarder(by_built, {name});\n\ @@ -12079,6 +12285,15 @@ fn emit_module_init(module: &ModuleIr) -> String { ); } } + // and the docstring, which the adoption leaves behind for the same reason. before + // any class decorator init applies, which may write one of its own + for (slot, class) in twins.iter().enumerate() { + let _ = writeln!( + adopt_init, + " if (By_CarryDoc(by_twin[{slot}], {}_OBJ) < 0) return -1;", + class.type_name(module.name.dotted()) + ); + } // the dataclass bookkeeping the adoption above leaves behind because it is spelled // as a dunder. it has no slot, so nothing about it can disagree with one — and it // is a `dataclasses.Field` dict rather than code, which is why it is carried @@ -12316,6 +12531,32 @@ fn emit_module_init(module: &ModuleIr) -> String { if !ready.is_empty() { let _ = writeln!(installed, "{ready}"); } + // before anything has been decorated or adopted, which is what lets the whole + // module still stand down + if let Some(slot) = slot + && stands_on_an_emitted_base(module, class) + { + installed.push_str(&standing_down_over_an_orphaned_base( + module, + class, + &twins, + slot, + &ModuleUnwind { + twin_count: twins.len() + forwarder_twins.len(), + record_interpreted, + drop_installer: &drop_installer, + release_bodies, + }, + )); + } + // python's own messages name a type by its `tp_name`, which a `class` statement + // leaves the bare name and a spec leaves the dotted one it was given + if heap_type(class) { + let _ = writeln!( + installed, + " if (By_BareTypeName({type_name}_OBJ) < 0) return -1;" + ); + } // a method the class wrote binds as a function does, before anything below can // read the type's dict or hand the class to a decorator let written = class.table_methods().count(); @@ -12458,7 +12699,7 @@ fn emit_module_init(module: &ModuleIr) -> String { continue; }; let setter = if class.exported { - format!("{type_name}_set_{}", field.name) + format!("{type_name}_set_{}", mangle(&field.name)) } else { "NULL".to_string() }; @@ -12467,8 +12708,8 @@ fn emit_module_init(module: &ModuleIr) -> String { " if (By_HoldFieldDefault((PyTypeObject *){type_name}_OBJ, {}, &{cell}, {}, {type_name}_get_{}, {setter}, {type_name}_has_{}) < 0) return -1;", c_string(&field.name), i32::from(holds), - field.name, - field.name + mangle(&field.name), + mangle(&field.name) ); } // a dunder the body assigned is now in the type's dict under its own name, and @@ -12658,18 +12899,23 @@ fn emit_module_init(module: &ModuleIr) -> String { // the body is run either way; what a class-level constant needs is what each `class` // statement wrote *before* its own decorators, which only the capturing run keeps + // + // `by_ran` is the code object the body ran, which is what says which function each + // `def` statement made — see `By_DefinitionCodes` let run_body = if captures_bodies { "\x20 PyObject *by_bodies = NULL;\n\ \x20 PyObject *by_statements = NULL;\n\ \x20 PyObject *by_bound = NULL;\n\ + \x20 PyObject *by_ran = NULL;\n\ \x20 if (by_fallback_source[0] != '\\0') {\n\ - \x20 by_bodies = By_RunModuleBody(&by_fallback, dict, &by_statements, &by_bound);\n\ + \x20 by_bodies = By_RunModuleBody(&by_fallback, dict, &by_statements, &by_bound, &by_ran);\n\ \x20 if (by_bodies == NULL) return -1;\n\ \x20 }\n" .to_string() } else { - "\x20 if (by_fallback_source[0] != '\\0') {\n\ - \x20 PyObject *result = By_ExecModuleBody(&by_fallback, dict);\n\ + "\x20 PyObject *by_ran = NULL;\n\ + \x20 if (by_fallback_source[0] != '\\0') {\n\ + \x20 PyObject *result = By_ExecModuleBody(&by_fallback, dict, &by_ran);\n\ \x20 if (result == NULL) return -1;\n\ \x20 Py_DECREF(result);\n\ \x20 }\n" @@ -12744,9 +12990,10 @@ fn emit_module_init(module: &ModuleIr) -> String { let type_name = owner.type_name(module.name.dotted()); let arm = publishing_class(module, owner, &field).map(|publisher| { format!( - " By_ArmField(&by_member, {type_name}_OBJ, {}, {}_get_{field});\n", + " By_ArmField(&by_member, {type_name}_OBJ, {}, {}_get_{});\n", c_string(&field), publisher.type_name(module.name.dotted()), + mangle(&field), ) }); let _ = writeln!( @@ -12810,6 +13057,10 @@ fn emit_module_init(module: &ModuleIr) -> String { for class in &module.classes { let type_name = class.type_name(module.name.dotted()); let _ = writeln!(strip_docs, " By_StripDocsAtOO({type_name}_methods);"); + // a written `__new__` is in no table either, for the reason `publishes_new` gives + if publishes_new(class).is_some() { + let _ = writeln!(strip_docs, " By_StripDocAtOO(&{type_name}_new_def);"); + } // a property's halves are in no table, so each is named on its own for property in &class.properties { let symbol = property_symbol(&type_name, &property.name); @@ -12826,33 +13077,54 @@ fn emit_module_init(module: &ModuleIr) -> String { // does with the definitions' own names — a class decorator resolved out of the // namespace above all — and before the natives and before the decorators, which is // where they always went. `By_BuildForwarders` says why the two moments differ + // + // whether each definition is still what its name holds is asked first, before init + // does anything with the names itself: `by_stands` says which forwarders are bound + // and which sites may ever go native — see `By_DefinitionStanding` let (build_forwarders, install_forwarders) = match &module.shims { Some(shims) => ( format!( - " PyObject *by_built = By_BuildForwarders(module, dict, by_forwarded, {}, {});\n\ + " char by_stands[{count}];\n\ + \x20 {{ PyObject *by_definitions = By_DefinitionCodes(by_ran);\n\ + \x20 Py_CLEAR(by_ran);\n\ + \x20 if (by_definitions == NULL) return -1;\n\ + {stands}\ + \x20 Py_DECREF(by_definitions); }}\n\ + \x20 PyObject *by_built = By_BuildForwarders(module, dict, by_forwarded, {count}, {installer});\n\ \x20 if (by_built == NULL) return -1;\n", - shims.functions.len(), - c_string(&shims.installer) + count = shims.functions.len(), + installer = c_string(&shims.installer), + stands = shims.functions.iter().enumerate().fold( + String::new(), + |mut stands, (at, name)| { + let _ = writeln!( + stands, + " by_stands[{at}] = By_DefinitionStanding(dict, {}, by_definitions) != NULL;", + c_string(name) + ); + stands + } + ), ), format!( - " {{ int by_installed = By_InstallForwarders(dict, by_built, by_forwarded, {}, {});\n\ + " {{ int by_installed = By_InstallForwarders(dict, by_built, by_forwarded, {}, {}, by_stands);\n\ \x20 Py_DECREF(by_built);\n\ \x20 if (by_installed < 0) return -1; }}\n{}", shims.functions.len(), c_string(&shims.installer), - shims - .functions - .iter() - .fold(String::new(), |mut arms, name| { + shims.functions.iter().enumerate().fold( + String::new(), + |mut arms, (at, name)| { let _ = writeln!( arms, - " if (By_ArmFunctionSite(&{}, dict, {}, {}) < 0) return -1;", + " if (By_ArmFunctionSite(&{}, dict, {}, {}, by_stands[{at}]) < 0) return -1;", function_site(name), c_string(name), i32::from(asks_whether_it_stands(module, name)) ); arms - }) + } + ) ), ), None => (String::new(), String::new()), @@ -12939,12 +13211,6 @@ fn emit_module_init(module: &ModuleIr) -> String { out } -fn mangle(name: &str) -> String { - name.chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect() -} - #[cfg(test)] mod tests { use super::*; @@ -13059,7 +13325,7 @@ mod tests { fn a_native_signature_uses_the_mangled_symbol() { let module = module_with(add()); let c = emit_module(&module); - assert!(c.contains("ByTagged by_app_add(ByTagged r0, ByTagged r1)")); + assert!(c.contains("ByTagged by_3app_3add(ByTagged r0, ByTagged r1)")); } /// `class Cell: v: int` with the field always assigned, and one method of it @@ -13140,7 +13406,7 @@ mod tests { // the value is only ever an unset field's error word when it is not a short, and // a read that keeps a reference already asks that let c = emit_module(&cell(reads_v_plus_one(false))); - let body = body_of(&c, "by_app_Cell_next(By_app_Cell * r0) {"); + let body = body_of(&c, "by_3app_4Cell_4next(By_3app_4Cell * r0) {"); let short = body .find("if (BY_UNLIKELY(!By_IsShort(by_t))) {") .unwrap_or_else(|| panic!("the read asks whether it has a short: {body}")); @@ -13156,7 +13422,7 @@ mod tests { // a read that keeps nothing leaves the value to the addition, whose test that both // operands are shorts already sends an unset field's odd error word the slow way let c = emit_module(&cell(reads_v_plus_one(true))); - let body = body_of(&c, "by_app_Cell_next(By_app_Cell * r0) {"); + let body = body_of(&c, "by_3app_4Cell_4next(By_3app_4Cell * r0) {"); let fast = body .find("if (BY_UNLIKELY(!By_IntAddShort(") .unwrap_or_else(|| panic!("the addition has a fast path: {body}")); @@ -13174,7 +13440,7 @@ mod tests { fn a_small_body_with_no_loop_is_offered_to_be_written_in_place() { let c = emit_module(&module_with(add())); assert!( - c.contains("static inline ByTagged by_app_add(ByTagged r0, ByTagged r1)"), + c.contains("static inline ByTagged by_3app_3add(ByTagged r0, ByTagged r1)"), "{c}" ); @@ -13188,7 +13454,7 @@ mod tests { looping.blocks.push(again); let c = emit_module(&module_with(looping)); assert!( - c.contains("static ByTagged by_app_add(ByTagged r0, ByTagged r1)"), + c.contains("static ByTagged by_3app_3add(ByTagged r0, ByTagged r1)"), "{c}" ); } @@ -13366,7 +13632,7 @@ mod tests { let c = emit_module(&module); assert!( c.contains( - "By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_app_Wrapped_Type_spec);" + "By_3app_7Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_3app_7Wrapped_Type_spec);" ), "the spec construction is the guard's: {c}" ); @@ -13377,7 +13643,7 @@ mod tests { // and a refusal is the whole module's, taken before anything of this module's // own is installed let refusal = c - .find(&refuses_whole_module("By_app_Wrapped_Type")) + .find(&refuses_whole_module("By_3app_7Wrapped_Type")) .expect("a refusal leaves the module interpreted"); let install = c .find("PyModule_AddFunctions") @@ -13406,12 +13672,12 @@ mod tests { assert!( c.contains( - "By_app_Wrapped *by_new = ((By_app_Wrapped *)By_TypeData(by_obj, By_app_Wrapped_Type_OBJ));" + "By_3app_7Wrapped *by_new = ((By_3app_7Wrapped *)By_TypeData(by_obj, By_3app_7Wrapped_Type_OBJ));" ), "the storage is found, not assumed: {c}" ); assert!( - !c.contains("(By_app_Wrapped *)by_type->tp_alloc"), + !c.contains("(By_3app_7Wrapped *)by_type->tp_alloc"), "the allocation is an object, not a field struct: {c}" ); // and the value handed on is still the object — the field storage is not one @@ -13440,7 +13706,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains("By_app_Owned *by_new = (By_app_Owned *)by_obj;"), + c.contains("By_3app_5Owned *by_new = (By_3app_5Owned *)by_obj;"), "no lookup for a class that owns its layout: {c}" ); assert!(!c.contains("By_TypeData(by_obj"), "{c}"); @@ -13460,22 +13726,22 @@ mod tests { assert!( c.contains( - "By_app_Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_app_Deeper_Type_spec, \"Wrapped\", By_app_Wrapped_Type_OBJ);" + "By_3app_6Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_3app_6Deeper_Type_spec, \"Wrapped\", By_3app_7Wrapped_Type_OBJ);" ), "the subclass stands on the emitted base: {c}" ); // the base is still the one construction that has to answer from reality, and // it is built first let base = c - .find("By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\"") + .find("By_3app_7Wrapped_Type = By_SpecClass(dict, \"Wrapped\"") .expect("the base is built from the twin's own bases"); let derived = c - .find("By_app_Deeper_Type = By_SpecSubclass(") + .find("By_3app_6Deeper_Type = By_SpecSubclass(") .expect("the subclass is built after it"); assert!(base < derived, "the base is built first: {c}"); // and a refusal is still the whole module's assert!( - c.contains(&refuses_whole_module("By_app_Deeper_Type")), + c.contains(&refuses_whole_module("By_3app_6Deeper_Type")), "{c}" ); } @@ -13501,14 +13767,14 @@ mod tests { !c.contains("== NULL) return 0;"), "nothing outside the pair reads either, so the module is not refused: {c}" ); - let shared = "if (By_app_Wrapped_Type != NULL && By_app_Deeper_Type != NULL) {"; + let shared = "if (By_3app_7Wrapped_Type != NULL && By_3app_6Deeper_Type != NULL) {"; assert_eq!( c.matches(shared).count(), 2, "both halves of the pair wait on both types: {c}" ); assert!( - !c.contains("if (By_app_Wrapped_Type != NULL) {"), + !c.contains("if (By_3app_7Wrapped_Type != NULL) {"), "so the base is never installed on its own: {c}" ); } @@ -13528,11 +13794,11 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "the base a still-running function reads refuses for the module: {c}" ); assert!( - c.contains(&refuses_whole_module("By_app_Deeper_Type")), + c.contains(&refuses_whole_module("By_3app_6Deeper_Type")), "and the subclass goes with it, because it cannot stand alone on a base \ the module gave up: {c}" ); @@ -13550,21 +13816,21 @@ mod tests { assert!( c.contains( - "By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_app_Wrapped_Type_spec);" + "By_3app_7Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_3app_7Wrapped_Type_spec);" ), "the construction is unchanged: {c}" ); assert!( - !c.contains("return 0;\n By_app_Wrapped_Type") + !c.contains("return 0;\n By_3app_7Wrapped_Type") && !c.contains("Wrapped_Type == NULL) return 0"), "nothing reaches into it, so the module is not refused: {c}" ); assert!( - c.contains("if (By_app_Wrapped_Type != NULL) {"), + c.contains("if (By_3app_7Wrapped_Type != NULL) {"), "the install waits on the type: {c}" ); assert!( - c.contains("if (PyDict_SetItemString(dict, \"Wrapped\", By_app_Wrapped_Type_OBJ) < 0) return -1;"), + c.contains("if (PyDict_SetItemString(dict, \"Wrapped\", By_3app_7Wrapped_Type_OBJ) < 0) return -1;"), "and installs it where there is one: {c}" ); } @@ -13595,7 +13861,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a module function reaching into it refuses for the module: {c}" ); } @@ -13623,7 +13889,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a class the module cannot answer for keeps the whole-module refusal: {c}" ); } @@ -13652,7 +13918,7 @@ mod tests { let c = emit_module(&module); assert!( - !c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + !c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a class that holds the attribute does not hold the module: {c}" ); } @@ -13680,12 +13946,12 @@ mod tests { assert!( c.contains( "{\"__dictoffset__\", BY_DICT_OFFSET_MEMBER,\n\ - \x20 offsetof(By_app_Open, by_dict), BY_DICT_OFFSET_FLAGS}" + \x20 offsetof(By_3app_4Open, by_dict), BY_DICT_OFFSET_FLAGS}" ), "the type is told where the word is: {c}" ); assert!( - c.contains("{Py_tp_members, (void *)By_app_Open_Type_members}"), + c.contains("{Py_tp_members, (void *)By_3app_4Open_Type_members}"), "the members table reaches the type: {c}" ); } @@ -13712,7 +13978,7 @@ mod tests { assert!( c.contains( - "struct By_app_Tight {\n\ + "struct By_3app_5Tight {\n\ \x20 PyObject_HEAD\n\ \x20 PyObject *by_dict;" ), @@ -13720,7 +13986,7 @@ mod tests { ); assert!( c.contains( - "struct By_app_Loose {\n\ + "struct By_3app_5Loose {\n\ \x20 PyObject_HEAD\n\ \x20 PyObject *by_dict;" ), @@ -13728,11 +13994,11 @@ mod tests { ); // only the subclass can reach one, so only the subclass names the offset assert!( - !c.contains("offsetof(By_app_Tight, by_dict)"), + !c.contains("offsetof(By_3app_5Tight, by_dict)"), "the slotted base publishes no dict: {c}" ); assert!( - c.contains("offsetof(By_app_Loose, by_dict)"), + c.contains("offsetof(By_3app_5Loose, by_dict)"), "the subclass publishes one: {c}" ); } @@ -13775,7 +14041,7 @@ mod tests { let c = emit_module(&module); assert!( - !c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + !c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a class with a dict answers for the write itself: {c}" ); } @@ -13804,7 +14070,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a register typed on it reaches into it: {c}" ); } @@ -13826,7 +14092,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "the type it answers with reaches into it: {c}" ); } @@ -13852,7 +14118,7 @@ mod tests { "nothing still running reads either, so the module is not refused: {c}" ); assert!( - c.contains("if (By_app_Wrapped_Type != NULL && By_app_Reader_Type != NULL) {"), + c.contains("if (By_3app_7Wrapped_Type != NULL && By_3app_6Reader_Type != NULL) {"), "the reader waits on what it reads as well as on itself: {c}" ); } @@ -13880,8 +14146,8 @@ mod tests { assert!( c.contains( - "if (By_app_Wrapped_Type != NULL && By_app_Reader_Type != NULL \ - && By_app_Outer_Type != NULL) {" + "if (By_3app_7Wrapped_Type != NULL && By_3app_6Reader_Type != NULL \ + && By_3app_5Outer_Type != NULL) {" ), "the outer class waits on everything its methods can reach: {c}" ); @@ -13906,11 +14172,11 @@ mod tests { let c = emit_module(&module); assert!( - !c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + !c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a helper nothing else builds does not hold the module: {c}" ); assert!( - c.contains("if (By_app_Wrapped_Type != NULL) {"), + c.contains("if (By_3app_7Wrapped_Type != NULL) {"), "the class is still left out on its own: {c}" ); } @@ -13935,14 +14201,14 @@ mod tests { ); assert!( c.contains( - "if (By_app_Wrapped_Type != NULL && By_app_Wrapped_step_gen_Type != NULL) {" + "if (By_3app_7Wrapped_Type != NULL && By_3app_16Wrapped_step_gen_Type != NULL) {" ), "it waits on the class it reads: {c}" ); assert!( c.contains( "if (PyDict_SetItemString(dict, \"Wrapped_step_gen\", \ - By_app_Wrapped_step_gen_Type_OBJ) < 0) return -1;" + By_3app_16Wrapped_step_gen_Type_OBJ) < 0) return -1;" ), "and the name it publishes is inside that wait: {c}" ); @@ -13962,7 +14228,7 @@ mod tests { let c = emit_module(&module); assert!( - c.contains(&refuses_whole_module("By_app_Wrapped_Type")), + c.contains(&refuses_whole_module("By_3app_7Wrapped_Type")), "a helper the module still builds holds it: {c}" ); } @@ -14026,20 +14292,20 @@ mod tests { let c = emit_module(&module); let declaration = c - .split("struct By_app_Deeper {") + .split("struct By_3app_6Deeper {") .nth(1) .and_then(|rest| rest.split_once("};")) .map(|(body, _)| body.to_string()) .expect("the subclass declares a struct"); - assert!(declaration.contains("by_f_depth;"), "{declaration}"); - assert!(!declaration.contains("by_f_tag;"), "{declaration}"); + assert!(declaration.contains("by_f_5depth;"), "{declaration}"); + assert!(!declaration.contains("by_f_3tag;"), "{declaration}"); // and a read of the inherited field goes to the base's region rather than this // one's, which is what keeps the single copy single assert!( c.contains( - "static PyObject *By_app_Deeper_Type_get_tag(PyObject *selfobj, void *closure) {\n\ + "static PyObject *By_3app_6Deeper_Type_get_3tag(PyObject *selfobj, void *closure) {\n\ \x20 (void)closure;\n\ - \x20 By_app_Wrapped *self = ((By_app_Wrapped *)By_TypeData(selfobj, By_app_Wrapped_Type_OBJ));" + \x20 By_3app_7Wrapped *self = ((By_3app_7Wrapped *)By_TypeData(selfobj, By_3app_7Wrapped_Type_OBJ));" ), "{c}" ); @@ -14056,25 +14322,25 @@ mod tests { let c = emit_module(&module); let init = c - .split("static int By_app_Deeper_Type_init(") + .split("static int By_3app_6Deeper_Type_init(") .nth(1) .and_then(|rest| rest.split_once("\n}\n")) .map(|(body, _)| body.to_string()) .expect("the subclass generates a constructor"); assert!( init.contains( - "By_app_Deeper *self = ((By_app_Deeper *)By_TypeData(selfobj, By_app_Deeper_Type_OBJ));" + "By_3app_6Deeper *self = ((By_3app_6Deeper *)By_TypeData(selfobj, By_3app_6Deeper_Type_OBJ));" ), "{init}" ); assert!( init.contains( - "By_app_Wrapped *by_up1 = ((By_app_Wrapped *)By_TypeData(selfobj, By_app_Wrapped_Type_OBJ));" + "By_3app_7Wrapped *by_up1 = ((By_3app_7Wrapped *)By_TypeData(selfobj, By_3app_7Wrapped_Type_OBJ));" ), "{init}" ); - assert!(init.contains("by_up1->by_f_tag = by_v;"), "{init}"); - assert!(init.contains("self->by_f_depth = by_v;"), "{init}"); + assert!(init.contains("by_up1->by_f_3tag = by_v;"), "{init}"); + assert!(init.contains("self->by_f_5depth = by_v;"), "{init}"); // and both are still bound from the call, in the order the class declares them assert!( init.contains("static const char *const by_names[] = { \"tag\", \"depth\" };"), @@ -14099,7 +14365,7 @@ mod tests { assert!(!c.contains("By_SpecSubclass"), "{c}"); assert!( c.contains( - "By_app_Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_app_Deeper_Type_spec);" + "By_3app_6Deeper_Type = By_SpecClass(dict, \"Deeper\", &By_3app_6Deeper_Type_spec);" ), "{c}" ); @@ -14125,23 +14391,23 @@ mod tests { assert!( c.contains( - "By_app_Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_app_Wrapped_Type_spec);" + "By_3app_7Wrapped_Type = By_SpecClass(dict, \"Wrapped\", &By_3app_7Wrapped_Type_spec);" ), "the base is built from a spec of its own: {c}" ); assert!( c.contains( - "By_app_Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_app_Deeper_Type_spec, \"Wrapped\", By_app_Wrapped_Type_OBJ);" + "By_3app_6Deeper_Type = By_SpecSubclass(dict, \"Deeper\", &By_3app_6Deeper_Type_spec, \"Wrapped\", By_3app_7Wrapped_Type_OBJ);" ), "and the subclass stands on the finished type: {c}" ); assert!( - c.contains("{Py_tp_dealloc, (void *)By_app_Wrapped_Type_dealloc},"), + c.contains("{Py_tp_dealloc, (void *)By_3app_7Wrapped_Type_dealloc},"), "the base carries the deallocator that breaks the chain: {c}" ); // and it binds no storage, because it declared no region for one assert!( - !c.contains("By_app_Wrapped *by_f"), + !c.contains("By_3app_7Wrapped *by_f"), "nothing of the base's own is reached: {c}" ); } @@ -14160,7 +14426,7 @@ mod tests { assert!(!c.contains("By_SpecClass"), "{c}"); assert!(c.contains("By_BuildClass"), "{c}"); assert!( - !c.contains("{Py_tp_dealloc, (void *)By_app_Wrapped_Type_dealloc},"), + !c.contains("{Py_tp_dealloc, (void *)By_3app_7Wrapped_Type_dealloc},"), "no slot takes the base's freeing over: {c}" ); } @@ -14201,7 +14467,7 @@ mod tests { // and the class is still built by calling the metaclass, which is the whole point // of putting the value where the metaclass can read it assert!( - c.contains(", by_kwds, By_app_Wrapped_Type_methods, NULL, 1, &by_carried);"), + c.contains(", by_kwds, By_3app_7Wrapped_Type_methods, NULL, 1, &by_carried);"), "the metaclass is what builds it, and it is handed the carry: {c}" ); } @@ -14214,7 +14480,7 @@ mod tests { /// installed nothing fn refuses_whole_module(type_name: &str) -> String { format!( - "if ({type_name} == NULL) {{\n by_record_interpreted();\n Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n return 0;\n }}" + "if ({type_name} == NULL) {{\n by_record_interpreted();\n Py_XDECREF(by_bodies);\n Py_XDECREF(by_statements);\n Py_XDECREF(by_bound);\n Py_CLEAR(by_ran);\n return 0;\n }}" ) } @@ -14276,7 +14542,7 @@ mod tests { assert!( c.contains( - "if (By_VerifyClass(\"app\", dict, \"Wrapped\", by_type[0], By_app_Wrapped_Type_OBJ," + "if (By_VerifyClass(\"app\", dict, \"Wrapped\", by_type[0], By_3app_7Wrapped_Type_OBJ," ), "what the name means is checked against what was emitted for it: {c}" ); @@ -14287,7 +14553,7 @@ mod tests { "the base's own slot is named: {c}" ); assert!( - c.contains("By_app_Deeper_Type_methods,"), + c.contains("By_3app_6Deeper_Type_methods,"), "the method table is what says which names were lowered: {c}" ); // before the interpreted definitions are released, because comparing against one @@ -14396,10 +14662,10 @@ mod tests { let c = emit_module(&module); // the forward declaration precedes the body, so take the last split let use_body = c - .rsplit("static inline ByTagged by_app_use") + .rsplit("static inline ByTagged by_3app_3use") .next() .expect("the caller is emitted"); - assert!(use_body.contains("by_app_pure()"), "{use_body}"); + assert!(use_body.contains("by_3app_4pure()"), "{use_body}"); assert!(!use_body.contains("if (r0 == BY_INT_ERROR) goto by_error;")); } @@ -14410,7 +14676,7 @@ mod tests { // releasing them here *and* in the wrapper was a double-decref that only // survived because small ints are unrefcounted let body = c - .split("static inline ByTagged by_app_add(ByTagged r0, ByTagged r1) {") + .split("static inline ByTagged by_3app_3add(ByTagged r0, ByTagged r1) {") .nth(1) .and_then(|rest| rest.split("static PyObject").next()) .expect("the body is emitted"); @@ -14480,7 +14746,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(a))); let c = emit_module(&module_with(builder.finish())); let body = c - .split("static inline PyObject * by_app_first(PyObject * r0) {") + .split("static inline PyObject * by_3app_5first(PyObject * r0) {") .nth(1) .and_then(|rest| rest.split("static PyObject *byw").next()) .expect("the body is emitted"); @@ -14516,7 +14782,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(n))); let c = emit_module(&module_with(builder.finish())); let body = c - .split("static inline ByTagged by_app_halve(ByTagged r0) {") + .split("static inline ByTagged by_3app_5halve(ByTagged r0) {") .nth(1) .and_then(|rest| rest.split("static PyObject").next()) .expect("the body is emitted"); @@ -14858,8 +15124,8 @@ mod tests { let mut function = add(); function.exported = false; let c = emit_module(&module_with(function)); - assert!(!c.contains("byw_app_add")); - assert!(c.contains("static inline ByTagged by_app_add")); + assert!(!c.contains("byw_3app_3add")); + assert!(c.contains("static inline ByTagged by_3app_3add")); } #[test] @@ -14886,7 +15152,7 @@ mod tests { assert!(c.contains("PyMODINIT_FUNC PyInit_app(void)")); assert!(c.contains("\"app\"")); assert!(c.contains( - "{\"add\", (PyCFunction)(void(*)(void))byw_pkg_app_add, METH_FASTCALL | METH_KEYWORDS, NULL}," + "{\"add\", (PyCFunction)(void(*)(void))byw_10pkgZ2e_app_3add, METH_FASTCALL | METH_KEYWORDS, NULL}," )); } @@ -14993,7 +15259,10 @@ mod tests { let c = emit_module(&module); // the natives must be installed from the exec slot, not from m_methods, // or the interpreted definitions would overwrite them - assert!(c.contains("By_ExecModuleBody(&by_fallback, dict)"), "{c}"); + assert!( + c.contains("By_ExecModuleBody(&by_fallback, dict, &by_ran)"), + "{c}" + ); assert!( c.contains("PyModule_AddFunctions(module, by_methods)"), "{c}" @@ -15365,6 +15634,24 @@ mod tests { ); } + #[test] + fn a_class_read_of_a_class_the_module_does_not_emit_refuses_to_compile() { + // the verifier refuses this before codegen sees it. writing nothing would leave + // `r0` unassigned and the return reading whatever the stack held + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let class = builder.temp(RType::OBJECT); + builder.push(Op::LoadClass { + dest: class, + class: "Ghost".to_string(), + }); + builder.terminate(Terminator::Return(Value::Register(class))); + let c = emit_module(&module_with(builder.finish())); + assert!( + c.contains("#error \"a class read names `Ghost`, which this module does not emit\"\n"), + "{c}" + ); + } + #[test] fn a_string_literal_is_escaped_for_c() { let mut builder = FunctionBuilder::new("greet", RType::STR); @@ -15421,14 +15708,14 @@ mod tests { let c = emit_module(&module); let named = c - .find("typedef struct By_app_Point By_app_Point;") + .find("typedef struct By_3app_5Point By_3app_5Point;") .expect("the instance struct is named"); let slot = c - .find("typedef struct { By_app_Point * f0; ByTagged f1; } ByTuple_iPoint_int;") + .find("typedef struct { By_3app_5Point * f0; ByTagged f1; } ByTuple_i5Point_int;") .expect("the slot is a pointer to it"); assert!(named < slot, "{c}"); let defined = c - .find("struct By_app_Point {\n") + .find("struct By_3app_5Point {\n") .expect("and defined after both"); assert!(slot < defined, "{c}"); } @@ -15469,7 +15756,7 @@ mod tests { } let c = emit_module(&module); - assert_eq!(c.matches("} ByTuple_iPoint_int;").count(), 1, "{c}"); + assert_eq!(c.matches("} ByTuple_i5Point_int;").count(), 1, "{c}"); } #[test] @@ -15510,12 +15797,12 @@ mod tests { function.registers[inner.index()].borrowed = true; let text = emit_function(&ModuleIr::new("app"), &function); - assert!(text.contains("r1 = r0->by_f_inner;"), "{text}"); + assert!(text.contains("r1 = r0->by_f_5inner;"), "{text}"); // the intermediate is neither retained nor released - assert!(!text.contains("By_XIncRef(r0->by_f_inner)"), "{text}"); + assert!(!text.contains("By_XIncRef(r0->by_f_5inner)"), "{text}"); assert!(!text.contains("Py_XDECREF(r1)"), "{text}"); // the value that leaves still is - assert!(text.contains("By_XIncRef(r1->by_f_label)"), "{text}"); + assert!(text.contains("By_XIncRef(r1->by_f_5label)"), "{text}"); } #[test] @@ -15647,13 +15934,13 @@ mod tests { let c = emit_function(&ModuleIr::new("app"), &function); assert!( - c.contains("static ByGlobalUnboxedSite by_gu__limit = BY_GLOBAL_UNBOXED_SITE_INIT;"), + c.contains("static ByGlobalUnboxedSite by_gu_6_limit = BY_GLOBAL_UNBOXED_SITE_INIT;"), "{c}" ); assert!( c.contains( - "ByTagged by_t = By_LookupGlobalTaggedSite(&by_gu__limit, &by_gs__limit, \ - by_module_dict, by_g__limit);" + "ByTagged by_t = By_LookupGlobalTaggedSite(&by_gu_6_limit, &by_gs_6_limit, \ + by_module_dict, by_g_6_limit);" ), "{c}" ); @@ -15705,7 +15992,7 @@ mod tests { assert_eq!(verify(&function), Ok(())); let c = emit_function(&ModuleIr::new("app"), &function); - assert!(c.contains("By_LookupGlobalSite(&by_gs__limit"), "{c}"); + assert!(c.contains("By_LookupGlobalSite(&by_gs_6_limit"), "{c}"); assert!(c.contains("By_UnboxInt("), "{c}"); assert!(!c.contains("ByGlobalUnboxedSite"), "{c}"); } @@ -15802,7 +16089,7 @@ mod tests { let register = local(value); assert!( c.contains(&format!( - "->by_f_tag = {register};\n {register} = NULL;\n" + "->by_f_3tag = {register};\n {register} = NULL;\n" )), "{c}" ); @@ -16011,7 +16298,7 @@ mod tests { #[test] fn a_constructors_computed_default_is_optional_and_hands_the_call_over() { let c = emit_module(&holder()); - let slot = body_of(&c, "static int By_app_Holder_Type_init("); + let slot = body_of(&c, "static int By_3app_6Holder_Type_init("); // the arity is the *written* one: it is the value that is unavailable here, // not the parameter assert!( @@ -16023,7 +16310,7 @@ mod tests { // default is a value, and handing the call over would only make it slower assert!( slot.contains( - "if (by_bound[0] == NULL) return By_InitInterpreted(byi_app_Holder___init__, \ + "if (by_bound[0] == NULL) return By_InitInterpreted(byi_3app_6Holder_8__init__, \ \"Holder.__init__\", selfobj, args, kwds);" ), "{slot}" @@ -16031,7 +16318,7 @@ mod tests { assert!(!c.contains("By_TwinDefault("), "{c}"); // which means the slot needs the handle the method wrapper already takes assert!( - c.contains("static PyObject *byi_app_Holder___init__ = NULL;"), + c.contains("static PyObject *byi_3app_6Holder_8__init__ = NULL;"), "{c}" ); } @@ -16041,11 +16328,11 @@ mod tests { #[test] fn a_constructor_handed_its_marker_hands_the_call_over() { let c = emit_module(&holder_holding(true)); - let slot = body_of(&c, "static int By_app_Holder_Type_init("); + let slot = body_of(&c, "static int By_3app_6Holder_Type_init("); assert!( slot.contains( - "if ((by_bound[0] == NULL || by_bound[0] == byi_app_Holder___init___d1)) \ - return By_InitInterpreted(byi_app_Holder___init__, \"Holder.__init__\", \ + "if ((by_bound[0] == NULL || by_bound[0] == byi_3app_6Holder_8__init___d1)) \ + return By_InitInterpreted(byi_3app_6Holder_8__init__, \"Holder.__init__\", \ selfobj, args, kwds);" ), "{slot}" @@ -16053,7 +16340,7 @@ mod tests { // the marker is read off the definition once the module has run assert!( c.contains( - "byi_app_Holder___init___d1 = By_TwinDefault(byi_app_Holder___init__, 1, \"value\");" + "byi_3app_6Holder_8__init___d1 = By_TwinDefault(byi_3app_6Holder_8__init__, 1, \"value\");" ), "{c}" ); @@ -16116,7 +16403,7 @@ mod tests { // first and `body_of` takes whichever it finds let wrapper = body_of( &c, - "static PyObject *byw_app_outer_env_cb(PyObject *self, PyObject *const *args, \ + "static PyObject *byw_3app_12outerZ24_env_2cb(PyObject *self, PyObject *const *args, \ Py_ssize_t nargs, PyObject *kwnames) {", ); // the parameter stays optional, and the omitted case reads the field rather @@ -16126,12 +16413,12 @@ mod tests { "{wrapper}" ); assert!( - wrapper.contains("By_ReadCell(a0->by_gf__default_cb_name, \"name\", 0)"), + wrapper.contains("By_ReadCell(a0->by_gf_25Z24_defaultZ24_cbZ24_name, \"name\", 0)"), "{wrapper}" ); // and no edge to a definition that was never bound under a name assert!(!wrapper.contains("by_wrap_defer"), "{wrapper}"); - assert!(!c.contains("byi_app_outer_env_cb"), "{c}"); + assert!(!c.contains("byi_3app_12outerZ24_env_2cb"), "{c}"); } /// a class whose `__init__` is `def __init__(self, a, /, *rest, **extra)`: a @@ -16199,7 +16486,7 @@ mod tests { #[test] fn a_constructor_builds_its_variadic_parameters_rather_than_binding_them() { let c = emit_module(&variadic_init()); - let slot = body_of(&c, "static int By_app_Var_Type_init("); + let slot = body_of(&c, "static int By_3app_3Var_Type_init("); // only `a` is bound: the other two are what is *left over*, so treating them as // named parameters made a call supplying neither an arity error assert!( @@ -16233,7 +16520,7 @@ mod tests { // the brace is what tells the definition from the forward declaration let wrapper = body_of( &c, - "byw_app_Var___init__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {", + "byw_3app_3Var_8__init__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) {", ); assert!( wrapper.contains( @@ -16322,12 +16609,12 @@ mod tests { let c = emit_module(&module); assert!( - c.contains("By_MakeFunction(&byw_app_Env_call_spec, (PyObject *)(r0))"), + c.contains("By_MakeFunction(&byw_3app_3Env_4call_spec, (PyObject *)(r0))"), "{c}" ); assert!( c.contains( - "static const ByFunctionSpec byw_app_Env_call_spec = { byw_app_Env_call, \"call\", \"make..call\", NULL, &by_module_dict, NULL, 0, NULL, NULL, NULL, &by_fallback, &byw_app_Env_call_code };" + "static const ByFunctionSpec byw_3app_3Env_4call_spec = { byw_3app_3Env_4call, \"call\", \"make..call\", NULL, &by_module_dict, NULL, 0, NULL, NULL, NULL, &by_fallback, &byw_3app_3Env_4call_code };" ), "{c}" ); @@ -16344,21 +16631,21 @@ mod tests { let c = emit_module(&module); assert!( - c.contains("{Py_tp_alloc, (void *)By_app_Point_Type_alloc},") - && c.contains("{Py_tp_free, (void *)By_app_Point_Type_free},"), + c.contains("{Py_tp_alloc, (void *)By_3app_5Point_Type_alloc},") + && c.contains("{Py_tp_free, (void *)By_3app_5Point_Type_free},"), "the type takes both halves over: {c}" ); assert!( - c.contains("static PyObject *By_app_Point_Type_recycled[16];"), + c.contains("static PyObject *By_3app_5Point_Type_recycled[16];"), "and holds sixteen blocks: {c}" ); // a request for any other type is one these blocks are not the size of assert!( - c.contains("&& by_type == (PyTypeObject *)By_app_Point_Type_OBJ)"), + c.contains("&& by_type == (PyTypeObject *)By_3app_5Point_Type_OBJ)"), "the allocator asks what it is being asked for: {c}" ); assert!( - c.contains(" By_app_Point_Type_drain();\n") + c.contains(" By_3app_5Point_Type_drain();\n") && c.contains("by_slots, NULL, NULL, by_module_free"), "and the module hands the blocks back when it goes: {c}" ); @@ -16375,13 +16662,13 @@ mod tests { module.classes = vec![counted]; let c = emit_module(&module); assert!( - c.contains("static void By_app_Point_Type_unset(PyObject *selfobj) {") - && c.contains("self->by_f_x = BY_INT_ERROR;"), + c.contains("static void By_3app_5Point_Type_unset(PyObject *selfobj) {") + && c.contains("self->by_f_1x = BY_INT_ERROR;"), "{c}" ); assert!( - c.contains("By_app_Point_Type_unset(by_block);") - && c.contains("if (by_fresh != NULL) By_app_Point_Type_unset(by_fresh);"), + c.contains("By_3app_5Point_Type_unset(by_block);") + && c.contains("if (by_fresh != NULL) By_3app_5Point_Type_unset(by_fresh);"), "a recycled block and a fresh one are both reset: {c}" ); diff --git a/crates/by_ir/src/function.rs b/crates/by_ir/src/function.rs index c101530cea..6ccfbc724b 100644 --- a/crates/by_ir/src/function.rs +++ b/crates/by_ir/src/function.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use crate::mangle::mangle; use crate::ops::{BlockId, Op, RegisterId, Terminator, Value}; use crate::rtype::RType; @@ -704,12 +705,11 @@ impl FieldDecl { /// reserved words would have to be kept correct against every C version and /// compiler extension forever; a prefix cannot collide with any of them /// - /// the prefix also says whether the name was generated, and it has - /// to. `$` is not a portable C identifier character — msvc rejects it — so `mangle` - /// turns it into `_`, which is exactly the character a source name may hold: `$state` - /// and a local called `_state` both came out as `by_f__state`, and a generator taking - /// a parameter of that name declared one struct member twice and failed the *build*. - /// separating the two prefixes is what the `$` was already meant to be doing + /// the name is [mangled](crate::mangle::mangle), which keeps every two names apart: + /// `$state` and a local called `_state` once both came out as `by_f__state`, and a + /// generator taking a parameter of that name declared one struct member twice and + /// failed the *build*. the prefix also says whether the name was generated, which is + /// what reading the emitted struct wants to know pub fn member(&self) -> String { format!("{}{}", self.member_prefix("f"), mangle(&self.name)) } @@ -1084,14 +1084,15 @@ impl ClassIr { } /// whether the twin's `class` statement records the class it bound, after its - /// decorators, for that check to read the class against + /// decorators, for module init to read the class against /// /// a `data class` is the one class whose twin is not what its statement built: the /// `@dataclass` the transpiler wrote makes it again and writes every dunder it /// generates, so the body the statement left says nothing about which of those the - /// module body wrote over afterwards + /// module body wrote over afterwards — and the class it built is not what its name + /// holds, so it says nothing about whether the module body wrote over the name either pub fn records_what_it_bound(&self) -> bool { - self.dataclass && self.checked_after_the_body() + self.dataclass && self.exported } /// the methods python reaches by name, through the method table @@ -1779,8 +1780,18 @@ impl ModuleIr { /// the C identifier for the module's init function, which cpython looks up by /// name when loading the extension + /// + /// the name is not [mangled](crate::mangle::mangle), because cpython asks for exactly + /// `PyInit_`. a character a C identifier cannot hold becomes `_`, which no + /// interpreter will find either, but which leaves the C building pub fn init_symbol(&self) -> String { - format!("PyInit_{}", mangle(self.name.last_component())) + let name: String = self + .name + .last_component() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect(); + format!("PyInit_{name}") } } @@ -1893,13 +1904,6 @@ impl From for ModuleName { } } -/// map a dotted or otherwise punctuated name onto a C identifier -fn mangle(name: &str) -> String { - name.chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) - .collect() -} - #[cfg(test)] mod tests { use std::path::Path; @@ -1976,8 +1980,26 @@ mod tests { #[test] fn symbols_are_c_identifiers() { let f = function(); - assert_eq!(f.native_symbol("pkg.mod"), "by_pkg_mod_add"); - assert_eq!(f.wrapper_symbol("pkg.mod"), "byw_pkg_mod_add"); + assert_eq!(f.native_symbol("pkg.mod"), "by_10pkgZ2e_mod_3add"); + assert_eq!(f.wrapper_symbol("pkg.mod"), "byw_10pkgZ2e_mod_3add"); + } + + /// a function whose name spells a class's and a method's with `_` between is not that + /// method, and the two once shared every symbol and failed the build + #[test] + fn a_method_and_a_function_spelled_alike_have_their_own_symbols() { + let mut spelled_alike = function(); + spelled_alike.name = "A_b".to_string(); + let mut method = function(); + method.name = "b".to_string(); + method.owner = Some("A".to_string()); + for symbol in [ + Function::native_symbol, + Function::wrapper_symbol, + Function::interpreted_symbol, + ] { + assert_ne!(symbol(&spelled_alike, "m"), symbol(&method, "m")); + } } #[test] diff --git a/crates/by_ir/src/lib.rs b/crates/by_ir/src/lib.rs index e026b2bd65..5e6e0f6475 100644 --- a/crates/by_ir/src/lib.rs +++ b/crates/by_ir/src/lib.rs @@ -10,6 +10,7 @@ pub mod builder; pub mod call_graph; pub mod function; +pub mod mangle; pub mod ops; pub mod print; pub mod rtype; diff --git a/crates/by_ir/src/mangle.rs b/crates/by_ir/src/mangle.rs new file mode 100644 index 0000000000..2b0fccb18e --- /dev/null +++ b/crates/by_ir/src/mangle.rs @@ -0,0 +1,195 @@ +//! how a python name becomes part of a C identifier +//! +//! every emitted symbol is literal text and names spliced together — `By__`, +//! `by___`, `_get_` — and two different programs must +//! never splice to one symbol, or the C does not build. a name may hold any character +//! python allows in an identifier, and a generated one a `$` or a `<` besides, where a C +//! identifier holds only ASCII letters, digits and `_`. replacing each of the rest with `_` +//! sent `é1` and `_1` to one symbol, and spliced with `_` between, a function `A_b` and the +//! method `b` of a class `A` arrived at one symbol too +//! +//! so a name is spelled in two layers. the first maps it onto the characters a C +//! identifier may hold and can be read back: a letter, a digit or `_` stands for itself, and +//! anything else, `Z` itself included, is a `Z` and the character's code point in lowercase +//! hex closed by `_`; `ZZ` is a literal `Z`. a leading digit is spelled the long way too, so +//! the spelling never begins with one. the second layer writes the spelling's length in +//! front of it, which is what makes a mangled name end where it says: whatever follows it +//! in a symbol, a literal or another mangled name, cannot be read as part of it. a symbol's +//! literal text never puts a digit where a mangled name begins, so a symbol is read back +//! one way only + +use std::fmt::Write as _; + +/// the escape that introduces a character spelled by its code point +const ESCAPE: char = 'Z'; + +/// `name` as part of a C identifier, distinct from every other name's and ending where it +/// says — see the module documentation +pub fn mangle(name: &str) -> String { + let spelled = spell(name); + format!("{}{spelled}", spelled.len()) +} + +/// the first layer: `name` in the characters a C identifier may hold, never beginning with +/// a digit +fn spell(name: &str) -> String { + let mut out = String::with_capacity(name.len()); + for (at, character) in name.chars().enumerate() { + let plain = character != ESCAPE + && (character.is_ascii_alphabetic() + || character == '_' + || (character.is_ascii_digit() && at > 0)); + if plain { + out.push(character); + } else if character == ESCAPE { + out.push(ESCAPE); + out.push(ESCAPE); + } else { + let _ = write!(out, "{ESCAPE}{:x}_", u32::from(character)); + } + } + out +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::{ESCAPE, mangle}; + + /// read one mangled name off the front of `text`, handing back the name and what follows + /// it — the inverse that proves [`mangle`] loses nothing + fn unmangle(text: &str) -> Option<(String, &str)> { + let digits = text.chars().take_while(char::is_ascii_digit).count(); + let length: usize = text.get(..digits)?.parse().ok()?; + let spelled = text.get(digits..digits + length)?; + let rest = text.get(digits + length..)?; + let mut name = String::new(); + let mut characters = spelled.chars(); + while let Some(character) = characters.next() { + if character != ESCAPE { + name.push(character); + continue; + } + match characters.next()? { + ESCAPE => name.push(ESCAPE), + first => { + let mut hex = String::from(first); + loop { + match characters.next()? { + '_' => break, + digit => hex.push(digit), + } + } + name.push(char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?); + } + } + } + Some((name, rest)) + } + + /// names that collided, or would have, under some way of spelling them + const ADVERSARIAL: &[&str] = &[ + "é1", + "_1", + "_", + "__", + "Z", + "ZZ", + "Z5a_", + "Zz", + "z", + "a_b", + "a", + "b", + "A_b", + "value$set", + "value_set", + "$state", + "_state", + "", + "_lambda_", + "Foo_Type", + "Foo", + "1", + "11", + "Z31_", + "a.b", + "a_b.c", + "ü", + "u", + "日本", + "x\u{0}y", + "𝔘nicode", + "", + ]; + + #[test] + fn every_name_reads_back_as_itself() { + for name in ADVERSARIAL { + let mangled = mangle(name); + assert!( + mangled + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_'), + "{name:?} became {mangled:?}, which is not C" + ); + assert_eq!( + unmangle(&mangled), + Some(((*name).to_string(), "")), + "{name:?} became {mangled:?}" + ); + } + } + + /// a left inverse is injectivity, and this enumerates every name up to four characters + /// over an alphabet holding each kind of character the spelling treats apart + #[test] + fn no_two_names_mangle_alike() { + let alphabet = ['a', 'Z', 'z', '_', '1', 'é', '$', '.', '5']; + let mut names = vec![String::new()]; + let mut frontier = vec![String::new()]; + for _ in 0..4 { + frontier = frontier + .iter() + .flat_map(|prefix| alphabet.iter().map(move |c| format!("{prefix}{c}"))) + .collect(); + names.extend(frontier.iter().cloned()); + } + let mut seen: HashMap = HashMap::new(); + for name in &names { + let mangled = mangle(name); + if let Some(other) = seen.insert(mangled.clone(), name) { + panic!("{other:?} and {name:?} both became {mangled:?}"); + } + assert_eq!(unmangle(&mangled), Some((name.clone(), ""))); + } + } + + /// a mangled name ends where it says, so names spliced together with literal text + /// between them are read back as the same names — which is what keeps + /// `by__` apart from `by___` + #[test] + fn spliced_names_read_back_apart() { + for left in ADVERSARIAL { + for right in ADVERSARIAL { + let spliced = format!("{}_{}_Type", mangle(left), mangle(right)); + let (first, rest) = unmangle(&spliced).expect("the first name reads back"); + let rest = rest.strip_prefix('_').expect("the separator follows it"); + let (second, rest) = unmangle(rest).expect("the second name reads back"); + assert_eq!( + (first.as_str(), second.as_str(), rest), + (*left, *right, "_Type") + ); + } + } + assert_ne!( + format!("by_{}_{}", mangle("m"), mangle("A_b")), + format!("by_{}_{}_{}", mangle("m"), mangle("A"), mangle("b")), + ); + assert_ne!( + format!("By_{}_{}", mangle("m"), mangle("Foo_Type")), + format!("By_{}_{}_Type", mangle("m"), mangle("Foo")), + ); + } +} diff --git a/crates/by_ir/src/rtype.rs b/crates/by_ir/src/rtype.rs index 9a7eef754b..d6311821bf 100644 --- a/crates/by_ir/src/rtype.rs +++ b/crates/by_ir/src/rtype.rs @@ -263,11 +263,14 @@ impl RType { /// whether a value of this type is, or holds as an element, a reference to an /// object — something that may have a finalizer, or hold one that does + /// + /// a tagged `int` is one. its pointer case is an exact `int` too wide for the tag or an + /// `int` subclass kept as itself, and a subclass can write `__del__` pub fn holds_object_reference(&self) -> bool { match self { Self::Tuple(items) => items.iter().any(Self::holds_object_reference), Self::Array(element) => element.holds_object_reference(), - _ => self.is_object_reference(), + _ => self.owns_one_reference(), } } @@ -385,7 +388,7 @@ fn mangle_component(ty: &RType) -> String { Primitive::Tuple => "tuple".to_string(), }, RType::Tuple(items) => format!("t{}", tuple_mangle(items)), - RType::Instance { class, .. } => format!("i{class}"), + RType::Instance { class, .. } => format!("i{}", crate::mangle::mangle(class)), } } diff --git a/crates/by_ir/src/verify.rs b/crates/by_ir/src/verify.rs index 2badd88221..678e741d47 100644 --- a/crates/by_ir/src/verify.rs +++ b/crates/by_ir/src/verify.rs @@ -310,6 +310,26 @@ impl Verifier<'_> { }); } + /// an operation answering with the type object of a class, or building an instance of + /// one, must name a class the module emits + /// + /// codegen reaches that class's type through the module, so a name it does not find + /// there leaves the destination unwritten — a register every later read takes for a + /// value. a function verified on its own has no module to ask + fn expect_emitted_class(&mut self, block: BlockId, class: &str, what: &str) { + if let Some(module) = self.module + && !module + .classes + .iter() + .any(|candidate| candidate.name == class) + { + self.error( + Some(block), + format!("{what} names `{class}`, which this module does not emit"), + ); + } + } + /// a call's arguments must have the representations the callee declares /// /// codegen emits the call as a plain C call, so a mismatch here is a mismatch the @@ -1477,9 +1497,13 @@ impl Verifier<'_> { }, "an allocation", ); + self.expect_emitted_class(block, class, "an allocation"); } Op::MakeClosure { - dest, class, env, .. + dest, + class, + method, + env, } => { self.expect( block, @@ -1491,6 +1515,22 @@ impl Verifier<'_> { "a closure's environment", ); self.expect_dest(block, *dest, &RType::OBJECT, "a closure"); + // codegen builds the function from the nested method's own spec + if let Some(module) = self.module + && !module + .classes + .iter() + .filter(|candidate| candidate.name == *class) + .flat_map(|owner| &owner.methods) + .any(|candidate| candidate.name == *method && candidate.nested.is_some()) + { + self.error( + Some(block), + format!( + "a closure names `{class}.{method}`, which this module does not emit as a nested function" + ), + ); + } } Op::LoadGlobal { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "a global read"); @@ -1536,8 +1576,9 @@ impl Verifier<'_> { // no type to check: the destination keeps whatever representation it was // declared with, and the deletion only puts it back to unbound Op::DeleteLocal { .. } => {} - Op::LoadClass { dest, .. } => { + Op::LoadClass { dest, class } => { self.expect_dest(block, *dest, &RType::OBJECT, "a class read"); + self.expect_emitted_class(block, class, "a class read"); } Op::ImportModule { dest, .. } => { self.expect_dest(block, *dest, &RType::OBJECT, "an import"); @@ -2755,6 +2796,30 @@ mod tests { assert!(errors.iter().any(|e| e.message.contains("past the end"))); } + #[test] + fn a_class_read_must_name_a_class_the_module_emits() { + // codegen finds the type object through the module, and a class it does not find + // would leave the destination unwritten + let mut reader = add(); + reader.registers.push(temp(RType::OBJECT)); + reader.blocks[0].ops.insert( + 0, + Op::LoadClass { + dest: RegisterId(3), + class: "Ghost".to_string(), + }, + ); + assert_eq!(verify(&reader), Ok(())); + let mut module = ModuleIr::new("m"); + module.functions.push(reader); + let errors = verify_module(&module).unwrap_err(); + assert!( + errors.iter().any(|error| error.message + == "a class read names `Ghost`, which this module does not emit"), + "{errors:?}" + ); + } + #[test] fn verify_module_collects_errors_from_every_function() { let mut broken = add(); diff --git a/crates/by_irbuild/src/lib.rs b/crates/by_irbuild/src/lib.rs index af91d0d8de..b6359c7101 100644 --- a/crates/by_irbuild/src/lib.rs +++ b/crates/by_irbuild/src/lib.rs @@ -17,7 +17,7 @@ pub mod shims; pub mod single_file; mod surface; -pub use single_file::module_from_source; +pub use single_file::{module_from_source, module_from_source_at}; /// which language a source is written in /// @@ -3170,24 +3170,6 @@ fn lower_class<'a>( clash.name ))); } - // a property's halves are named with a `$`, which no source can write — but the - // symbol they mangle to replaces it with an underscore, and a method a source *did* - // write may already hold that symbol. two bodies under one symbol is a module that - // does not build at all, which is worse than one that declines - if let Some(clash) = published - .iter() - .flat_map(|property| [&property.getter, &property.setter, &property.deleter]) - .flatten() - .find(|accessor| { - lowered - .iter() - .any(|method| method.name != **accessor && mangles_alike(&method.name, accessor)) - }) - { - return Err(Decline::new(format!( - "a property half named `{clash}` reaches the same symbol as a method of this class" - ))); - } let slots = class_slots(class, &fields, inherited)?; Ok(( by_ir::function::ClassIr { @@ -3223,19 +3205,6 @@ fn lower_class<'a>( )) } -/// whether two python names reach one C identifier -/// -/// a symbol is the name with every character that is not alphanumeric replaced, so -/// `value$set` and `value_set` arrive at the same one. a property's halves are named -/// with a `$` no source can write, and this is what keeps that from being the only thing -/// standing between them and a method a source did write -fn mangles_alike(left: &str, right: &str) -> bool { - left.len() == right.len() - && left.chars().zip(right.chars()).all(|(left, right)| { - left == right || (!left.is_ascii_alphanumeric() && !right.is_ascii_alphanumeric()) - }) -} - /// `__getattr__` stands *behind* the ordinary lookup rather than replacing it /// /// the hook runs the ordinary lookup first and falls back to the method only where that @@ -6898,12 +6867,11 @@ fn presence_where_a_finalizer_reads( /// every field a `del` in this module names, written down as one that may be absent /// -/// `del self.buffer` has somewhere to go only where the field can record that it is -/// gone, and that is the presence byte an optional one carries: the delete clears it, -/// a read while it is clear raises `AttributeError` the way python does, and a later -/// write sets it again. a field with no byte has no absent state at all, so a delete -/// on one can only refuse — which is the `cannot delete an attribute` an emitted class -/// used to answer where the interpreted twin deleted. +/// `del self.buffer` clears the presence byte an optional field carries, a read while it +/// is clear raises `AttributeError` the way python does, and a later write sets it again. +/// a field without the byte is deleted too, into the state an instance `__init__` never +/// ran on is in — see the emitter's `delete_field` — which is how a delete from another +/// module reaches a field this one never deletes. /// /// the names are collected module-wide rather than per class, for two reasons. a /// delete need not be written in a method at all — `del handle.buffer` in a plain @@ -7571,7 +7539,7 @@ fn lower_function_with_receiver( arrays: &[(usize, RType)], frame: Frame, ) -> Lowered<(Function, Vec)> { - surface::gate_function(unit.language.source_type(), function)?; + surface::gate_function(unit.language.source_type(), unit.model, function)?; let function = &comprehension_scopes::scoped(function)?; let Unit { env, @@ -7717,6 +7685,21 @@ fn lower_function_with_receiver( unit.arrays, ); locals_here.retain(|(name, _)| !declared_global.contains(name)); + // an override's `_`s take the names of the method it overrides, and the python binds + // `_` to the first of them at the top of the body, so `_` is a local holding that + // parameter's value as well as whatever the body writes to it + let underscore_at_entry = underscore_bound_at_entry(&function.parameters, *unit.written) + .filter(|_| !declared_global.contains("_")) + .and_then(|parameter| { + let (_, rtype) = params.iter().find(|(name, _)| *name == parameter)?; + Some((parameter, rtype.clone())) + }); + if let Some((_, rtype)) = &underscore_at_entry { + match locals_here.iter_mut().find(|(name, _)| name == "_") { + Some((_, written)) => *written = covering(written, rtype), + None => locals_here.push(("_".to_string(), rtype.clone())), + } + } let bound: HashSet = params .iter() .map(|(name, _)| name.clone()) @@ -8214,6 +8197,9 @@ fn lower_function_with_receiver( }), }; lowering.check_parameters(function)?; + if let Some((parameter, _)) = &underscore_at_entry { + lowering.bind_underscore_at_entry(parameter)?; + } lowering.block(&function.body)?; // a body that falls off the end returns `None` — python's implicit return. what it @@ -9124,6 +9110,35 @@ fn signature( }) } +/// the name the first of a parameter list's repeated `_`s takes in the python, where that is +/// not `_` itself +/// +/// an override's `_`s take the names the method it overrides gives those positions, and the +/// python a body mentioning `_` is transpiled to binds `_` to the first of them on entry: +/// +/// ```python +/// class Quiet(Handler): +/// @override +/// def handle(self, event: int, source: str): +/// _ = event +/// del _ +/// ``` +/// +/// every read of `_` that binding reaches is refused, so what it holds is never seen, but +/// whether it is bound is: `del _` succeeds on it. a generator keeps its locals in its +/// state object, where a `del` of one declines, so it has no way to tell and is not given +/// the binding +fn underscore_bound_at_entry( + parameters: &ast::Parameters, + written: by_transforms::WrittenNames, +) -> Option { + let first = parameters + .iter() + .find(|parameter| parameter.name() == "_")?; + let name = by_transforms::python_parameter_name(parameters, first.as_parameter(), written); + (name != "_").then(|| name.to_string()) +} + /// the representation every `return` in a body agrees on /// whether control can fall off the end of this body /// @@ -15762,6 +15777,18 @@ impl Lowering<'_, '_> { }) } + /// bind `_` to `parameter`, the name the first of an override's repeated `_`s takes in + /// the python, as the python does at the top of the body + fn bind_underscore_at_entry(&mut self, parameter: &str) -> Lowered<()> { + let (Some(source), Some(target)) = (self.place(parameter), self.place("_")) else { + return Err(Decline::new(format!( + "the `_` an override binds to `{parameter}` on entry has no place in the frame" + ))); + }; + let (value, ty) = self.read_place(&source)?; + self.write_place(&target, value, &ty) + } + /// the entry checks the interpreted build makes on `function`'s own parameters, /// where its body begins fn check_parameters(&mut self, function: &ast::StmtFunctionDef) -> Lowered<()> { diff --git a/crates/by_irbuild/src/single_file.rs b/crates/by_irbuild/src/single_file.rs index 7474b9c231..ba52b78193 100644 --- a/crates/by_irbuild/src/single_file.rs +++ b/crates/by_irbuild/src/single_file.rs @@ -9,7 +9,7 @@ use ruff_db::files::{File, system_path_to_file}; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; -use ruff_python_ast::Stmt; +use ruff_python_ast::{PythonVersion, Stmt}; use ty_project::{ProjectMetadata, TestDb}; use ty_python_semantic::{LoweringModel, ProgramEnvironment}; @@ -31,6 +31,7 @@ pub(crate) fn with_source( /// build a one-file db from source of a given language and hand its model and /// suite to `f` +#[cfg(test)] pub(crate) fn with_source_in( source: &str, language: Language, @@ -41,7 +42,22 @@ pub(crate) fn with_source_in( &[Stmt], ) -> T, ) -> T { - let (db, file) = make_db(source, language); + with_source_at(source, language, PythonVersion::latest_ty(), f) +} + +/// as [`with_source_in`], with the module analysed at `python_version` +fn with_source_at( + source: &str, + language: Language, + python_version: PythonVersion, + f: impl FnOnce( + &dyn ty_python_semantic::Db, + &ProgramEnvironment<'_>, + &LoweringModel<'_>, + &[Stmt], + ) -> T, +) -> T { + let (db, file) = make_db(source, language, python_version); let program_file = ty_python_semantic::Db::program_file(&db, file); let parsed = ruff_db::parsed::parsed_module(&db, program_file.python_file(&db)).load(&db); let model = LoweringModel::new(&db, program_file); @@ -49,12 +65,13 @@ pub(crate) fn with_source_in( f(&db, &env, &model, parsed.suite()) } -fn make_db(source: &str, language: Language) -> (TestDb, File) { +fn make_db(source: &str, language: Language, python_version: PythonVersion) -> (TestDb, File) { let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/"), )); - db.init_program().expect("program init failed"); + db.init_program_with_python_version(python_version) + .expect("program init failed"); // the extension is what tells the parser which language this is let path = format!("/input.{}", language.extension()); db.write_file(&path, source).expect("write failed"); @@ -62,14 +79,34 @@ fn make_db(source: &str, language: Language) -> (TestDb, File) { (db, file) } -/// lower source into a module named `module_name` +/// lower source into a module named `module_name`, analysed at ty's newest python +/// +/// a module that is built for an interpreter wants [`module_from_source_at`]: what a name +/// is, and so what a value's representation is, can depend on the version pub fn module_from_source( source: &str, module_name: impl Into, options: impl Into, +) -> by_ir::function::ModuleIr { + module_from_source_at(source, module_name, options, PythonVersion::latest_ty()) +} + +/// lower source into a module named `module_name`, analysed at `python_version` +/// +/// a `sys.version_info` test settles which branch binds a name, so a module analysed at +/// a newer python than the one it runs on can lay a value out in the representation the +/// branch it never takes would have given it +pub fn module_from_source_at( + source: &str, + module_name: impl Into, + options: impl Into, + python_version: PythonVersion, ) -> by_ir::function::ModuleIr { let options = options.into(); - with_source_in(source, options.language, |db, env, model, suite| { - crate::build_module(db, env, model, suite, module_name, options) - }) + with_source_at( + source, + options.language, + python_version, + |db, env, model, suite| crate::build_module(db, env, model, suite, module_name, options), + ) } diff --git a/crates/by_irbuild/src/surface.rs b/crates/by_irbuild/src/surface.rs index a60ec822f6..0dcbfa29ae 100644 --- a/crates/by_irbuild/src/surface.rs +++ b/crates/by_irbuild/src/surface.rs @@ -32,13 +32,17 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_parameter, walk_stmt, walk_with_item}; use ruff_python_ast::{self as ast, Expr, Parameter, PySourceType, Stmt, TypeParam, WithItem}; use ruff_python_stdlib::identifiers::is_identifier; +use ty_python_semantic::SemanticModel; use ty_python_semantic::reified::{reified_class_reads, reified_type_param_names}; +use ty_python_semantic::types::ide_support::repeated_underscore_read_of; +use ty_python_semantic::types::repeated_underscore::RepeatedUnderscoreRead; use crate::mapper::{Decline, Lowered}; /// whether this function's body may be lowered at all pub(crate) fn gate_function( source_type: PySourceType, + model: &SemanticModel<'_>, function: &ast::StmtFunctionDef, ) -> Lowered<()> { // a reified type parameter is a runtime value the *specialization step* supplies, @@ -52,57 +56,60 @@ pub(crate) fn gate_function( "`{first}` is a reified type parameter, and a specialization has no emitted function to rebuild" ))); } - if reads_a_repeated_underscore(function) { - return Err(Decline::new( - "a body reading a repeated `_` parameter reads the first of them, where ty reads the last", - )); + if let Some(read) = repeated_underscore_read(model, function) { + return Err(Decline::new(format!( + "{}, which has no one parameter to lower it to", + read.message() + ))); } let mut scanner = Scanner { found: None }; scanner.walk_function(function); scanner.into_result() } -/// whether `function` repeats the parameter `_` and its body reads that name +/// the first read of `_` in `function` that may read one of a parameter list's repeated +/// `_`s — ty's answer, the one it reports and the transpiler refuses, so a module that +/// reaches the lowering through a transpiled build never holds one /// -/// python refuses a repeated parameter, so the transpiler keeps the first `_` and -/// numbers the rest — a read of `_` is the first parameter's value. ty binds each -/// parameter in turn and takes the read for the *last* one, so a lowering built on -/// ty's type for the read would treat one parameter's value as the other's type. -/// a body that only ignores its `_` parameters is unaffected, and is lowered under the -/// numbered names -fn reads_a_repeated_underscore(function: &ast::StmtFunctionDef) -> bool { - struct Reads { - found: bool, +/// python binds `_` to the first of the parameters and ty to the last, so such a read has +/// no one meaning. a read that only a rebinding of `_` reaches reads that rebinding, and +/// `del _` takes no value, so neither is one +fn repeated_underscore_read( + model: &SemanticModel<'_>, + function: &ast::StmtFunctionDef, +) -> Option { + struct Reads<'m, 'db> { + model: &'m SemanticModel<'db>, + found: Option, } - impl<'a> Visitor<'a> for Reads { + impl Reads<'_, '_> { + fn ask(&mut self, name: &ast::ExprName) { + if self.found.is_none() && name.id == "_" { + self.found = repeated_underscore_read_of(self.model, name); + } + } + } + impl<'a> Visitor<'a> for Reads<'_, '_> { fn visit_stmt(&mut self, stmt: &'a Stmt) { + // `_ += 1` reads `_` before it binds it if let Stmt::AugAssign(node) = stmt - && matches!(node.target.as_ref(), Expr::Name(name) if name.id == "_") + && let Expr::Name(name) = node.target.as_ref() { - self.found = true; + self.ask(name); } walk_stmt(self, stmt); } fn visit_expr(&mut self, expr: &'a Expr) { if let Expr::Name(name) = expr - && name.id == "_" - && !name.ctx.is_store() + && name.ctx.is_load() { - self.found = true; + self.ask(name); } walk_expr(self, expr); } } - let underscores = function - .parameters - .iter() - .filter(|parameter| parameter.name() == "_") - .count(); - if underscores < 2 { - return false; - } - let mut reads = Reads { found: false }; + let mut reads = Reads { model, found: None }; reads.visit_body(&function.body); reads.found } diff --git a/crates/by_irbuild/src/tests.rs b/crates/by_irbuild/src/tests.rs index 8484b01126..3130fb513a 100644 --- a/crates/by_irbuild/src/tests.rs +++ b/crates/by_irbuild/src/tests.rs @@ -1156,14 +1156,12 @@ def outer(xs: list[list[int]], y: list[int]) -> object: #[test] fn a_body_reading_a_repeated_underscore_parameter_declines() { // python's `_` is the first of the parameters, the transpiler numbering the rest, - // where ty takes a read of it for the last one. a body that ignores them all is - // lowered, and a nested frame reading the name counts as a read + // where ty takes a read of it for the last one. a transpiled build refuses such a read, + // and a lowering reached without one declines it, asking ty which reads they are. a + // nested frame reading the name counts as a read assert_eq!( declines( "\ -def ignores(_: int, _: str) -> int: - return 1 - def reads(_: int, _: str) -> object: return _ @@ -1175,13 +1173,80 @@ def captures(_: int, _: str) -> int: def inner() -> object: return _ return 0 + +def on_one_path(_: int, _: int, flag: bool) -> int: + if flag: + _ = 5 + return _ " ), - ["reads", "adds", "captures"] + ["reads", "adds", "captures", "on_one_path"] .map(|name| ( name.to_string(), - "a body reading a repeated `_` parameter reads the first of them, where ty reads the last" - .to_string() + format!( + "reading a repeated `_` parameter of `{name}`, which has no one parameter to lower it to" + ) + )) + .to_vec() + ); +} + +#[test] +fn a_body_binding_a_repeated_underscore_before_reading_it_is_lowered() { + // a write gives `_` a local of its own, so a read only a write reaches is not a read of + // any parameter, and `del _` reads no value. ty accepts all of these, and none of them + // is declined + assert_eq!( + declines( + "\ +_ = 5 + +def ignores(_: int, _: str) -> int: + return 1 + +def rebinds(_: int, _: str) -> int: + _ = 7 + return _ + +def deletes(_: int, _: str) -> int: + del _ + return 1 + +def declares_global(_: int, _: str) -> int: + def module() -> int: + global _ + return _ + return module() + +def binds_its_own(_: int, _: str) -> list[int]: + return [_ * 2 for _ in range(3)] +" + ), + [] + ); +} + +#[test] +fn an_init_of_repeated_underscore_fields_declines_as_any_init_does() { + // `init(let _: int, let _: int)` stands for a `self._ = _` the parser writes for each + // parameter, and the transpiler writes the second as `self._ = _2`. no `init(...)` is + // lowered yet, so this one declines for that reason alone — a lowering of it has to + // read each of those assignments by the name its own parameter takes, or both read + // the first + assert_eq!( + declines( + "\ +class Named: + init(let a: int, let b: int) + +class Pair: + init(let _: int, let _: int) +" + ), + ["Named", "Pair"] + .map(|name| ( + name.to_string(), + "this modifier changes what the definition is bound as, not what it is".to_string() )) .to_vec() ); diff --git a/crates/by_opt/src/borrow.rs b/crates/by_opt/src/borrow.rs index 38308038a5..edcc84810d 100644 --- a/crates/by_opt/src/borrow.rs +++ b/crates/by_opt/src/borrow.rs @@ -1318,10 +1318,10 @@ mod tests { } #[test] - fn a_named_int_destination_borrows_an_element() { - // an `int` has no finalizer, so when python lets go of what the name held is - // not something a program can see, and `part` in `head, part = split(s)` may - // hold its element on loan from the tuple + fn a_named_int_destination_does_not_borrow_an_element() { + // `part` in `head, part = split(s)` can hold an `int` subclass with a `__del__`, + // which python lets go of when the name goes. on loan from the tuple it would go + // when the tuple did, after the last read of `part` let mut builder = FunctionBuilder::new("f", RType::INT); let text = builder.param("s", RType::STR); let pair = builder.temp(RType::Tuple(Box::new([RType::STR, RType::INT]))); @@ -1348,7 +1348,7 @@ mod tests { builder.terminate(Terminator::Return(Value::Register(sum))); let mut m = module(builder.finish()); run(&mut m); - assert!(m.functions[0].registers[part.index()].borrowed); + assert!(!m.functions[0].registers[part.index()].borrowed); assert_eq!(verify(&m.functions[0]), Ok(())); } diff --git a/crates/by_opt/src/dead_allocations.rs b/crates/by_opt/src/dead_allocations.rs index c9eedd95e2..214ae4bf6e 100644 --- a/crates/by_opt/src/dead_allocations.rs +++ b/crates/by_opt/src/dead_allocations.rs @@ -12,18 +12,184 @@ //! runs to a fixpoint: the tuple goes first, and its two boxed elements are only //! unread once it has. -use std::collections::HashSet; +use std::collections::{BTreeMap, BTreeSet, HashSet}; use by_ir::function::{Function, ModuleIr}; use by_ir::ops::{Op, RegisterId, Value}; +use by_ir::rtype::{Primitive, RType}; pub(crate) fn run(module: &mut ModuleIr) { for function in module.all_functions_mut() { + unbox_unread_tuples(function); // bounded by the op count: every round removes at least one op, or stops while drop_unread(function) {} } } +/// what [`unbox_unread_tuples`] writes into one register: the type of the struct, and each +/// display rewritten, by block and op index +type Rewrite = (RType, Vec<(usize, usize, Op)>); + +/// hold a tuple nothing reads as the struct of its unboxed elements rather than as an +/// object built from their boxes +/// +/// a tuple nothing reads is still a holder of what it was built from: a name keeps it +/// until it is rebound, and letting it go lets its elements go last to first. that is all +/// of it a program can see, and a fixed-length tuple held as a struct does exactly the +/// same — it holds a reference per element and releases them last to first — without an +/// allocation for the tuple or for a box per element. `pair = (whole, part)` over two +/// `int`s, with its reads folded into reads of the elements, is the shape: an `int` +/// subclass with a `__del__` can see when the tuple goes, so the tuple cannot simply go, +/// but nothing can see what the tuple is. +/// +/// every write of the register has to be such a display, each with the same element +/// types, because the register takes the struct's type. an element is taken from its box +/// only where the box is in the same block with nothing writing the boxed register between +/// the two, so the struct holds the value the box was made from +fn unbox_unread_tuples(function: &mut Function) { + let mut mentioned: HashSet = HashSet::new(); + for block in &function.blocks { + for op in &block.ops { + for operand in op.operands() { + if let Value::Register(id) = operand { + mentioned.insert(*id); + } + } + mentioned.extend(op.unbinds()); + mentioned.extend(op.loop_cursor()); + } + for operand in block.terminator.operands() { + if let Value::Register(id) = operand { + mentioned.insert(*id); + } + } + } + for default in function.defaults.iter().flatten() { + if let Value::Register(id) = default { + mentioned.insert(*id); + } + } + + // each candidate's rewritten displays, by block and op index, and the element types + // they all agree on — `None` once a write disagrees or cannot be rewritten + let mut rewrites: BTreeMap> = BTreeMap::new(); + for (block_index, block) in function.blocks.iter().enumerate() { + for (op_index, op) in block.ops.iter().enumerate() { + let Some(dest) = op.dest() else { + continue; + }; + if function.param_count > dest.index() + || mentioned.contains(&dest) + || function.register(dest).is_none_or(|decl| { + decl.may_be_unassigned + || !matches!(decl.ty, RType::OBJECT | RType::Primitive(Primitive::Tuple)) + }) + { + continue; + } + let rewritten = match op { + Op::BuildTuple { items, moves, .. } => { + unboxed_display(function, &block.ops[..op_index], dest, items, moves) + } + _ => None, + }; + let entry = rewrites + .entry(dest) + .or_insert_with(|| Some((RType::OBJECT, Vec::new()))); + match (entry.as_mut(), rewritten) { + (Some((ty, ops)), Some((element_types, rewritten))) => { + if ops.is_empty() { + *ty = element_types; + } else if *ty != element_types { + *entry = None; + continue; + } + ops.push((block_index, op_index, rewritten)); + } + _ => *entry = None, + } + } + } + for (register, rewrite) in rewrites { + let Some((ty, ops)) = rewrite else { + continue; + }; + if ops.is_empty() { + continue; + } + for (block_index, op_index, op) in ops { + function.blocks[block_index].ops[op_index] = op; + } + if let Some(decl) = function.registers.get_mut(register.index()) { + decl.ty = ty; + } + } +} + +/// a tuple display written into `dest` as the struct of its elements, each box among them +/// replaced by what it boxed, with the struct's type; `None` where nothing was a box +fn unboxed_display( + function: &Function, + before: &[Op], + dest: RegisterId, + items: &[Value], + moves: &BTreeSet, +) -> Option<(RType, Op)> { + let mut unboxed = Vec::with_capacity(items.len()); + let mut kept_moves = BTreeSet::new(); + let mut any = false; + for (index, item) in items.iter().enumerate() { + let boxed = match item { + Value::Register(id) => boxed_in_block(before, *id), + _ => None, + }; + match boxed { + Some(src) => { + unboxed.push(src.clone()); + any = true; + } + None => { + if moves.contains(&index) { + kept_moves.insert(index); + } + unboxed.push(item.clone()); + } + } + } + if !any { + return None; + } + let types = unboxed + .iter() + .map(|item| function.value_type(item)) + .collect::>>()?; + Some(( + RType::Tuple(types.into()), + Op::TupleBuild { + dest, + items: unboxed, + moves: kept_moves, + }, + )) +} + +/// what `register` boxes, where the last write of it among `before` is a box and nothing +/// after that box writes the boxed register +fn boxed_in_block(before: &[Op], register: RegisterId) -> Option<&Value> { + let at = before.iter().rposition(|op| op.dest() == Some(register))?; + let Op::Box { src, .. } = &before[at] else { + return None; + }; + if let Value::Register(source) = src + && before[at + 1..] + .iter() + .any(|op| op.dest() == Some(*source) || op.unbinds() == Some(*source)) + { + return None; + } + Some(src) +} + /// whether an op does nothing but compute the value it names /// /// the whitelist is deliberately short and every entry is checked against the C it @@ -139,9 +305,10 @@ fn drop_unread(function: &mut Function) -> bool { /// /// a representation with no object reference in it is one, and so is an object every /// write of which boxes such a value, or builds a tuple of them. that is `pair` in -/// `pair = (whole, part)` over two `int`s, once its reads have been folded into reads -/// of the elements. a tagged `int` counts as having no finalizer, as it does for the -/// release pass +/// `pair = (whole, part)` over two `float`s, once its reads have been folded into reads +/// of the elements. a tagged `int` is not one: the object behind its pointer can be an +/// `int` subclass with a `__del__`, and a tuple of two of them lets go of the second +/// first fn finalizer_free(function: &Function) -> HashSet { let mut free: HashSet = function .registers @@ -245,10 +412,42 @@ mod tests { } #[test] - fn a_name_nothing_reads_holding_only_ints_goes() { - // `pair = (whole, part)` over two `int`s, with its reads folded into reads of - // the elements: letting go of a tuple of boxed ints runs nothing, so when it + fn a_name_nothing_reads_holding_only_floats_goes() { + // `pair = (whole, part)` over two `float`s, with its reads folded into reads of + // the elements: letting go of a tuple of boxed floats runs nothing, so when it // happens cannot be told apart, and the tuple and both boxes go + let mut builder = FunctionBuilder::new("f", RType::FLOAT); + let whole = builder.param("whole", RType::FLOAT); + let part = builder.param("part", RType::FLOAT); + let first = builder.temp(RType::OBJECT); + let second = builder.temp(RType::OBJECT); + let pair = builder.local("pair".to_string(), RType::OBJECT); + builder.push(Op::Box { + dest: first, + src: Value::Register(whole), + }); + builder.push(Op::Box { + dest: second, + src: Value::Register(part), + }); + builder.push(Op::BuildTuple { + dest: pair, + items: vec![Value::Register(first), Value::Register(second)], + moves: BTreeSet::new(), + }); + builder.terminate(Terminator::Return(Value::Register(whole))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert!(module.functions[0].blocks[0].ops.is_empty()); + } + + #[test] + fn a_name_nothing_reads_holding_ints_is_held_as_their_struct() { + // `pair = (whole, part)` over two `int`s, with its reads folded into reads of + // the elements: the name still holds both until it goes, and holding them as a + // struct of the two lets them go in the same order with nothing allocated let mut builder = FunctionBuilder::new("f", RType::INT); let whole = builder.param("whole", RType::INT); let part = builder.param("part", RType::INT); @@ -266,14 +465,79 @@ mod tests { builder.push(Op::BuildTuple { dest: pair, items: vec![Value::Register(first), Value::Register(second)], - moves: BTreeSet::new(), + moves: [0, 1].into_iter().collect(), }); builder.terminate(Terminator::Return(Value::Register(whole))); let mut module = module(builder.finish()); super::run(&mut module); - assert!(module.functions[0].blocks[0].ops.is_empty()); + let function = &module.functions[0]; + assert_eq!( + function.blocks[0].ops, + [Op::TupleBuild { + dest: pair, + items: vec![Value::Register(whole), Value::Register(part)], + moves: BTreeSet::new(), + }] + ); + assert_eq!( + function.registers[pair.index()].ty, + RType::Tuple(Box::new([RType::INT, RType::INT])) + ); + assert_eq!(by_ir::verify::verify(function), Ok(())); + } + + #[test] + fn a_tuple_read_after_it_is_built_keeps_its_boxes() { + let mut builder = FunctionBuilder::new("f", RType::OBJECT); + let whole = builder.param("whole", RType::INT); + let first = builder.temp(RType::OBJECT); + let pair = builder.local("pair".to_string(), RType::OBJECT); + builder.push(Op::Box { + dest: first, + src: Value::Register(whole), + }); + builder.push(Op::BuildTuple { + dest: pair, + items: vec![Value::Register(first)], + moves: BTreeSet::new(), + }); + builder.terminate(Terminator::Return(Value::Register(pair))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert!(matches!( + module.functions[0].blocks[0].ops[1], + Op::BuildTuple { .. } + )); + assert_eq!( + module.functions[0].registers[pair.index()].ty, + RType::OBJECT + ); + } + + #[test] + fn a_tuple_nothing_reads_holding_ints_stays() { + // `ints(f)` thrown away, where `ints` answers `(f(1), f(2))` as two `int`s: either + // can be an `int` subclass with a `__del__`, and the tuple is what lets them go + // last to first rather than each after its own last read + let mut builder = FunctionBuilder::new("f", RType::INT); + let first = builder.param("first", RType::INT); + let second = builder.param("second", RType::INT); + let pair = builder.temp(RType::Tuple(Box::new([RType::INT, RType::INT]))); + builder.push(Op::TupleBuild { + dest: pair, + items: vec![Value::Register(first), Value::Register(second)], + moves: BTreeSet::new(), + }); + builder.terminate(Terminator::Return(Value::Register(first))); + + let mut module = module(builder.finish()); + super::run(&mut module); + + assert_eq!(module.functions[0].blocks[0].ops.len(), 1); } #[test] diff --git a/crates/by_opt/src/field_proofs.rs b/crates/by_opt/src/field_proofs.rs index d888b10461..7c11a956d7 100644 --- a/crates/by_opt/src/field_proofs.rs +++ b/crates/by_opt/src/field_proofs.rs @@ -2,11 +2,9 @@ //! //! an instance `__new__` made without running `__init__` has none of the fields //! `__init__` gives every other instance, so a read of one is preceded by a test that -//! raises `AttributeError` where the field is absent. but a field that is always -//! assigned cannot be taken away again once it has been — `del` of one is refused, and -//! only a field `__init__` might leave out can be deleted, whose read tests for itself. -//! so once a function has written a field of an object, or tested it, every later read -//! of that field of that same object finds it: +//! raises `AttributeError` where the field is absent. once a function has written a field +//! of an object, or tested it, a later read of that field of that same object finds it — +//! until python code runs, because python code can `del` it again: //! //! ```python //! def step(state: State) -> None: @@ -17,12 +15,16 @@ //! "that same object" is a register that has not been written in between. a register //! written again may hold another object, and one written on some path into a block is //! as good as written. a handler is entered from the middle of the blocks it covers, so -//! what it can rely on is only what held all through each of them +//! what it can rely on is only what held all through each of them. what python code can +//! run is [`Effects`]' answer, which is the one every other pass that holds a fact across +//! operations asks use std::collections::{HashMap, HashSet}; -use by_ir::function::{BasicBlock, Function, ModuleIr}; -use by_ir::ops::{BlockId, Op, RegisterId, Value}; +use by_ir::function::{Function, ModuleIr}; +use by_ir::ops::{BlockId, Op, RegisterId, Terminator, Value}; + +use crate::runs_python::{Effects, Held}; /// a field of the object a register holds, which the function knows the object has type Known = (RegisterId, String); @@ -31,8 +33,9 @@ type Known = (RegisterId, String); type Layouts = HashMap>>; pub(crate) fn run(module: &mut ModuleIr) { - // only a field nothing can take away again: an optional one can be deleted by - // anything the function calls, and a field no instance lacks has nothing to prove + let effects = Effects::of(module); + // only a field a read tests: an optional one's read asks for itself, and a field no + // instance lacks has nothing to prove let layouts: HashMap>> = module .classes .iter() @@ -50,10 +53,55 @@ pub(crate) fn run(module: &mut ModuleIr) { }) .collect(); for function in module.all_functions_mut() { - prove(function, &layouts); + let runs = runs_python(&effects, function); + prove(function, &layouts, &runs); } } +/// for each operation of each block, whether python code can run while it does — and so +/// take away a field the function had seen. a block the analysis never reached is taken to +/// run it everywhere +fn runs_python(effects: &Effects, function: &Function) -> Vec> { + let entries = Held::solve( + function, + Function::entry(), + effects.held_at_entry(function), + |_| true, + ); + function + .blocks + .iter() + .enumerate() + .map(|(index, block)| match entries.get(&BlockId(index)) { + Some(entry) => { + let mut held = entry.clone(); + block + .ops + .iter() + .map(|op| { + let runs = effects.op_runs_python(function, op, &held); + held.step(function, op); + runs + }) + .collect() + } + None => vec![true; block.ops.len()], + }) + .collect() +} + +/// whether control leaves a block through `terminator` without running python code: a +/// jump, or a test of a value the block already holds +fn leaves_quietly(terminator: &Terminator) -> bool { + matches!( + terminator, + Terminator::Goto(_) + | Terminator::Branch { .. } + | Terminator::NarrowShort { .. } + | Terminator::MachineStep { .. } + ) +} + /// what an operation lets the function know about fields afterwards fn learned(op: &Op, layouts: &Layouts) -> Vec { let permanent = |class: &str, field: &str| { @@ -97,25 +145,22 @@ fn rebinds(op: &Op) -> Vec { .collect() } -/// `known` after `op` has run -fn step(known: &mut HashSet, op: &Op, layouts: &Layouts) { +/// `known` after `op` has run, where `runs` says whether python code can run while it does +/// +/// what the operation itself writes or tests is learned after what python code took away: +/// a store releases the value it replaces only once the new one is in place, so a +/// finalizer that release runs sees the field written +fn step(known: &mut HashSet, op: &Op, runs: bool, layouts: &Layouts) { + if runs { + known.clear(); + } for register in rebinds(op) { known.retain(|(held, _)| *held != register); } known.extend(learned(op, layouts)); } -/// what holds at every point of a block, for a handler it can raise into -fn throughout(block: &BasicBlock, entry: &HashSet) -> HashSet { - let rebound: HashSet = block.ops.iter().flat_map(rebinds).collect(); - entry - .iter() - .filter(|(held, _)| !rebound.contains(held)) - .cloned() - .collect() -} - -fn prove(function: &mut Function, layouts: &Layouts) { +fn prove(function: &mut Function, layouts: &Layouts, runs: &[Vec]) { let count = function.blocks.len(); if count == 0 { return; @@ -132,8 +177,14 @@ fn prove(function: &mut Function, layouts: &Layouts) { }; let block = &function.blocks[index]; let mut known = start.clone(); - for op in &block.ops { - step(&mut known, op, layouts); + // what held at every point of the block, for a handler it can raise into + let mut throughout = start.clone(); + for (at, op) in block.ops.iter().enumerate() { + step(&mut known, op, runs[index][at], layouts); + throughout.retain(|fact| known.contains(fact)); + } + if !leaves_quietly(&block.terminator) { + known.clear(); } // the exception edge is the last of the successors, and it carries only what // held all through the block @@ -146,7 +197,7 @@ fn prove(function: &mut Function, layouts: &Layouts) { .map(|successor| (successor, known.clone())) .collect(); if let Some(handler) = block.error_target { - flows.push((handler, throughout(block, &start))); + flows.push((handler, throughout)); } for (successor, facts) in flows { let Some(slot) = entry.get_mut(successor.index()) else { @@ -169,7 +220,7 @@ fn prove(function: &mut Function, layouts: &Layouts) { continue; }; let mut kept = Vec::with_capacity(block.ops.len()); - for op in block.ops.drain(..) { + for (at, op) in block.ops.drain(..).enumerate() { let op = match op { Op::RequireField { receiver: Value::Register(receiver), @@ -202,7 +253,7 @@ fn prove(function: &mut Function, layouts: &Layouts) { }, other => other, }; - step(&mut known, &op, layouts); + step(&mut known, &op, runs[index][at], layouts); kept.push(op); } block.ops = kept; @@ -349,6 +400,49 @@ mod tests { ); } + #[test] + fn a_field_is_tested_again_once_python_code_can_have_run() { + // python code can `del` a field the function had seen, so a call into it ends what + // the function knows — while a store keeps what it wrote, which is in place before + // the value it replaced is let go of + let mut builder = FunctionBuilder::new("f", RType::INT); + let s = builder.param("s", state()); + let value = builder.param("v", RType::INT); + let out = builder.temp(RType::INT); + let called = builder.temp(RType::OBJECT); + builder.push(require(s, "a")); + builder.push(Op::CallPython { + dest: called, + callee: "hook".to_string(), + args: vec![Value::Register(s)], + }); + builder.push(require(s, "a")); + builder.push(Op::SetField { + receiver: Value::Register(s), + class: "State".to_string(), + field: "b".to_string(), + value: Value::Register(value), + moves: false, + present: false, + }); + builder.push(require(s, "a")); + builder.push(require(s, "b")); + builder.push(read(out, s, "a")); + builder.terminate(Terminator::Return(Value::Register(out))); + let mut module = ModuleIr::new("app"); + module.classes.push(class(&[("a", false), ("b", false)])); + module.functions.push(builder.finish()); + + super::run(&mut module); + + let ops = &module.functions[0].blocks[0].ops; + assert_eq!(requires(&module), 3, "{ops:?}"); + assert!( + matches!(&ops[4], Op::RequireField { field, .. } if field == "a"), + "{ops:?}" + ); + } + #[test] fn a_rebound_receiver_or_an_optional_field_is_tested_again() { // the register may hold another object once written, and an optional field can be diff --git a/crates/by_opt/src/release_temporaries.rs b/crates/by_opt/src/release_temporaries.rs index ea04a6b059..133339f236 100644 --- a/crates/by_opt/src/release_temporaries.rs +++ b/crates/by_opt/src/release_temporaries.rs @@ -221,11 +221,11 @@ fn release_temporaries(function: &mut Function) { /// the places this pass may release: those of temporaries that own one reference apiece /// -/// a tagged `int` is one of them. its short case owns nothing at all and its overflow -/// case owns exactly one `PyLongObject`, and an `int` has no finalizer, so nothing about -/// *when* it is let go of is observable — what is observable is that the register is left -/// holding a reference until the next write of it, and the release that write makes then -/// reads a value from whichever trip round the loop wrote it last +/// a tagged `int` is one of them. its short case owns nothing at all and its pointer case +/// owns exactly one object, which can be an `int` subclass with a `__del__` — so it is let +/// go of after its last read, which is where python drops a temporary, rather than left +/// holding a reference until the next write of the register, whose release would then +/// read a value from whichever trip round the loop wrote it last fn candidates(function: &Function) -> HashSet { // a loop's cursor is read and written by the operation that advances it, outside // the operands this pass follows, so it is never offered diff --git a/crates/by_rt/include/by.h b/crates/by_rt/include/by.h index 1613f2c36f..7eefce5209 100644 --- a/crates/by_rt/include/by.h +++ b/crates/by_rt/include/by.h @@ -1734,7 +1734,9 @@ static inline int By_InlineIntCompare(PyObject *a, PyObject *b, int op) { * some class and each literal is an `int` or a `str` that refuses it. it also skips the * recursion guard `PyObject_RichCompare` enters, which costs one level of headroom once * rather than a level per round of a recursion that goes through it — the call it makes - * next is counted either way. see runtime.md#how-deep-a-recursion-goes */ + * next is counted either way. that holds from 3.12, where the guard is against the C stack; + * 3.11 counts it against the recursion limit, and there it is entered as python enters it. + * see runtime.md#how-deep-a-recursion-goes */ static inline int By_RightOperandDecides(PyObject *a, PyObject *b, int op) { (void)b; return (op == Py_EQ || op == Py_NE) && a != b @@ -1750,7 +1752,20 @@ static inline int By_RightOperandDecides(PyObject *a, PyObject *b, int op) { static inline PyObject *By_AskRightOperand(PyObject *a, PyObject *b, int op) { richcmpfunc other = Py_TYPE(b)->tp_richcompare; if (other == NULL) return Py_NewRef(Py_NotImplemented); +#if PY_VERSION_HEX >= 0x030C0000 return other(b, a, op); +#else + /* 3.11 has one count for C calls and python frames, so the guard + * `PyObject_RichCompare` enters is a frame of the recursion limit there, and it is + * taken as python takes it */ + { + PyObject *answer; + if (Py_EnterRecursiveCall(" in comparison")) return NULL; + answer = other(b, a, op); + Py_LeaveRecursiveCall(); + return answer; + } +#endif } /* `a b` through the abstract protocol, as a bit — and 2 means an exception is set @@ -1823,6 +1838,26 @@ static inline PyObject *By_ObjRichCompare(PyObject *a, PyObject *b, int op) { return PyObject_RichCompare(a, b, op); } +/* name a class built from a spec as a `class` statement names one: by its bare name + * + * python's own messages print a type's `tp_name` — `'Box' object is not callable`, + * `cannot unpack non-iterable Box object` — which a `class` statement leaves the bare + * name, and a spec the dotted one it was given. a heap type answers `__module__`, + * `__name__` and `__qualname__` out of its dict and its own fields rather than out of + * `tp_name`, so pointing it at the bare name changes those messages and nothing else: + * `repr`, pickling and `__module__` are as they were. it is what assigning + * `type.__name__` does to a class, and the storage is the same — the type's own name + * object, which lives as long as the type does. a static type reads its module off the + * front of `tp_name`, and is left as it is */ +static inline int By_BareTypeName(PyObject *cls) { + if (!PyType_Check(cls) || !PyType_HasFeature((PyTypeObject *)cls, Py_TPFLAGS_HEAPTYPE)) + return 0; + const char *bare = PyUnicode_AsUTF8(((PyHeapTypeObject *)cls)->ht_name); + if (bare == NULL) return -1; + ((PyTypeObject *)cls)->tp_name = bare; + return 0; +} + /* the type python names in an `AttributeError` * * the instance's *own* type, not the class that declared the field: a subclass @@ -1840,14 +1875,18 @@ static inline const char *By_TypeName(PyObject *o) { * out of line, because it is the rare branch beside every field read nothing proves * assigned, and building the message there made those reads' functions too big to * inline into their callers. `qualified` names the type in full, which is how a slot's - * descriptor words it */ + * descriptor words it from 3.13 — `%T` is 3.13's, and before it the descriptor named the + * type by its `tp_name`, which for the twin's class is the bare name */ BY_COLD void By_FieldMissing(PyObject *o, const char *name, int qualified) { +#if PY_VERSION_HEX >= 0x030D0000 if (qualified) { PyErr_Format(PyExc_AttributeError, "'%T' object has no attribute '%s'", o, name); - } else { - PyErr_Format(PyExc_AttributeError, "'%s' object has no attribute '%s'", By_TypeName(o), - name); + return; } +#else + (void)qualified; +#endif + PyErr_Format(PyExc_AttributeError, "'%s' object has no attribute '%s'", By_TypeName(o), name); } /* raise the `AttributeError` a descriptor with no setter raises, for a field that @@ -2752,6 +2791,10 @@ BY_HOT char By_DepthEnter(ByDepth depth) { BY_HOT void By_DepthLeave(ByDepth depth) { (void)depth; } +BY_HOT char By_DepthEnterConstruction(ByDepth depth) { return By_DepthEnter(depth); } + +BY_HOT void By_DepthLeaveConstruction(ByDepth depth) { (void)depth; } + #else typedef struct { @@ -2799,6 +2842,31 @@ BY_HOT char By_DepthEnter(ByDepth depth) { /* give back the frame an `By_DepthEnter` that succeeded took */ BY_HOT void By_DepthLeave(ByDepth depth) { depth.thread->BY_PY_RECURSION_REMAINING++; } +/* a construction compiled code makes on a cycle of compiled calls, which is python's call + * of the type and then the frame of the `__init__` that call runs. from 3.12 the type's + * call is counted against the C stack's own count and only the frame is the recursion + * limit's, but 3.11 has one count for both, so there a construction takes two */ +BY_HOT char By_DepthEnterConstruction(ByDepth depth) { +#if PY_VERSION_HEX < 0x030C0000 + if (By_DepthEnter(depth)) return 1; + if (BY_UNLIKELY(By_DepthEnter(depth))) { + By_DepthLeave(depth); + return 1; + } + return 0; +#else + return By_DepthEnter(depth); +#endif +} + +/* give back what a `By_DepthEnterConstruction` that succeeded took */ +BY_HOT void By_DepthLeaveConstruction(ByDepth depth) { +#if PY_VERSION_HEX < 0x030C0000 + By_DepthLeave(depth); +#endif + By_DepthLeave(depth); +} + #endif /* BY_RECURSION_STACK_ONLY */ /* a global read that remembers what the name resolved to last time @@ -3446,10 +3514,15 @@ static void By_FindFunctionWatch(void) { * at all — before 3.12, or free-threaded — takes note of what the function holds * instead, and asks again at each call. `called` says whether any compiled call asks the * site: one nobody asks needs no word of writes to its name, only of what happens to the - * function its own python entry stands for */ -static int By_ArmFunctionSite(ByFunctionSite *site, PyObject *dict, const char *name, int called) { + * function its own python entry stands for. `published` says whether init published the + * function at all: one whose name the module body wrote over is not, and its site is left + * never standing, so every call goes through whatever the name holds — see + * `By_DefinitionStanding` */ +static int By_ArmFunctionSite(ByFunctionSite *site, PyObject *dict, const char *name, int called, + int published) { if (site->name == NULL) site->name = By_InternedStr(name, (Py_ssize_t)strlen(name)); if (site->name == NULL) return -1; + if (!published) return 0; PyObject *forwarder = PyDict_GetItemWithError(dict, site->name); if (forwarder == NULL) { if (!PyErr_Occurred()) { @@ -4317,8 +4390,8 @@ static PyObject *By_ClassMethodCall(PyObject *fn, PyObject *const *args, size_t return result; } -static PyObject *By_Method_vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, - PyObject *kwnames) { +static PyObject *By_Method_dispatch(PyObject *callable, PyObject *const *args, size_t nargsf, + PyObject *kwnames) { PyObject *fn = ((ByMethodObject *)callable)->fn; if (BY_LIKELY(Py_IS_TYPE(fn, &PyMethodDescr_Type))) { vectorcallfunc call = ((PyMethodDescrObject *)fn)->vectorcall; @@ -4330,6 +4403,28 @@ static PyObject *By_Method_vectorcall(PyObject *callable, PyObject *const *args, return PyObject_Vectorcall(fn, args, nargsf, kwnames); } +/* a call to a compiled method, which counts the frame python's method would have pushed + * itself, once the receiver is checked + * + * the builtin the call goes to enters a recursion of its own first. from 3.12 that is + * against the C stack's count, but 3.11 has one count for both, so there a call from + * python would have taken two frames where python's own method takes one. the frame + * is given back for the length of the call, which leaves the builtin's entry taking + * nothing and the compiled body's own count the only one, as on every later version */ +static PyObject *By_Method_vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, + PyObject *kwnames) { +#if PY_VERSION_HEX >= 0x030C0000 + return By_Method_dispatch(callable, args, nargsf, kwnames); +#else + PyThreadState *thread = PyThreadState_Get(); + PyObject *result; + thread->recursion_remaining++; + result = By_Method_dispatch(callable, args, nargsf, kwnames); + thread->recursion_remaining--; + return result; +#endif +} + static PyObject *By_Method_descr_get(PyObject *self, PyObject *obj, PyObject *type) { (void)type; if (obj == NULL || obj == Py_None) return By_NewRef(self); @@ -4479,7 +4574,10 @@ static PyTypeObject By_MethodType = { /* settle `by_method_type`: the type an earlier module shared, or else this module's own, * readied and shared. the key carries the layout's version, so two modules only ever - * share a type whose objects they both read the same way */ + * share a type whose objects they both read the same way. a call is not the type's to + * answer: it goes through the `vectorcall` each object carries, which is the one the module + * that made the object wrote, so what a call counts against the recursion limit is never + * the first module's */ static int By_FindMethodType(void) { PyInterpreterState *interpreter; PyObject *state, *found; @@ -5685,34 +5783,49 @@ static inline char By_IsMatchSequence(PyObject *o) { return (char)(o != NULL && PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_SEQUENCE)); } -/* the refusal `__annotations__` gives where a class's own could not be carried across +/* the refusal `__annotations__` gives where a class's own could not be carried across, + * and `__annotate__` where the function computing them could not * * `type_get_annotations` hands whatever it finds under that name in `tp_dict` to its own - * `tp_descr_get`, so a descriptor written there is how an emitted type says no. it has - * to be able to: an *absent* `__annotations__` is not a loud failure the way an absent - * attribute is, because python invents an empty mapping for it on the spot — which is a - * wrong answer wearing a right one's clothes */ + * `tp_descr_get`, and 3.14's `type_get_annotate` does the same, so a descriptor written + * there is how an emitted type says no. it has to be able to: an *absent* + * `__annotations__` is not a loud failure the way an absent attribute is, because python + * invents an empty mapping for it on the spot — which is a wrong answer wearing a right + * one's clothes — and an absent `__annotate__` is `None`, which says the class wrote no + * annotations at all */ +typedef struct { + PyObject_HEAD + const char *name; +} By_LostAnnotationsObject; + static PyObject *By_LostAnnotations_get(PyObject *self, PyObject *object, PyObject *type) { - (void)self; (void)object; - PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '__annotations__'", + PyErr_Format(PyExc_AttributeError, "type object '%s' has no attribute '%s'", type != NULL && PyType_Check(type) ? ((PyTypeObject *)type)->tp_name - : ""); + : "", + ((By_LostAnnotationsObject *)self)->name); return NULL; } static PyTypeObject By_LostAnnotationsType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "by.lost_annotations", - .tp_basicsize = sizeof(PyObject), + .tp_basicsize = sizeof(By_LostAnnotationsObject), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_descr_get = By_LostAnnotations_get, }; -static inline PyObject *By_LostAnnotations(void) { +static inline PyObject *By_LostAnnotationsNamed(const char *name) { + By_LostAnnotationsObject *lost; if (PyType_Ready(&By_LostAnnotationsType) < 0) return NULL; - return PyObject_New(PyObject, &By_LostAnnotationsType); + lost = PyObject_New(By_LostAnnotationsObject, &By_LostAnnotationsType); + if (lost != NULL) lost->name = name; + return (PyObject *)lost; +} + +static inline PyObject *By_LostAnnotations(void) { + return By_LostAnnotationsNamed("__annotations__"); } /* where a class keeps the annotations its body wrote, once they are a mapping @@ -5864,15 +5977,86 @@ static int By_SettlesInPlace(PyObject *value, const By_Twins *twins, int depth) return same; } -/* settle every value of a dict, in place, and say whether the dict now holds no twin +/* whether every way python has of writing to `container` is `base`'s own, so a write made + * through the C API is the one any of its own operations would make + * + * a subclass that writes one of them may keep its own account of what it holds beside the + * table — `OrderedDict` keeps its order in a list of its own — and a write that went past + * that account would leave it describing members the table no longer holds */ +static int By_WritesAsBase(PyObject *container, PyTypeObject *base) { + static const char *const dict_writes[] = {"__setitem__", "__delitem__", "clear", "update", + "pop", "popitem", "setdefault", "__ior__", NULL}; + static const char *const set_writes[] = {"add", "discard", "remove", "clear", "update", + "pop", "difference_update", "intersection_update", + "symmetric_difference_update", "__ior__", "__iand__", + "__isub__", "__ixor__", NULL}; + const char *const *name; + PyTypeObject *type = Py_TYPE(container); + if (type == base) return 1; + for (name = base == &PyDict_Type ? dict_writes : set_writes; *name != NULL; name++) { + PyObject *key = PyUnicode_InternFromString(*name); + int same; + if (key == NULL) { + PyErr_Clear(); + return 0; + } + same = _PyType_Lookup(type, key) == _PyType_Lookup(base, key); + Py_DECREF(key); + if (!same) return 0; + } + return 1; +} + +/* put `rekeyed` — `(key, replacement)` pairs — in place of those keys of a dict whose writes + * are `dict`'s own, in the order the dict holds them, and say whether that was done + * + * a dict is keyed on what an object hashes as, so a key is replaced by building the + * mapping again rather than by a removal and an insertion, which would move the entry to + * the end. it is built aside first and the original only emptied and filled from it once + * every key has gone in, so a key that cannot be hashed leaves the dict as it was */ +static int By_RekeyDict(PyObject *dict, PyObject *rekeyed) { + PyObject *items = PyDict_Items(dict); + PyObject *rebuilt = PyDict_New(); + Py_ssize_t at, pair; + int done = items != NULL && rebuilt != NULL; + for (at = 0; done && at < PyList_GET_SIZE(items); at++) { + PyObject *item = PyList_GET_ITEM(items, at); + PyObject *key = PyTuple_GET_ITEM(item, 0); + for (pair = 0; pair < PyList_GET_SIZE(rekeyed); pair++) { + PyObject *replacing = PyList_GET_ITEM(rekeyed, pair); + if (PyTuple_GET_ITEM(replacing, 0) == key) { + key = PyTuple_GET_ITEM(replacing, 1); + break; + } + } + done = PyDict_SetItem(rebuilt, key, PyTuple_GET_ITEM(item, 1)) == 0; + } + done = done && PyDict_GET_SIZE(rebuilt) == PyList_GET_SIZE(items); + if (done) { + PyDict_Clear(dict); + done = PyDict_Update(dict, rebuilt) == 0; + } + if (!done) PyErr_Clear(); + Py_XDECREF(items); + Py_XDECREF(rebuilt); + return done; +} + +/* settle every key and value of a dict, in place, and say whether the dict now holds no + * twin * - * a twin used as a *key* is refused rather than moved: a dict is keyed on the object a - * class hashes as, so replacing one is a removal and an insertion, and doing that to a - * mapping somebody else is holding is a bigger claim than this has evidence for */ + * a key moves the way a value does — a twin becomes the type standing for it and an + * instance one of them built is moved onto it — because a mapping keyed on a class or on + * one of its instances is asked with what the class makes *now*: `TABLE[Pair(1, 2)]` + * builds its key with the compiled class, and a dispatch table is asked with + * `type(obj)`. only a dict whose writes are all `dict`'s own is keyed again (see + * `By_RekeyDict` and `By_WritesAsBase`): a subclass that writes one of its own may keep its + * own account of its keys beside the table, and is refused where a key moves */ static int By_SettleDictValues(PyObject *dict, const By_Twins *twins, int depth) { /* the keys first: a value is settled one at a time and anything at all may run while * that happens, so nothing may be walking the dict itself */ PyObject *keys = PyDict_Keys(dict); + PyObject *rekeyed = NULL; Py_ssize_t at; int settled = 1; if (keys == NULL) { @@ -5884,13 +6068,25 @@ static int By_SettleDictValues(PyObject *dict, const By_Twins *twins, int depth) /* held rather than borrowed: settling one entry can run arbitrary code, and the * entry this is about to write back over must not have gone away underneath it */ PyObject *value = By_NewRef(PyDict_GetItem(dict, key)); - PyObject *stands; + PyObject *stands, *moved; if (value == NULL) continue; - if (!By_SettlesInPlace(key, twins, depth)) { + moved = By_SettledValue(key, twins, depth); + if (moved == NULL) { Py_DECREF(value); settled = 0; continue; } + if (moved != key) { + PyObject *pair = + By_WritesAsBase(dict, &PyDict_Type) ? PyTuple_Pack(2, key, moved) : NULL; + if (pair == NULL || (rekeyed == NULL && (rekeyed = PyList_New(0)) == NULL) + || PyList_Append(rekeyed, pair) < 0) { + PyErr_Clear(); + settled = 0; + } + Py_XDECREF(pair); + } + Py_DECREF(moved); stands = By_SettledValue(value, twins, depth); if (stands == NULL) { Py_DECREF(value); @@ -5905,9 +6101,161 @@ static int By_SettleDictValues(PyObject *dict, const By_Twins *twins, int depth) Py_DECREF(value); } Py_DECREF(keys); + if (rekeyed != NULL) { + if (!By_RekeyDict(dict, rekeyed)) settled = 0; + Py_DECREF(rekeyed); + } return settled; } +/* the slot of a set's table holding `member` itself, or -1 */ +static Py_ssize_t By_SetSlotOf(PySetObject *table, PyObject *member) { + Py_ssize_t slot; + for (slot = 0; slot <= table->mask; slot++) { + if (table->table[slot].key == member) return slot; + } + return -1; +} + +/* settle every member of a set, and hand back the set that holds them settled — the set + * itself where it could be written in place, or NULL where a member is refused + * + * a set is hashed on its members, so a member is replaced without moving any other where + * that can be done: a replacement that hashes as the member it replaces takes the member's + * own entry, so the set iterates in the order it did. where one hashes differently the set + * is built again from its members — aside first, and the original only emptied and filled + * from that once every member has gone in, so a member that cannot be hashed leaves the set + * as it was. a replacement equal to a member already there, or to another replacement, is a + * refusal, since one of the two would be lost. + * + * a `frozenset` is written in place too where every replacement keeps its member's hash, + * which is all its own hash is made of; where one does not, a settled copy stands in its + * place, as a tuple's does. a subclass is written only where every write it has is `set`'s + * own (see `By_WritesAsBase`), and a `frozenset` subclass, which has its own say in what it + * is made from, is refused where a member moves */ +static PyObject *By_SettledSet(PyObject *set, const By_Twins *twins, int depth) { + PySetObject *table = (PySetObject *)set; + PyObject *members = PyList_New(0); + PyObject *hashes = PyList_New(0); + PyObject *stood = NULL; + PyObject *answer = NULL; + Py_ssize_t at, size; + int settled = 1, moved = 0, rehashed = 0; + int frozen = PyFrozenSet_Check(set); + if (members == NULL || hashes == NULL) goto refused; + /* the members first, with the hash each is filed under: settling one can run arbitrary + * code, so nothing may be walking the table itself */ + for (at = 0; at <= table->mask; at++) { + PyObject *key = table->table[at].key; + PyObject *hash; + /* an empty slot, or the marker a removed member leaves, which is filed under the + * one hash no member can have */ + if (key == NULL || table->table[at].hash == -1) continue; + hash = PyLong_FromSsize_t(table->table[at].hash); + if (hash == NULL || PyList_Append(hashes, hash) < 0 || PyList_Append(members, key) < 0) { + Py_XDECREF(hash); + goto refused; + } + Py_DECREF(hash); + } + size = PyList_GET_SIZE(members); + if (size != PySet_GET_SIZE(set)) goto refused; + stood = PyList_New(size); + if (stood == NULL) goto refused; + for (at = 0; at < size; at++) { + PyObject *member = PyList_GET_ITEM(members, at); + PyObject *stands = By_SettledValue(member, twins, depth); + if (stands == NULL) { + settled = 0; + stands = By_NewRef(member); + } + if (stands != member) { + Py_hash_t hash = PyObject_Hash(stands); + if (hash == -1 && PyErr_Occurred()) { + PyErr_Clear(); + settled = 0; + } + moved = 1; + if (hash != PyLong_AsSsize_t(PyList_GET_ITEM(hashes, at))) rehashed = 1; + } + PyList_SET_ITEM(stood, at, stands); + } + if (!settled) goto refused; + if (!moved) { + answer = By_NewRef(set); + goto done; + } + if (frozen ? !PyFrozenSet_CheckExact(set) : !By_WritesAsBase(set, &PySet_Type)) { + goto refused; + } + /* each member being replaced is still where this read it, and no replacement is equal + * to a member already there or to another replacement */ + if (PySet_GET_SIZE(set) != size) goto refused; + { + PyObject *replacements = PySet_New(NULL); + Py_ssize_t count = 0; + if (replacements == NULL) goto refused; + for (at = 0; at < size; at++) { + PyObject *member = PyList_GET_ITEM(members, at); + PyObject *stands = PyList_GET_ITEM(stood, at); + if (stands == member) continue; + count++; + if (By_SetSlotOf(table, member) < 0 || PySet_Contains(set, stands) != 0 + || PySet_Add(replacements, stands) < 0) { + Py_DECREF(replacements); + goto refused; + } + } + if (PySet_GET_SIZE(replacements) != count) { + Py_DECREF(replacements); + goto refused; + } + Py_DECREF(replacements); + } + if (!rehashed) { + for (at = 0; at < size; at++) { + PyObject *member = PyList_GET_ITEM(members, at); + PyObject *stands = PyList_GET_ITEM(stood, at); + if (stands == member) continue; + table->table[By_SetSlotOf(table, member)].key = By_NewRef(stands); + Py_DECREF(member); + } + answer = By_NewRef(set); + goto done; + } + if (frozen) { + answer = PyFrozenSet_New(stood); + if (answer == NULL || PySet_GET_SIZE(answer) != size) { + Py_CLEAR(answer); + goto refused; + } + goto done; + } + { + PyObject *rebuilt = PySet_New(stood); + PyObject *filled; + if (rebuilt == NULL) goto refused; + if (PySet_Clear(set) < 0) { + Py_DECREF(rebuilt); + goto refused; + } + filled = PyNumber_InPlaceOr(set, rebuilt); + Py_DECREF(rebuilt); + if (filled == NULL) goto refused; + Py_DECREF(filled); + answer = By_NewRef(set); + goto done; + } +refused: + PyErr_Clear(); + answer = NULL; +done: + Py_XDECREF(members); + Py_XDECREF(hashes); + Py_XDECREF(stood); + return answer; +} + /* settle every item of a list, in place, and say whether the list now holds no twin */ static int By_SettleListItems(PyObject *list, const By_Twins *twins, int depth) { Py_ssize_t at; @@ -5939,6 +6287,63 @@ static int By_SettleListItems(PyObject *list, const By_Twins *twins, int depth) return settled; } +/* whether the cell at `at` of a function's closure is `__classdict__`: the namespace a class + * body ran in, which python 3.12 and later close an annotation or type-parameter scope over + * so that it can read the names the body bound before it reads the module's */ +static int By_IsClassNamespaceCell(PyObject *freevars, Py_ssize_t at) { + PyObject *name; + if (freevars == NULL || !PyTuple_Check(freevars) || at >= PyTuple_GET_SIZE(freevars)) { + return 0; + } + name = PyTuple_GET_ITEM(freevars, at); + return PyUnicode_Check(name) && PyUnicode_CompareWithASCIIString(name, "__classdict__") == 0; +} + +/* settle, in place, the entries of a class namespace that `fn` can read, and say whether + * they now hold no twin + * + * a body that bound a name to one of this module's classes bound it to the twin, so the + * entry is moved as a dict's value is. only the names `fn` can look up are asked about — + * its code's `co_names`, which is every name it reads out of the namespace and the + * attribute names it reads besides — because the namespace also holds the class's other + * members, and among them the very function being settled: a 3.14 class keeps its + * `__annotate__` there, which closes over the namespace again, and walking that cycle runs + * into the depth bound and refuses. a mapping that is not a dict, which a metaclass's + * `__prepare__` may make, is refused rather than written */ +static int By_SettleNamespaceReads(PyObject *fn, PyObject *namespace_, const By_Twins *twins, + int depth) { + PyObject *names; + Py_ssize_t at; + int settled = 1; + if (!PyDict_Check(namespace_)) return 0; + names = PyObject_GetAttrString(PyFunction_GetCode(fn), "co_names"); + if (names == NULL || !PyTuple_Check(names)) { + PyErr_Clear(); + Py_XDECREF(names); + return 0; + } + for (at = 0; at < PyTuple_GET_SIZE(names); at++) { + PyObject *name = PyTuple_GET_ITEM(names, at); + /* held rather than borrowed: settling can run arbitrary code */ + PyObject *value = By_NewRef(PyDict_GetItem(namespace_, name)); + PyObject *stands; + if (value == NULL) continue; + stands = By_SettledValue(value, twins, depth); + if (stands == NULL) { + settled = 0; + } else { + if (stands != value && PyDict_SetItem(namespace_, name, stands) < 0) { + PyErr_Clear(); + settled = 0; + } + Py_DECREF(stands); + } + Py_DECREF(value); + } + Py_DECREF(names); + return settled; +} + /* settle what a function captured, in place, and say whether it now holds no twin * * a `def` evaluates its defaults and closes over its cells where it stands, and everything @@ -5965,7 +6370,9 @@ static int By_SettleFunction(PyObject *fn, const By_Twins *twins, int depth) { Py_ssize_t at; int settled = 1; - if (closure != NULL && PyTuple_Check(closure)) { + if (closure != NULL && PyTuple_Check(closure) && PyTuple_GET_SIZE(closure) > 0) { + PyObject *freevars = PyObject_GetAttrString(PyFunction_GetCode(fn), "co_freevars"); + if (freevars == NULL) PyErr_Clear(); for (at = 0; at < PyTuple_GET_SIZE(closure); at++) { PyObject *cell = PyTuple_GET_ITEM(closure, at); PyObject *held, *stands; @@ -5978,6 +6385,11 @@ static int By_SettleFunction(PyObject *fn, const By_Twins *twins, int depth) { PyErr_Clear(); continue; } + if (By_IsClassNamespaceCell(freevars, at)) { + if (!By_SettleNamespaceReads(fn, held, twins, depth)) settled = 0; + Py_DECREF(held); + continue; + } stands = By_SettledValue(held, twins, depth); if (stands == NULL) { settled = 0; @@ -5990,6 +6402,7 @@ static int By_SettleFunction(PyObject *fn, const By_Twins *twins, int depth) { } Py_DECREF(held); } + Py_XDECREF(freevars); } /* read after the closure and not before: settling a cell can run arbitrary code, and @@ -6153,6 +6566,10 @@ static void By_UnwindMoves(PyObject *moved, Py_ssize_t mark) { * be found on the twin *first*, and an instance carrying state the emitted type has nowhere * to put is refused outright rather than moved with the remainder dropped. * + * each is written the way `object.__setattr__` writes it, past a `__setattr__` the class + * wrote: the state is being handed over, not assigned by the program, so a class whose + * `__setattr__` refuses — a `frozen data class` — moves like any other. + * * nowhere is narrower than "outside the layout". an emitted class keeps a dict beside its * layout wherever the source did not declare `__slots__` throughout, and a name written on * the twin from outside `__init__` goes into it — the same place `o.brand_new = 7` on a @@ -6174,6 +6591,17 @@ static void By_UnwindMoves(PyObject *moved, Py_ssize_t mark) { * `layouts[i]` is NULL for a class this cannot be done for at all, and the emitting side * decides that: a class whose instance is not wholly its own field run — one standing on a * base python allocates — keeps state in a place nothing here can read */ +/* `object.__setattr__(obj, name, value)`, which reaches a data descriptor's setter or the + * instance dict without asking a `__setattr__` the object's class wrote */ +static int By_SetAttrPastClass(PyObject *obj, const char *name, PyObject *value) { + PyObject *key = PyUnicode_InternFromString(name); + int failed; + if (key == NULL) return -1; + failed = PyObject_GenericSetAttr(obj, key, value); + Py_DECREF(key); + return failed; +} + static PyObject *By_MovedInstance(PyObject *value, const By_Twins *twins, int depth) { PyObject *type = NULL; const By_Field *layout = NULL; @@ -6308,12 +6736,14 @@ static PyObject *By_MovedInstance(PyObject *value, const By_Twins *twins, int de continue; } stands = By_SettledValue(held, twins, depth - 1); - /* the setter is the type's own, so the value goes through the same conversion an + /* the setter is the field's own, so the value goes through the same conversion an * assignment from python does — and a value that could not be settled keeps what - * the twin gave it rather than costing the instance the whole attribute */ - failed = PyObject_SetAttrString(fresh, field->name, - stands != NULL ? stands : held) - < 0; + * the twin gave it rather than costing the instance the whole attribute. it is + * reached the way `object.__setattr__` reaches it, past any `__setattr__` the class + * has: this is the state the twin already holds being handed over, not an + * assignment the program makes, and a `frozen data class` refuses every assignment + * there while its fields take one write each underneath */ + failed = By_SetAttrPastClass(fresh, field->name, stands != NULL ? stands : held) < 0; Py_XDECREF(stands); Py_DECREF(held); if (failed) { @@ -6350,7 +6780,7 @@ static PyObject *By_MovedInstance(PyObject *value, const By_Twins *twins, int de if (held == NULL) continue; Py_INCREF(held); stands = By_SettledValue(held, twins, depth - 1); - failed = PyObject_SetAttr(fresh, name, stands != NULL ? stands : held) < 0; + failed = PyObject_GenericSetAttr(fresh, name, stands != NULL ? stands : held) < 0; Py_XDECREF(stands); Py_DECREF(held); if (failed) { @@ -6474,24 +6904,7 @@ static PyObject *By_SettledValue(PyObject *value, const By_Twins *twins, int dep return By_SettleDictValues(value, twins, depth - 1) ? By_NewRef(value) : NULL; } - /* a set's members are what it is hashed on, so one holding a twin is refused rather - * than settled — the same reason a dict's keys are */ - if (PyAnySet_Check(value)) { - PyObject *members = PySequence_List(value); - Py_ssize_t at; - int settled = 1; - if (members == NULL) { - PyErr_Clear(); - return NULL; - } - for (at = 0; settled && at < PyList_GET_SIZE(members); at++) { - if (!By_SettlesInPlace(PyList_GET_ITEM(members, at), twins, depth - 1)) { - settled = 0; - } - } - Py_DECREF(members); - return settled ? By_NewRef(value) : NULL; - } + if (PyAnySet_Check(value)) return By_SettledSet(value, twins, depth - 1); if (PyFunction_Check(value)) { return By_SettleFunction(value, twins, depth - 1) ? By_NewRef(value) : NULL; @@ -6629,10 +7042,10 @@ static PyObject *By_TwinReplacement(PyObject *value, const By_Twins *twins) { * module wrote whatever the body's expressions evaluated to, and neither can be worked * out again from here. * - * `__annotations__` is the one dunder carried, and it is carried because the objection to - * the rest does not reach it: it fills no type slot, so there is no second answer for it - * to disagree with. the getset *is* the only way to read it, on a compiled class and an - * interpreted one alike. + * `__annotations__` is carried, with 3.14's `__annotate__` beside it (see + * `By_CarryAnnotate`), because the objection to the other dunders does not reach it: it + * fills no type slot, so there is no second answer for it to disagree with. the getset + * *is* the only way to read it, on a compiled class and an interpreted one alike. * * the values are subject to the same rule every carried attribute is — see * `By_TwinReplacement` — and where one of them fails it, the whole mapping is replaced by @@ -6685,6 +7098,46 @@ static inline int By_CarryAnnotations(PyObject *source, PyObject *target, return failed ? -1 : 0; } +/* the function a class's annotations are computed by, carried onto the type that takes the + * class's place + * + * 3.14 keeps it in the class's own dict — under `__annotate_func__`, or `__annotate__` + * where the body assigned one — and `__annotate__` answers with it, read through a getset + * on the metatype that looks at that dict and no base's. `None` there says the class + * wrote no annotations, and is what the getset writes for a class it finds nothing on, so + * a type that carried nothing would say exactly that. this is the second dunder carried, + * for the reason `By_CarryAnnotations` gives for the first. + * + * the function evaluates the annotations afresh at each call, the names the body bound + * read out of the namespace it ran in. it is settled as any carried value is, which + * settles what it can read of that namespace (see `By_SettleNamespaceReads`), and where + * that is refused the refusal is written in */ +static inline int By_CarryAnnotate(PyObject *source, PyObject *target, const By_Twins *twins) { +#if PY_VERSION_HEX >= 0x030E0000 + static const char *const keys[] = {"__annotate__", "__annotate_func__"}; + size_t at; + for (at = 0; at < sizeof(keys) / sizeof(keys[0]); at++) { + /* held rather than borrowed: settling can run arbitrary code */ + PyObject *written = By_NewRef(PyDict_GetItemString(source, keys[at])); + PyObject *carried; + int failed; + if (written == NULL) continue; + carried = By_TwinReplacement(written, twins); + Py_DECREF(written); + if (carried == NULL) carried = By_LostAnnotationsNamed("__annotate__"); + if (carried == NULL) return -1; + failed = PyDict_SetItemString(target, keys[at], carried) < 0; + Py_DECREF(carried); + if (failed) return -1; + } +#else + (void)source; + (void)target; + (void)twins; +#endif + return 0; +} + /* the abstract-base registry a module body built, carried onto the type that takes the * class's place * @@ -6787,9 +7240,9 @@ static int By_AdoptMethodTwins(PyObject *source, PyObject *target) { * * a dunder is never carried. a name written into `tp_dict` does not fill a type slot, * so an adopted `__ge__` would answer `a.__ge__(b)` while `a >= b` still went to the - * slot — two answers where the interpreted class has one. `__annotations__` is the one - * exception, and `By_CarryAnnotations` says why: it fills no slot, so there is nothing - * for it to disagree with */ + * slot — two answers where the interpreted class has one. `__annotations__` and 3.14's + * `__annotate__` are the exceptions, and `By_CarryAnnotations` says why: they fill no + * slot, so there is nothing for them to disagree with */ static inline int By_AdoptTwinAttributes(const By_Twins *twins) { Py_ssize_t index; for (index = 0; index < twins->count; index++) { @@ -6803,6 +7256,7 @@ static inline int By_AdoptTwinAttributes(const By_Twins *twins) { target = ((PyTypeObject *)type)->tp_dict; if (source == NULL || target == NULL) continue; if (By_CarryAnnotations(source, target, twins) < 0) return -1; + if (By_CarryAnnotate(source, target, twins) < 0) return -1; if (By_AdoptMethodTwins(source, target) < 0) return -1; /* ahead of the walk below, which leaves a name the target already holds alone — * and the target holds an `_abc_impl` of its own, the empty one it was built with */ @@ -7007,7 +7461,7 @@ static inline PyObject *By_DataclassRepr(PyObject *self, const By_DataField *fie return result; } -/* the `__eq__` a `@dataclass` generates, which is +/* the `__eq__` a `@dataclass` generates, which from 3.13 is * * if self is other: return True * if other.__class__ is self.__class__: @@ -7027,7 +7481,22 @@ static inline PyObject *By_DataclassRepr(PyObject *self, const By_DataField *fie * something other than True or False is what the whole comparison answers with — and the * chain stops at the first falsy one. `__ne__` is not generated: python's own derives it * from this by negating, so the four ordering opcodes are refused here and `Py_NE` is the - * negation `object.__ne__` would have produced */ + * negation `object.__ne__` would have produced + * + * before 3.13 the generated method compares two tuples instead: + * + * if other.__class__ is self.__class__: + * return (self.a,self.b,)==(other.a,other.b,) + * return NotImplemented + * + * which answers differently in all three of the places above: there is no identity test, + * so `p == p` reads every field of both; every field of `self` is read before any of + * `other`'s; and a tuple compares its items with `PyObject_RichCompareBool`, so a field + * shared by identity is equal whatever its `__eq__` says, and the answer is always a bool. + * a `float` field is kept as a number rather than an object, so two instances sharing one + * NaN object compare unequal here where python finds the one object in both — see + * runtime.md */ +#if PY_VERSION_HEX >= 0x030D0000 static inline PyObject *By_DataclassEq(PyObject *self, PyObject *other, int op, const By_DataField *fields) { const By_DataField *field; @@ -7067,6 +7536,51 @@ static inline PyObject *By_DataclassEq(PyObject *self, PyObject *other, int op, Py_DECREF(result); return PyBool_FromLong(!truthy); } +#else +/* each field of `obj` read into a tuple, in order, as `(obj.a,obj.b,)` builds it */ +static inline PyObject *By_DataclassTerms(PyObject *obj, const By_DataField *fields) { + Py_ssize_t count = 0; + Py_ssize_t at; + PyObject *values; + while (fields[count].name != NULL) count++; + values = PyTuple_New(count); + if (values == NULL) return NULL; + for (at = 0; at < count; at++) { + PyObject *value = fields[at].get(obj, NULL); + if (value == NULL) { + Py_DECREF(values); + return NULL; + } + PyTuple_SET_ITEM(values, at, value); + } + return values; +} + +static inline PyObject *By_DataclassEq(PyObject *self, PyObject *other, int op, + const By_DataField *fields) { + PyObject *mine; + PyObject *theirs; + int equal; + if (op != Py_EQ && op != Py_NE) Py_RETURN_NOTIMPLEMENTED; + if (Py_TYPE(self) != Py_TYPE(other)) Py_RETURN_NOTIMPLEMENTED; + mine = By_DataclassTerms(self, fields); + if (mine == NULL) return NULL; + /* an instance's fields read twice are the same objects twice in python, which keeps + * each in a slot. a field kept unboxed is boxed afresh on each read, so an instance + * compared with itself compares its one tuple with itself instead — what python + * reads twice it has already read once, and a read that could fail has failed */ + theirs = self == other ? By_NewRef(mine) : By_DataclassTerms(other, fields); + if (theirs == NULL) { + Py_DECREF(mine); + return NULL; + } + equal = PyObject_RichCompareBool(mine, theirs, Py_EQ); + Py_DECREF(mine); + Py_DECREF(theirs); + if (equal < 0) return NULL; + return PyBool_FromLong(op == Py_EQ ? equal : !equal); +} +#endif /* `hash((self.a, self.b))`, which is what a frozen `@dataclass` hashes * @@ -7115,7 +7629,9 @@ static inline PyObject *By_FrozenInstanceError(void) { * * python's pair refuses every name on an instance of the class that declared them, and * only a *subclass* instance writing a name that is not a field gets through to the - * ordinary attribute machinery — which is why `owner` is asked about rather than assumed */ + * ordinary attribute machinery — which is why `owner` is asked about rather than assumed. + * that is 3.13's pair; below it the twin's own pair stands in this slot's place, see + * `By_CarryDataclassMembers` */ static inline int By_FrozenSetAttr(PyObject *self, PyObject *name, PyObject *value, PyTypeObject *owner, const By_DataField *fields) { const By_DataField *field; @@ -7134,6 +7650,34 @@ static inline int By_FrozenSetAttr(PyObject *self, PyObject *name, PyObject *val return PyObject_GenericSetAttr(self, name, value); } +/* the docstring a class statement wrote, carried off the twin's class + * + * python keeps a class's `__doc__` in its own dict: the string its body opens with, or + * `None` where the body opens with none, or whatever a decorator wrote over that — + * `@dataclass` writes the class's signature where the body wrote no docstring. an emitted + * type is built from tables holding none of it, so it answers the twin's. the twin was + * compiled at the level this interpreter runs at, so under `-OO` that is the `None` python + * answers. + * + * a type that already holds something of its own under the name — the descriptor of a + * field spelled `__doc__` — keeps it, since replacing that would leave the field with no + * way to be read */ +static inline int By_CarryDoc(PyObject *twin, PyObject *type) { + PyObject *doc, *own; + if (twin == NULL || type == NULL) return 0; + if (!PyType_Check(twin) || !PyType_Check(type)) return 0; + if (((PyTypeObject *)twin)->tp_dict == NULL || ((PyTypeObject *)type)->tp_dict == NULL) + return 0; + doc = PyDict_GetItemString(((PyTypeObject *)twin)->tp_dict, "__doc__"); + if (doc == NULL) return PyErr_Occurred() ? -1 : 0; + own = PyDict_GetItemString(((PyTypeObject *)type)->tp_dict, "__doc__"); + if (own == NULL && PyErr_Occurred()) return -1; + if (own != NULL && own != Py_None) return 0; + if (PyDict_SetItemString(((PyTypeObject *)type)->tp_dict, "__doc__", doc) < 0) return -1; + PyType_Modified((PyTypeObject *)type); + return 0; +} + /* the dataclass bookkeeping that has no slot to fill, carried off the twin's class * * `__dataclass_fields__` is a dict of `dataclasses.Field` objects and `__dataclass_params__` @@ -7154,9 +7698,6 @@ static inline int By_CarryDataclassMembers(PyObject *twin, PyObject *type) { /* `copy.replace` reaches for this, and the function it names calls * `dataclasses.replace`, which is carried above */ "__replace__", - /* `@dataclass` writes the class's signature here where the body wrote no - * docstring, and leaves the docstring alone where it did */ - "__doc__", /* the field names `slots=True` declares, which `copyreg` reads to learn what an * instance with no dict holds — and so what `copy` and `pickle` hand over */ "__slots__", @@ -7181,6 +7722,41 @@ static inline int By_CarryDataclassMembers(PyObject *twin, PyObject *type) { if (PyDict_SetItemString(target, *name, value) < 0) return -1; } PyType_Modified((PyTypeObject *)type); +#if PY_VERSION_HEX < 0x030D0000 + /* below 3.13 a frozen data class answers a write with the twin's own `__setattr__` + * and `__delattr__`, in place of the slot. + * + * `slots=True` made the twin's class again and left the pair holding the class it + * replaced, so `type(self) is cls` never holds for any instance and a name that is not + * a field reaches `super(cls, self)`, which refuses it — and the same is true of an + * emitted instance, which is no instance of that class either. and a C-level setter + * would refuse something python allows: `object.__setattr__`, which the decorator's + * `__setstate__` restores a copy through, refuses an object whose type sets attributes + * in C of its own, where 3.13 asks that only of a type. assigning the pair is what a + * class statement does, and installs python's own dispatch over it */ + { + static const char *const pair[] = {"__setattr__", "__delattr__", NULL}; + PyTypeObject *emitted = (PyTypeObject *)type; + PyObject *params = PyDict_GetItemString(source, "__dataclass_params__"); + PyObject *frozen = params == NULL ? NULL : PyObject_GetAttrString(params, "frozen"); + int is_frozen; + if (params != NULL && frozen == NULL) return -1; + is_frozen = frozen == NULL ? 0 : PyObject_IsTrue(frozen); + Py_XDECREF(frozen); + if (is_frozen < 0) return -1; + if (is_frozen) { + unsigned long immutable = emitted->tp_flags & Py_TPFLAGS_IMMUTABLETYPE; + int failed = 0; + emitted->tp_flags &= ~Py_TPFLAGS_IMMUTABLETYPE; + for (name = pair; *name != NULL && !failed; name++) { + PyObject *value = PyDict_GetItemString(source, *name); + if (value != NULL) failed = PyObject_SetAttrString(type, *name, value) < 0; + } + emitted->tp_flags |= immutable; + if (failed) return -1; + } + } +#endif return 0; } @@ -7709,11 +8285,15 @@ static int By_VerifyClass(const char *module, PyObject *dict, const char *name, for (at = 0; at < base_count; at++) { if (bases[at] == NULL || !PyType_Check(bases[at])) continue; if (!By_HasBase(stands, bases[at])) { + /* named in full, as `repr` names a class: its `tp_name` is the bare name */ + PyObject *base_name = PyType_GetQualName((PyTypeObject *)bases[at]); + if (base_name == NULL) return -1; PyErr_Format(PyExc_ImportError, - "%s.%s stands on an orphaned base: %s is what this module's " + "%s.%s stands on an orphaned base: %s.%U is what this module's " "own base of that name means and is not among its __bases__, " "so isinstance answers False where python answers True", - module, name, ((PyTypeObject *)bases[at])->tp_name); + module, name, module, base_name); + Py_DECREF(base_name); return -1; } } @@ -8031,15 +8611,88 @@ static inline PyObject *By_FallbackCode(const By_Fallback *fallback) { * `PyEval_EvalCode` and `PyRun_String` reach the same evaluator by the same route, and * both take the builtins the frame runs against from `dict["__builtins__"]` — which is * what lets the capture below swap that entry and have it seen */ -static inline PyObject *By_ExecModuleBody(const By_Fallback *fallback, PyObject *dict) { +/* run a module's interpreted twin in `dict`, handing back in `*ran` the code object that + * ran it, as a new reference + * + * the code object is what says which function each `def` statement made — see + * `By_DefinitionCodes` — so the source is compiled to one here rather than run straight + * from the text, under the name `PyRun_String` would have given it */ +static inline PyObject *By_ExecModuleBody(const By_Fallback *fallback, PyObject *dict, + PyObject **ran) { PyObject *code = By_FallbackCode(fallback), *result; - if (code != NULL) { - result = PyEval_EvalCode(code, dict, dict); + *ran = NULL; + if (code == NULL) { + if (PyErr_Occurred()) return NULL; + code = Py_CompileString(fallback->source, "", Py_file_input); + if (code == NULL) return NULL; + } + result = PyEval_EvalCode(code, dict, dict); + if (result == NULL) { Py_DECREF(code); - return result; + return NULL; } - if (PyErr_Occurred()) return NULL; - return PyRun_String(fallback->source, Py_file_input, dict, dict); + *ran = code; + return result; +} + +/* the code of each function a module-level `def` statement makes, by name + * + * a `def` makes its function out of a code object compiled into the module's own — a + * constant of it, or of the scope a generic definition's type parameters are evaluated + * in — whose qualified name is the definition's. so those constants say which object the + * statement made, and nothing else can make one out of the same code without being handed + * it. every code object the module's reaches is asked, which a nested definition's + * qualified name keeps apart (`outer..f`, `C.f`). a name two of them share is a + * module that defines it twice, and maps to `None`: neither can be told apart as the + * definition this module compiled */ +static int By_CollectDefinitionCodes(PyObject *codes, PyObject *code, int depth) { + PyObject *consts = ((PyCodeObject *)code)->co_consts; + Py_ssize_t at; + if (depth <= 0 || consts == NULL || !PyTuple_Check(consts)) return 0; + for (at = 0; at < PyTuple_GET_SIZE(consts); at++) { + PyObject *inner = PyTuple_GET_ITEM(consts, at); + PyObject *name; + int seen; + if (!PyCode_Check(inner)) continue; + name = ((PyCodeObject *)inner)->co_qualname; + seen = PyDict_Contains(codes, name); + if (seen < 0 || PyDict_SetItem(codes, name, seen ? Py_None : inner) < 0) return -1; + if (By_CollectDefinitionCodes(codes, inner, depth - 1) < 0) return -1; + } + return 0; +} + +/* `By_CollectDefinitionCodes` over the code a module body ran, as a new reference, or NULL + * with an exception set. the walk is bounded because a code object's constants are as deep + * as the source nests, and python's own compiler stops well short of this */ +static inline PyObject *By_DefinitionCodes(PyObject *ran) { + PyObject *codes = PyDict_New(); + if (codes == NULL || ran == NULL || !PyCode_Check(ran)) return codes; + if (By_CollectDefinitionCodes(codes, ran, 256) < 0) { + Py_DECREF(codes); + return NULL; + } + return codes; +} + +/* the function the module-level `def name` statement made, where the name still holds it, + * borrowed; NULL where it does not + * + * the module body runs before anything is installed, and a body can write over a name in + * ways no reading of the source sees: `globals()["f"] = g`, a helper handed `globals()`, an + * `exec`, a star import of an accelerator module. publishing the native under the name + * would throw that write away and answer with the definition python no longer has. so a + * definition is published only where the name still holds the function whose code is the + * statement's own (see `By_DefinitionCodes`), and is otherwise left as the module body + * left it */ +static inline PyObject *By_DefinitionStanding(PyObject *dict, const char *name, + PyObject *codes) { + PyObject *held = PyDict_GetItemString(dict, name); + PyObject *code; + if (held == NULL || codes == NULL || !PyFunction_Check(held)) return NULL; + code = PyDict_GetItemString(codes, name); + if (code == NULL || code == Py_None || PyFunction_GET_CODE(held) != code) return NULL; + return held; } /* run a module's interpreted twin, capturing each class body before its decorators run @@ -8082,9 +8735,11 @@ static inline PyObject *By_ExecModuleBody(const By_Fallback *fallback, PyObject * `{name: class}` alongside it, the class each of those statements built before any * decorator was handed it, and `*bound` is given `{name: (class, dict)}` for each statement * the twin wrote `@__build_class__(...)` over — see `By_RecordBound` — each as a new - * reference or NULL */ + * reference or NULL. `*ran` is given the code object the body ran, as `By_ExecModuleBody` + * hands it back */ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject *dict, - PyObject **statements, PyObject **bound) { + PyObject **statements, PyObject **bound, + PyObject **ran) { static PyMethodDef capture = {"__build_class__", (PyCFunction)(void (*)(void))By_CaptureClassBody, METH_VARARGS | METH_KEYWORDS, NULL}; @@ -8133,7 +8788,8 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * failed = failed || wrapper == NULL || PyDict_SetItemString(dict, "__builtins__", stood) < 0 || PyDict_SetItemString(mapping, "__build_class__", wrapper) < 0; - result = failed ? NULL : By_ExecModuleBody(fallback, dict); + *ran = NULL; + result = failed ? NULL : By_ExecModuleBody(fallback, dict, ran); Py_XDECREF(result); /* whatever the body did, the capture stops here */ if (state != NULL) { @@ -8155,6 +8811,7 @@ static inline PyObject *By_RunModuleBody(const By_Fallback *fallback, PyObject * Py_DECREF(bodies); Py_CLEAR(*statements); Py_CLEAR(*bound); + Py_CLEAR(*ran); return NULL; } return bodies; @@ -8166,6 +8823,29 @@ static inline PyObject *By_ClassBody(PyObject *bodies, const char *name) { return PyDict_GetItemString(bodies, name); } +/* whether a class's name no longer holds what its `class` statement bound, so that + * installing the type over it would throw away a write the module body made + * + * what the statement bound is the class it built (see `By_CaptureClassBody`) — init + * applies a compiled class's decorators itself, and the twin's statement carries none — + * or, where `rebuilt` says the twin's own decorator made the class again, what the + * statement's record says it bound (see `By_RecordBound`). a class with no record is a + * question that cannot be asked, and is answered the way that cannot be wrong. compiled + * code reaches a class without going through its name, so this is the whole module's + * refusal rather than the class's */ +static inline int By_ClassNameMoved(PyObject *statements, PyObject *bound, PyObject *dict, + const char *name, int rebuilt) { + PyObject *made; + if (rebuilt) { + PyObject *record = By_ClassBody(bound, name); + if (record == NULL || !PyTuple_Check(record) || PyTuple_GET_SIZE(record) != 2) return 1; + made = PyTuple_GET_ITEM(record, 0); + } else { + made = By_ClassBody(statements, name); + } + return made == NULL || PyDict_GetItemString(dict, name) != made; +} + /* whether `a` and `b` hold different objects under `key`, absence included */ static inline int By_HoldsApart(PyObject *a, PyObject *b, PyObject *key) { return PyDict_GetItem(a, key) != PyDict_GetItem(b, key); @@ -8218,8 +8898,9 @@ static inline int By_DunderApart(PyObject *now, PyObject *body, PyObject *key) { * those accesses looks. NULL for a class whose every access is checked, which copes with * a rewrite as it copes with one made after import * - * a class whose name no longer holds what its statement built is left alone, as - * `By_CarryRewrittenMembers` leaves it. `rebuilt` says the twin is a `data class`, which + * a class whose name no longer holds what its statement built is left alone here, as + * `By_CarryRewrittenMembers` leaves it: `By_ClassNameMoved` stands the module down for it + * on its own account. `rebuilt` says the twin is a `data class`, which * `@dataclass(slots=True)` made again out of what the statement built and wrote every * dunder it generates onto — so what that class is read against is not the body but what * the statement *bound*, after its decorators, which `bound` recorded (see @@ -9290,11 +9971,15 @@ static PyObject *By_InstanceDict_ior(PyObject *selfobj, PyObject *other) { return By_NewRef(selfobj); } -/* every name the object has, removed one at a time. a field the layout treats as always - * defined refuses, which is what an emitted instance being its layout means — and it - * refuses loudly rather than leaving half a mapping behind */ +/* every name the object has, removed, as python's `dict.clear` empties an instance's dict + * without asking any descriptor. a field is removed through its own setter, which takes a + * delete for every field the mapping names + * + * the entries are held until every one is gone, because python's own clear lets go of the + * values only once the dict is empty: a `__del__` that letting go of one runs finds the + * object already cleared, rather than the fields after its own still standing */ static PyObject *By_InstanceDict_clearmethod(PyObject *selfobj, PyObject *unused) { - PyObject *keys; + PyObject *items; Py_ssize_t index; Py_ssize_t count; (void)unused; @@ -9302,16 +9987,17 @@ static PyObject *By_InstanceDict_clearmethod(PyObject *selfobj, PyObject *unused PyDict_Clear(selfobj); Py_RETURN_NONE; } - keys = PyDict_Keys(selfobj); - if (keys == NULL) return NULL; - count = PyList_GET_SIZE(keys); + items = PyDict_Items(selfobj); + if (items == NULL) return NULL; + count = PyList_GET_SIZE(items); for (index = 0; index < count; index++) { - if (By_InstanceDict_assign(selfobj, PyList_GET_ITEM(keys, index), NULL) < 0) { - Py_DECREF(keys); + PyObject *key = PyTuple_GET_ITEM(PyList_GET_ITEM(items, index), 0); + if (By_InstanceDict_assign(selfobj, key, NULL) < 0) { + Py_DECREF(items); return NULL; } } - Py_DECREF(keys); + Py_DECREF(items); Py_RETURN_NONE; } @@ -9678,8 +10364,7 @@ static inline PyObject *By_InstanceState(PyObject *owner, const By_DictField *fi } /* `obj.__dict__ = mapping`, which python takes as replacing the whole of an object's - * state — so every name the object has and the mapping does not is removed first. a - * field the layout treats as always defined cannot be removed and says so */ + * state — so every name the object has and the mapping does not is removed first */ static inline int By_InstanceDictReplace(PyObject *owner, const By_DictField *fields, PyObject *value) { PyObject *standing; @@ -9742,6 +10427,46 @@ static inline Py_hash_t By_HashResult(PyObject *value) { return hash; } +#if PY_VERSION_HEX < 0x030C0000 +/* a call to a native entry that enters no recursion of its own + * + * the forwarder in front of it is a python frame, and that frame is the one python's own + * function would have pushed, so nothing behind it counts another. 3.11 has one count for + * C calls and python frames, and a builtin's own call enters it, so a call from a + * forwarder took a second frame and a recursion reached through one stopped a level early */ +static PyObject *By_NativeEntryVectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) { + PyCFunctionObject *function = (PyCFunctionObject *)callable; + return ((PyCFunctionFastWithKeywords)(void (*)(void))function->m_ml->ml_meth)( + function->m_self, args, PyVectorcall_NARGS(nargsf), kwnames); +} +#endif + +/* a builtin over `def`, one of the class's own entries, that counts no frame on its way in + * + * what python counts for the call is counted behind it, or is nothing: a written `__new__` + * counts the frame its body is, an `__init_subclass__` goes on up to the one python calls, + * and python's own allocation of a subclass calls no `__new__` at all. from 3.12 a + * builtin's entry is against the C stack and counts none of the limit anyway, so this is + * how every later version already counts — see `By_NativeEntryVectorcall` */ +static inline PyObject *By_NativeEntryFunctionOf(PyMethodDef *def, PyObject *self) { + PyObject *function = PyCFunction_NewEx(def, self, NULL); +#if PY_VERSION_HEX < 0x030C0000 + if (function != NULL + && (def->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS + | METH_METHOD)) + == (METH_FASTCALL | METH_KEYWORDS)) { + ((PyCFunctionObject *)function)->vectorcall = By_NativeEntryVectorcall; + } +#endif + return function; +} + +/* `By_NativeEntryFunctionOf` with no receiver of its own */ +static inline PyObject *By_NativeEntryFunction(PyMethodDef *def) { + return By_NativeEntryFunctionOf(def, NULL); +} + /* publish a written `__new__` by *assigning* it onto the finished type * * not through the spec. a `tp_new` filled from a slot table is a C function, and python @@ -9759,7 +10484,7 @@ static inline Py_hash_t By_HashResult(PyObject *value) { * the class arrives as the first argument rather than as a receiver, and the dispatcher * puts it there */ static inline int By_PublishNew(PyObject *type, PyMethodDef *def) { - PyObject *function = PyCFunction_NewEx(def, NULL, NULL); + PyObject *function = By_NativeEntryFunction(def); if (function == NULL) return -1; PyObject *published = PyStaticMethod_New(function); Py_DECREF(function); @@ -9769,6 +10494,31 @@ static inline int By_PublishNew(PyObject *type, PyMethodDef *def) { return stored; } +/* the `tp_new` a class statement gives a class whose `__new__` is written in python: the + * slot that looks the name up and calls it + * + * cpython does not export it. every class whose `__new__` is anything but a builtin bound + * to a type is given it, so one such class is made to read it off — once, as the entry + * that needs it is published, so that no construction near the recursion limit is the + * one to make it */ +static newfunc by_dispatching_new = NULL; +static PyObject *by_dunder_new = NULL; + +static int By_SettleDispatchingNew(void) { + PyObject *probe; + if (by_dunder_new == NULL) { + by_dunder_new = PyUnicode_InternFromString("__new__"); + if (by_dunder_new == NULL) return -1; + } + if (by_dispatching_new != NULL) return 0; + probe = PyObject_CallFunction((PyObject *)&PyType_Type, "s(){sO}", "by_dispatching_new", + "__new__", Py_None); + if (probe == NULL) return -1; + by_dispatching_new = ((PyTypeObject *)probe)->tp_new; + Py_DECREF(probe); + return 0; +} + /* publish the `__new__` that hands an interpreted subclass the allocator of the layout it * extends, while the class itself keeps `object`'s `tp_new` * @@ -9781,40 +10531,45 @@ static inline int By_PublishNew(PyObject *type, PyMethodDef *def) { * * a subclass is what the entry is for. the `class` statement that makes one finds a * `__new__` among its bases and installs the dispatcher that looks the name up, so every - * construction of the subclass comes through here and is handed the allocator first */ + * construction of the subclass comes through here and is handed the allocator first. + * + * python's class has no `__new__` of its own, so `C.__new__` is `object.__new__`, and the + * entry stands in for that: a builtin bound to `object`, as python's is, carrying its + * docstring and signature, and answering every call as `By_ObjectNew` does */ static inline int By_PublishAllocatingNew(PyObject *type, PyMethodDef *def) { PyTypeObject *owner = (PyTypeObject *)type; - PyObject *function = PyCFunction_NewEx(def, NULL, NULL); + if (By_SettleDispatchingNew() < 0) return -1; + PyObject *object_new = _PyType_Lookup(&PyBaseObject_Type, by_dunder_new); + if (object_new != NULL && PyCFunction_Check(object_new)) { + def->ml_doc = ((PyCFunctionObject *)object_new)->m_ml->ml_doc; + } + PyObject *function = By_NativeEntryFunctionOf(def, (PyObject *)&PyBaseObject_Type); if (function == NULL) return -1; - PyObject *published = PyStaticMethod_New(function); + int stored = PyDict_SetItemString(owner->tp_dict, "__new__", function); Py_DECREF(function); - if (published == NULL) return -1; - int stored = PyDict_SetItemString(owner->tp_dict, "__new__", published); - Py_DECREF(published); if (stored == 0) PyType_Modified(owner); return stored; } -/* the class a published `__new__` is asked to allocate, refused where python's own - * `__new__` refuses it: missing, not a class, or not built on `owner` */ -static inline PyTypeObject *By_NewTarget(PyObject *owner, PyObject *const *args, Py_ssize_t nargs) { - const char *name = ((PyTypeObject *)owner)->tp_name; - if (nargs < 1) { - PyErr_Format(PyExc_TypeError, "%s.__new__(): not enough arguments", name); - return NULL; - } - if (!PyType_Check(args[0])) { - PyErr_Format(PyExc_TypeError, "%s.__new__(X): X is not a type object (%s)", name, - Py_TYPE(args[0])->tp_name); - return NULL; - } - PyTypeObject *subtype = (PyTypeObject *)args[0]; - if (!PyType_IsSubtype(subtype, (PyTypeObject *)owner)) { - PyErr_Format(PyExc_TypeError, "%s.__new__(%s): %s is not a subtype of %s", name, - subtype->tp_name, subtype->tp_name, name); - return NULL; +/* the `tp_new` python gives `type` + * + * the entry `By_PublishAllocatingNew` puts in a class's dict is one python's class does not + * have, and a `class` statement reads it as a written `__new__`: a class built on it is + * given the dispatcher where python leaves it the `tp_new` it inherited. python keeps the + * inherited one where the `__new__` a class finds is a builtin bound to a type, and the + * entry is a builtin bound to `object`, so such a class answers as its base does */ +static newfunc By_PythonTpNew(PyTypeObject *type) { + while (type->tp_base != NULL && (type->tp_flags & Py_TPFLAGS_HEAPTYPE) + && type->tp_new == by_dispatching_new) { + PyObject *found = _PyType_Lookup(type, by_dunder_new); + if (found == NULL || !PyCFunction_CheckExact(found) + || PyCFunction_GET_SELF(found) != (PyObject *)&PyBaseObject_Type + || strcmp(((PyCFunctionObject *)found)->m_ml->ml_name, "__new__") != 0) { + break; + } + type = type->tp_base; } - return subtype; + return type->tp_new; } /* `obj.__weakref__`: the head of the weak references made of the object, or `None` @@ -9848,6 +10603,65 @@ static inline void By_AdoptAllocator(PyTypeObject *type, PyObject *owner, allocf } } +/* the body of the entry `By_PublishAllocatingNew` publishes: `object.__new__`, asked of a + * class built on `owner` after handing it `alloc` + * + * every refusal is python's own, in its words, and is decided as python would decide it for + * the class it would have built. the one place that differs is the dispatcher: a class + * built on `owner` reaches this through the dispatcher the entry made it, where python's + * would have kept `object`'s `tp_new` — and python's `object.__new__` refuses arguments + * beyond the class for a class whose `tp_new` is the dispatcher. the allocation is + * `object`'s own, abstract methods refused included */ +static PyObject *By_ObjectNew(PyObject *owner, allocfunc alloc, PyObject *const *args, + Py_ssize_t nargsf, PyObject *kwnames) { + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyTypeObject *subtype; + PyTypeObject *staticbase; + PyObject *empty; + PyObject *made; + newfunc object_new = PyBaseObject_Type.tp_new; + if (nargs < 1) { + PyErr_SetString(PyExc_TypeError, "object.__new__(): not enough arguments"); + return NULL; + } + if (!PyType_Check(args[0])) { + PyErr_Format(PyExc_TypeError, "object.__new__(X): X is not a type object (%s)", + Py_TYPE(args[0])->tp_name); + return NULL; + } + subtype = (PyTypeObject *)args[0]; + if (By_SettleDispatchingNew() < 0) return NULL; + /* the most derived base whose `__new__` is not written in python has to be one that + * allocates as `object` does, or `object.__new__(dict)` would hand back a dict nothing + * initialised */ + staticbase = subtype; + while (staticbase != NULL && By_PythonTpNew(staticbase) == by_dispatching_new) { + staticbase = staticbase->tp_base; + } + if (staticbase != NULL && By_PythonTpNew(staticbase) != object_new) { + PyErr_Format(PyExc_TypeError, "object.__new__(%s) is not safe, use %s.__new__()", + subtype->tp_name, staticbase->tp_name); + return NULL; + } + if (nargs > 1 || (kwnames != NULL && PyTuple_GET_SIZE(kwnames) > 0)) { + if (By_PythonTpNew(subtype) != object_new) { + PyErr_SetString(PyExc_TypeError, + "object.__new__() takes exactly one argument (the type to instantiate)"); + return NULL; + } + if (subtype->tp_init == PyBaseObject_Type.tp_init) { + PyErr_Format(PyExc_TypeError, "%.200s() takes no arguments", subtype->tp_name); + return NULL; + } + } + By_AdoptAllocator(subtype, owner, alloc); + empty = PyTuple_New(0); + if (empty == NULL) return NULL; + made = object_new(subtype, empty, NULL); + Py_DECREF(empty); + return made; +} + /* publish the `__init_subclass__` that hands every interpreted subclass the allocator of * the layout it extends, the moment the subclass is made * @@ -9862,7 +10676,7 @@ static inline void By_AdoptAllocator(PyTypeObject *type, PyObject *owner, allocf * fills no slot, so nothing else about the type changes by its being there */ static inline int By_PublishInitSubclass(PyObject *type, PyMethodDef *def) { PyTypeObject *owner = (PyTypeObject *)type; - PyObject *function = PyCFunction_NewEx(def, NULL, NULL); + PyObject *function = By_NativeEntryFunction(def); if (function == NULL) return -1; PyObject *published = PyClassMethod_New(function); Py_DECREF(function); @@ -10927,28 +11741,57 @@ BY_COLD PyObject *By_FunctionStr(PyObject *callable) { return result; } +/* the name python's own messages give `type` where they format its `tp_name` + * + * for the type a class statement makes that is its bare name. a type this compiler built + * from a spec carries its module in `tp_name` as well, and the bare name in `__name__`, + * which is what the interpreted definition's type has in both — so a heap type is named by + * its `__name__`, and anything else by its `tp_name` as python names it */ +static inline const char *By_TypeNameAsPython(PyTypeObject *type) { + if (type->tp_flags & Py_TPFLAGS_HEAPTYPE) { + const char *name = PyUnicode_AsUTF8(((PyHeapTypeObject *)type)->ht_name); + if (name != NULL) return name; + PyErr_Clear(); + } + return type->tp_name; +} + /* the error a failed merge into a call's keywords reports, in python's words * - * python rewords the two errors a merge raises with a single argument, whatever raised - * them: an `AttributeError` says the operand was no mapping, and a `KeyError` names the - * keyword given twice. anything else is left as it was raised */ + * python rewords two of the errors a merge raises, whatever raised them: any + * `AttributeError` says the operand was no mapping, and a `KeyError` with a single argument + * names the keyword given twice. anything else is left as it was raised. + * + * 3.11 asks the `KeyError` a narrower question: whether it still stands as the bare + * argument tuple it was raised with, which only one raised from C is — the merge's own, for + * a keyword given twice, while nothing is being handled. one a python `__getitem__` raised, + * or one raised inside an `except` block, is already an exception object there and is left + * as it is. `By_RaiseKeyError` raises exactly as the merge does, so the same question asked + * of it gets python's answer */ BY_COLD void By_KeywordsMergeError(PyObject *callee, PyObject *source) { int attribute = PyErr_ExceptionMatches(PyExc_AttributeError); PyObject *type, *value, *traceback, *args, *name; + PyObject *key = NULL; if (!attribute && !PyErr_ExceptionMatches(PyExc_KeyError)) return; PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - args = value == NULL ? NULL : ((PyBaseExceptionObject *)value)->args; - if (args == NULL || !PyTuple_Check(args) || PyTuple_GET_SIZE(args) != 1) { - PyErr_Restore(type, value, traceback); - return; + if (!attribute) { +#if PY_VERSION_HEX >= 0x030C0000 + PyErr_NormalizeException(&type, &value, &traceback); + args = value == NULL ? NULL : ((PyBaseExceptionObject *)value)->args; +#else + args = value; +#endif + if (args == NULL || !PyTuple_Check(args) || PyTuple_GET_SIZE(args) != 1) { + PyErr_Restore(type, value, traceback); + return; + } + key = PyTuple_GET_ITEM(args, 0); } name = By_FunctionStr(callee); if (name != NULL) { - PyObject *key = PyTuple_GET_ITEM(args, 0); if (attribute) { PyErr_Format(PyExc_TypeError, "%U argument after ** must be a mapping, not %.200s", - name, Py_TYPE(source)->tp_name); + name, By_TypeNameAsPython(Py_TYPE(source))); } else { PyErr_Format(PyExc_TypeError, "%U got multiple values for keyword argument '%S'", name, key); @@ -10960,15 +11803,18 @@ BY_COLD void By_KeywordsMergeError(PyObject *callee, PyObject *source) { Py_XDECREF(traceback); } -/* one key of a `**` merged into a call's keywords, refusing one already there */ -static int By_MergeKeyword(PyObject *keywords, PyObject *key, PyObject *value) { +/* whether `key` may still be merged into a call's keywords, raising for one already there + * + * asked before the value is read, as python asks it: a mapping is never asked for the value + * of a keyword given twice */ +static int By_KeywordIsFree(PyObject *keywords, PyObject *key) { int present = PyDict_Contains(keywords, key); - if (present < 0) return -1; + if (present < 0) return 0; if (present) { By_RaiseKeyError(key); - return -1; + return 0; } - return PyDict_SetItem(keywords, key, value); + return 1; } /* `**source` in a call to `callee`, merged into the keywords built so far @@ -10985,7 +11831,7 @@ static inline char By_MergeKeywords(PyObject *keywords, PyObject *source, PyObje while (!failed && PyDict_Next(source, &at, &key, &value)) { Py_INCREF(key); Py_INCREF(value); - failed = By_MergeKeyword(keywords, key, value) < 0; + failed = !By_KeywordIsFree(keywords, key) || PyDict_SetItem(keywords, key, value) < 0; Py_DECREF(value); Py_DECREF(key); } @@ -10996,8 +11842,9 @@ static inline char By_MergeKeywords(PyObject *keywords, PyObject *source, PyObje Py_XDECREF(keys); failed = iterator == NULL; while (!failed && (key = PyIter_Next(iterator)) != NULL) { - PyObject *value = PyObject_GetItem(source, key); - failed = value == NULL || By_MergeKeyword(keywords, key, value) < 0; + PyObject *value = By_KeywordIsFree(keywords, key) ? PyObject_GetItem(source, key) + : NULL; + failed = value == NULL || PyDict_SetItem(keywords, key, value) < 0; Py_XDECREF(value); Py_DECREF(key); } @@ -11442,6 +12289,13 @@ static inline PyObject *By_BuildForwarders(PyObject *module, PyObject *dict, PyM Py_DECREF(staging); return NULL; } +#if PY_VERSION_HEX < 0x030C0000 + if ((methods[at].ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O + | METH_KEYWORDS | METH_METHOD)) + == (METH_FASTCALL | METH_KEYWORDS)) { + ((PyCFunctionObject *)native)->vectorcall = By_NativeEntryVectorcall; + } +#endif PyTuple_SET_ITEM(natives, at, native); } Py_DECREF(modname); @@ -11483,13 +12337,19 @@ static inline PyObject *By_BuiltForwarder(PyObject *staging, const char *name) { /* bind each built forwarder under the name its definition holds, which is the moment * the definition stops being what that name means * + * `stands` says, forwarder by forwarder, whether the name still holds the definition — + * see `By_DefinitionStanding`. one the module body wrote over keeps what the body wrote. + * * the installer took itself back out of the *copy* it was handed, so the module's own * namespace is still carrying it and this is where that is undone */ static inline int By_InstallForwarders(PyObject *dict, PyObject *staging, PyMethodDef *methods, - Py_ssize_t count, const char *installer) { + Py_ssize_t count, const char *installer, + const char *stands) { Py_ssize_t at; for (at = 0; at < count; at++) { - PyObject *built = By_BuiltForwarder(staging, methods[at].ml_name); + PyObject *built; + if (!stands[at]) continue; + built = By_BuiltForwarder(staging, methods[at].ml_name); if (built == NULL) { PyErr_Format(PyExc_ImportError, "this module's installer published no forwarder for '%s'", @@ -12414,6 +13274,15 @@ static inline PyObject *By_ThrowIntoDelegate(PyObject *self, PyObject **sent, Py * machine with no suspension point does not reach the frame either, but a throw does * finish it — see below. otherwise it is the resumption that decides, and a body that * catches what was thrown leaves the machine usable */ +#if PY_VERSION_HEX < 0x030C0000 +/* the exception a throw is raising at a delegation the inner iterator had no `throw` for, + * which 3.11 does not take as the delegation finishing — see `By_StopIterationValue`. + * borrowed, and only ever compared: it is set just before the frame resumes and the + * frame reads it at the suspension it resumes at, with nothing between that could run + * python, so the global interpreter lock is all the guard it needs */ +static PyObject *by_raised_past_delegation = NULL; +#endif + static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject **thrown, PyObject **returned, int64_t *state, int frame, PyObject *const *args, Py_ssize_t nargs, @@ -12459,6 +13328,15 @@ static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject * Py_DECREF(instance); return NULL; } +#if PY_VERSION_HEX < 0x030C0000 + if (inner != NULL) { + PyObject *answer; + by_raised_past_delegation = instance; + answer = By_ResumeRaising(self, sent, thrown, returned, state, frame, instance, resume); + by_raised_past_delegation = NULL; + return answer; + } +#endif return By_ResumeRaising(self, sent, thrown, returned, state, frame, instance, resume); } @@ -12466,12 +13344,22 @@ static inline PyObject *By_ThrowInto(PyObject *self, PyObject **sent, PyObject * * * a `StopIteration` raised at a frame's delegation is the delegation *finishing*: python * takes its value as what `yield from` or `await` evaluates to, which is how an inner - * iterator's `throw` that ends it hands back a result */ + * iterator's `throw` that ends it hands back a result. + * + * from 3.12 that holds for one thrown at the delegation as well, where the inner iterator + * has no `throw` to hand it to. 3.11 raises that one at the suspension as it raises any + * other exception, so it is not the delegation's value there */ static inline PyObject *By_StopIterationValue(PyObject *exception) { if (exception == NULL || !PyExceptionInstance_Check(exception) || !PyErr_GivenExceptionMatches(exception, PyExc_StopIteration)) { return NULL; } +#if PY_VERSION_HEX < 0x030C0000 + if (exception == by_raised_past_delegation) { + by_raised_past_delegation = NULL; + return NULL; + } +#endif return By_NewRef(((PyStopIterationObject *)exception)->value); } @@ -12493,7 +13381,8 @@ static inline void By_CloseUnraisable(PyObject *self) { * coroutine over as the warning's `source` — which is what `tracemalloc` hangs the * allocation traceback off. the stack level is the frame the coroutine was dropped in, * the same one python's `stacklevel=2` reaches from the helper it warns through. a failure - * to warn has nowhere to go and is reported as unraisable, as python reports it */ + * to warn has nowhere to go and is reported as unraisable, as python reports it: against the + * coroutine, which 3.14 says it was finalizing */ static inline void By_WarnUnawaitedCoroutine(PyObject *coroutine, const char *qualname) { PyObject *warnings = PyImport_ImportModule("warnings"); PyObject *warn = warnings == NULL ? NULL : PyObject_GetAttrString(warnings, "warn"); @@ -12510,12 +13399,60 @@ static inline void By_WarnUnawaitedCoroutine(PyObject *coroutine, const char *qu Py_XDECREF(message); Py_XDECREF(warn); if (result == NULL) { +#if PY_VERSION_HEX >= 0x030E0000 + PyErr_FormatUnraisable("Exception ignored while finalizing coroutine %R", coroutine); +#else PyErr_WriteUnraisable(coroutine); +#endif return; } Py_DECREF(result); } +/* python's `RuntimeWarning` for one of an async generator's awaitables dropped before it + * was ever stepped: `__anext__()` and `asend()` report themselves as `asend`, and + * `athrow()` and `aclose()` under their own names. python only warns from 3.13 + * + * the warning is issued the way `By_WarnUnawaitedCoroutine` issues its own, with the + * generator as the `source`. a failure to warn is reported as unraisable against the + * generator, which 3.14 says it was finalizing */ +static inline void By_WarnUnawaitedAgenMethod(PyObject *agen, const char *method, + const char *qualname) { + PyObject *exc_type, *exc_value, *exc_tb; + PyErr_Fetch(&exc_type, &exc_value, &exc_tb); + PyObject *warnings = PyImport_ImportModule("warnings"); + PyObject *warn = warnings == NULL ? NULL : PyObject_GetAttrString(warnings, "warn"); + Py_XDECREF(warnings); + PyObject *method_name = warn == NULL ? NULL : PyUnicode_FromString(method); + PyObject *qualified = method_name == NULL ? NULL : PyUnicode_FromString(qualname); + PyObject *message = + qualified == NULL + ? NULL + : PyUnicode_FromFormat("coroutine method %R of %R was never awaited", method_name, + qualified); + PyObject *level = message == NULL ? NULL : PyLong_FromLong(1); + PyObject *result = NULL; + if (level != NULL) { + PyObject *args[] = {message, PyExc_RuntimeWarning, level, agen}; + result = PyObject_Vectorcall(warn, args, 4, NULL); + } + Py_XDECREF(level); + Py_XDECREF(message); + Py_XDECREF(qualified); + Py_XDECREF(method_name); + Py_XDECREF(warn); + if (result == NULL) { +#if PY_VERSION_HEX >= 0x030E0000 + PyErr_FormatUnraisable("Exception ignored while finalizing async generator %R", agen); +#else + PyErr_WriteUnraisable(agen); +#endif + } else { + Py_DECREF(result); + } + PyErr_Restore(exc_type, exc_value, exc_tb); +} + /* `close()`: throw `GeneratorExit` in and accept the three legal outcomes, answering * what `close()` returns. * @@ -14066,10 +15003,14 @@ static PyObject *By_AnnotatedDefinition(PyObject *self) { PyObject *made = function->made_type_params == NULL ? Py_None : function->made_type_params; PyObject *definition = PyObject_CallFunctionObjArgs(factory, values, made, NULL); Py_DECREF(values); + /* a function has type parameters from 3.12. before it nothing declares any, so the + * factory is handed `None` every time and never reads it */ +#if PY_VERSION_HEX >= 0x030C0000 if (definition != NULL && function->made_type_params == NULL) { function->made_type_params = PyObject_GetAttrString(definition, "__type_params__"); if (function->made_type_params == NULL) Py_CLEAR(definition); } +#endif return definition; } @@ -14097,8 +15038,10 @@ static inline PyObject *By_AnnotateAtDefinition(PyObject *self) { return self; } +#if PY_VERSION_HEX >= 0x030C0000 /* python makes a function's type parameters with the function, so they are made once for - * each closure and kept */ + * each closure and kept. a function has none before 3.12, and no `__type_params__` to + * answer with */ static PyObject *By_Function_get_type_params(PyObject *self, void *closure) { (void)closure; ByFunctionObject *function = (ByFunctionObject *)self; @@ -14127,6 +15070,7 @@ static int By_Function_set_type_params(PyObject *self, PyObject *value, void *cl Py_XSETREF(((ByFunctionObject *)self)->type_params, By_NewRef(value)); return 0; } +#endif #if PY_VERSION_HEX >= 0x030E0000 /* from 3.14 python evaluates annotations when they are first asked for, through an @@ -14308,6 +15252,31 @@ static PyObject *By_Function_default_half(PyObject *self, Py_ssize_t half) { return answer; } +/* `f.__closure__`: `None` where the definition captures nothing, as python's is + * + * a definition that does capture names has its cells in python, and a compiled one keeps + * those names in one environment its siblings share, where a write to one is a write every + * sibling sees. a tuple of new cells would hold the values the names have now and go on + * holding them, and a write to `cell_contents` would reach nothing, so such a function + * refuses the attribute rather than answer with cells that only look like its own */ +static PyObject *By_Function_get_closure(PyObject *self, void *closure) { + (void)closure; + PyObject *code = By_Function_get_code(self, NULL); + PyObject *free; + Py_ssize_t captured; + if (code == NULL) return NULL; + free = PyCode_GetFreevars((PyCodeObject *)code); + Py_DECREF(code); + if (free == NULL) return NULL; + captured = PyTuple_GET_SIZE(free); + Py_DECREF(free); + if (captured == 0) Py_RETURN_NONE; + PyErr_SetString(PyExc_AttributeError, + "'function' object has no attribute '__closure__': a compiled closure " + "keeps the names it captures in an environment rather than in cells"); + return NULL; +} + static PyObject *By_Function_get_defaults(PyObject *self, void *closure) { (void)closure; return By_Function_default_half(self, 0); @@ -14345,7 +15314,10 @@ static PyGetSetDef By_Function_getset[] = { {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, {"__globals__", By_Function_get_globals, NULL, NULL, NULL}, {"__code__", By_Function_get_code, NULL, NULL, NULL}, + {"__closure__", By_Function_get_closure, NULL, NULL, NULL}, +#if PY_VERSION_HEX >= 0x030C0000 {"__type_params__", By_Function_get_type_params, By_Function_set_type_params, NULL, NULL}, +#endif {"__defaults__", By_Function_get_defaults, NULL, NULL, NULL}, {"__kwdefaults__", By_Function_get_kwdefaults, NULL, NULL, NULL}, {"__annotations__", By_Function_get_annotations, By_Function_set_annotations, NULL, NULL}, @@ -14355,6 +15327,41 @@ static PyGetSetDef By_Function_getset[] = { {NULL, NULL, NULL, NULL, NULL}, }; +/* an attribute read, with python's own wording for a name a function does not have + * + * the generic lookup names the type by its `tp_name`, which here is `by.function` where + * python's function says `function`. what is reworded is only the lookup's own refusal, + * the one that names this object and this name, so an `AttributeError` a getter raised + * stays as it was raised. which names a function has moves between versions — + * `__type_params__` is 3.12's and `__annotate__` 3.14's — so this is what a program + * reading one of those on an earlier version is told */ +static PyObject *By_Function_getattro(PyObject *self, PyObject *name) { + PyObject *found = PyObject_GenericGetAttr(self, name); + PyObject *type, *value, *traceback; + if (found != NULL || !PyErr_ExceptionMatches(PyExc_AttributeError)) return found; + PyErr_Fetch(&type, &value, &traceback); + PyErr_NormalizeException(&type, &value, &traceback); + if (value != NULL && PyErr_GivenExceptionMatches(value, PyExc_AttributeError) + && ((PyAttributeErrorObject *)value)->obj == self + && ((PyAttributeErrorObject *)value)->name != NULL + && PyUnicode_Check(((PyAttributeErrorObject *)value)->name) + && PyUnicode_Compare(((PyAttributeErrorObject *)value)->name, name) == 0) { + PyObject *message = + PyUnicode_FromFormat("'function' object has no attribute '%U'", name); + PyObject *args = message == NULL ? NULL : PyTuple_Pack(1, message); + Py_XDECREF(message); + if (args == NULL) { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + return NULL; + } + Py_XSETREF(((PyBaseExceptionObject *)value)->args, args); + } + PyErr_Restore(type, value, traceback); + return NULL; +} + static PyTypeObject By_FunctionType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "by.function", @@ -14364,7 +15371,7 @@ static PyTypeObject By_FunctionType = { .tp_vectorcall_offset = offsetof(ByFunctionObject, vectorcall), .tp_repr = (reprfunc)By_Function_repr, .tp_call = PyVectorcall_Call, - .tp_getattro = PyObject_GenericGetAttr, + .tp_getattro = By_Function_getattro, .tp_setattro = PyObject_GenericSetAttr, /* a method descriptor as python's own function is one: `obj.f()` through the type * calls it with `obj` in front rather than building a bound method first */ diff --git a/crates/by_transforms/src/lib.rs b/crates/by_transforms/src/lib.rs index 01c6ae8b26..26581c5ed9 100644 --- a/crates/by_transforms/src/lib.rs +++ b/crates/by_transforms/src/lib.rs @@ -197,6 +197,7 @@ fn transpile_with_report( if config.is_python { return Ok((source.to_owned(), RuntimeRequirements::default())); } + verify_destructuring_target(source, config).map_err(|e| e.message)?; // one db over the original source, shared by the qualification phase below // and — as long as nothing rewrites the source — by phase 0's type-aware @@ -423,6 +424,7 @@ pub fn transpile_typed_with_report( RuntimeRequirements::default(), )); } + verify_destructuring_target(original_source, config)?; let parsed = ruff_db::parsed::parsed_module( db, @@ -637,9 +639,10 @@ fn line_count(s: &str) -> usize { newline_count(s) + usize::from(!(s.is_empty() || s.ends_with('\n'))) } -/// Re-runs the anon-named-tuple lowering on post-transform output to catch -/// expressions that other transforms (e.g. the PEP-695 polyfill) copied -/// verbatim from the source after the original pass ran +/// Re-runs the synthesized-type lowerings on post-transform output to catch the +/// types other transforms (e.g. the PEP-695 polyfill) copied verbatim from the +/// source after the original passes ran, and the anonymous named tuple value +/// forms among them fn run_anon_named_tuple_cleanup( mut source: String, written: WrittenNames, @@ -669,13 +672,14 @@ enum CleanupRound { /// one round of [`run_anon_named_tuple_cleanup`] over `source` /// -/// the only syntax the round lowers is an anonymous named tuple form and an inline protocol, -/// and the only other thing it does is construct each plain tuple literal the checker reads as -/// an anonymous named tuple. the checker makes that type only from a form the parser flags -/// (`DynamicNamedTupleAnchor::Anonymous`), and the round's db reads nothing but `source` and -/// the vendored stubs, none of which spells one (`vendored_stubs_spell_no_anonymous_named_tuple` -/// checks that). so a `source` that spells neither has nothing to lower or construct, and the -/// round leaves it before building a db +/// the only basedpython-only syntax the round lowers is an anonymous named tuple form and an +/// inline protocol — a typed dict literal type is a python dict display, which the round writes +/// as its class only beside one of those — and the only other thing it does is construct each +/// plain tuple literal the checker reads as an anonymous named tuple. the checker makes that +/// type only from a form the parser flags (`DynamicNamedTupleAnchor::Anonymous`), and the +/// round's db reads nothing but `source` and the vendored stubs, none of which spells one +/// (`vendored_stubs_spell_no_anonymous_named_tuple` checks that). so a `source` that spells +/// neither has nothing to lower or construct, and the round leaves it before building a db fn run_cleanup_round( source: &str, written: WrittenNames, @@ -703,14 +707,28 @@ fn run_cleanup_round( ty_python_semantic::Db::program_file(&db, file), ); + let synthesized = std::rc::Rc::new(transforms::synthesized_types::SynthesizedTypes::name_all( + src, + written, + quotes, + &model, + config, + module.suite(), + &[], + &[], + )); + if let Some(err) = synthesized.errors.first() { + return Err(err.clone()); + } let mut anon = transforms::anon_named_tuple::AnonNamedTuple::new( src, written, quotes, &model, config.clone(), - module.suite(), + synthesized.clone(), ); + anon.annotations_evaluated = annotations_evaluated(module.suite(), config); for stmt in module.suite() { anon.visit_stmt(stmt); } @@ -718,52 +736,57 @@ fn run_cleanup_round( if let Some(err) = anon.errors.first() { return Err(err.clone()); } - let protocol = - transforms::protocol_type::cleanup(src, written, quotes, &model, module.suite(), config)?; - if anon.edits.is_empty() && !anon.needs_import && protocol.is_none() { + if anon.edits.is_empty() && !anon.needs_import && synthesized.is_empty() { return Ok(CleanupRound::Unchanged); } + // an earlier run over the pre-splice source already emitted a class (and its + // imports) for a type whose edit an AST pass then re-rendered away, so only what + // the spliced output is actually missing gets prepended. matched as a run of + // whole lines: a bare `contains` would let `from typing import ProtocolFoo` + // suppress the `Protocol` import that is needed let mut preamble = String::new(); + let source_lines: Vec<&str> = source.lines().collect(); + let push_missing = |preamble: &mut String, entry: &str| { + let entry_lines: Vec<&str> = entry.lines().collect(); + let present = !entry_lines.is_empty() + && source_lines + .windows(entry_lines.len()) + .any(|window| window == entry_lines.as_slice()); + if !present { + preamble.push_str(entry); + preamble.push('\n'); + } + }; if anon.needs_import { - // an earlier run over the pre-splice source already emitted a class - // (and its imports) for an anon-NT whose edit an AST pass then - // re-rendered away, so only what the spliced output is actually - // missing gets prepended. matched as a run of whole lines, the way - // the inline-protocol cleanup does - let source_lines: Vec<&str> = source.lines().collect(); - let push_missing = |preamble: &mut String, entry: &str| { - let entry_lines: Vec<&str> = entry.lines().collect(); - let present = !entry_lines.is_empty() - && source_lines - .windows(entry_lines.len()) - .any(|window| window == entry_lines.as_slice()); - if !present { - preamble.push_str(entry); - preamble.push('\n'); - } - }; push_missing( &mut preamble, &written.import_from("typing", &["NamedTuple"]), ); - let (imports, helpers) = anon.callable.take_requirements(); - for line in imports - .into_iter() - .chain(runtime_entries(config, &helpers, source)) - { - push_missing(&mut preamble, line.trim_end_matches('\n')); - } - for defs in [anon.callable.class_defs().to_owned(), anon.class_defs()] { - for class_def in defs.split_inclusive("\n\n") { - push_missing(&mut preamble, class_def.trim_end_matches('\n')); - } + } + for line in synthesized.imports.iter().cloned().chain(runtime_entries( + config, + &synthesized.helpers, + source, + )) { + push_missing(&mut preamble, line.trim_end_matches('\n')); + } + for defs in [synthesized.declarations.clone(), anon.class_defs()] { + for class_def in defs.split_inclusive("\n\n") { + push_missing(&mut preamble, class_def.trim_end_matches('\n')); } } let mut edits = anon.edits; - if let Some((protocol_edits, protocol_preamble)) = protocol { - preamble.push_str(&protocol_preamble); - edits.extend(protocol_edits); + for kind in [ + transforms::synthesized_types::SynthesizedKind::TypedDict, + transforms::synthesized_types::SynthesizedKind::Protocol, + ] { + edits.extend( + synthesized + .of_kind(kind) + .into_iter() + .map(|(range, name)| Fix::safe_edit(Edit::range_replacement(name, range))), + ); } let (body, _) = apply_transforms_once(src, edits); @@ -947,6 +970,7 @@ fn run_lazy_import_phase( &eager_names, deferrable.keys().copied().collect(), &holder, + written, ); for stmt in module.suite() { lazy.visit_stmt(stmt); @@ -977,6 +1001,12 @@ fn run_lazy_import_phase( } let (mut body, _) = apply_transforms_once(src, edits); + // a builtin the module binds for itself is read under a name of the lowering's own, which + // is imported ahead of everything + let builtins = written.builtin_imports(&body, config.min_version.minor); + if !builtins.is_empty() { + body = splice_preamble(&body, &format!("{}\n", builtins.join("\n"))); + } // phase 0 already gave the module the helpers *it* needed. where that was an import of the // runtime written beside it, the helpers this phase needs belong on that same line rather // than on a second one naming the same module @@ -1076,17 +1106,18 @@ fn runtime_entries(config: &Config, helpers: &[runtime::Helper], module: &str) - } /// Version polyfill phase: rewrite syntax the target python cannot parse into -/// syntax it can. Runs over the finished python rather than over `.by`, so a -/// `match` an earlier lowering *generated* — for a `let` destructuring, an -/// `if let`, a statement expression — is lowered by the same code as one the -/// author wrote. +/// syntax it can — a `match` statement, and the types of a bracketless `except`. +/// Runs over the finished python rather than over `.by`, so a `match` an earlier +/// lowering *generated* — for a statement expression, an enum's exhaustiveness — +/// is lowered by the same code as one the author wrote. /// -/// Nothing here changes how many lines the file has: the polyfill rewrites -/// headers in place and pads them back to their original height, so the only -/// lines it adds are the runtime preamble's, at the top, where the line map +/// Nothing here changes how many lines the file has: the polyfills rewrite +/// headers in place and pad them back to their original height, so the only +/// lines they add are the runtime preamble's, at the top, where the line map /// already accounts for generated leading lines. fn run_version_polyfill_phase(source: String, config: &Config) -> String { - transforms::match_polyfill::lower(source, config) + let source = transforms::match_polyfill::lower(source, config); + transforms::except_polyfill::lower(source, config) } /// Re-parse the transpiled output *as the target python version* and report any @@ -1132,6 +1163,38 @@ fn verify_target_syntax(source: &str, config: &Config) -> Result<(), TranspileEr Ok(()) } +/// refuse a destructuring written for a target that has no `match` statement, as `by check` +/// reports it there +/// +/// a destructuring is written as a `match` statement, which the `match` lowering rewrites for a +/// target from 3.8, so [`verify_target_syntax`] finds nothing of it in the output. the parser +/// holds it to 3.10 all the same, in `UnsupportedSyntaxErrorKind::is_lowered_by_basedpython`, +/// which decides the version for this check and the checker both. every other kind is checked +/// on the output, which is the python the target runs +fn verify_destructuring_target(source: &str, config: &Config) -> Result<(), TranspileError> { + let source_type = if config.is_stub { + ruff_python_ast::PySourceType::BasedPythonStub + } else { + ruff_python_ast::PySourceType::BasedPython + }; + let options = + ruff_python_parser::ParseOptions::from(source_type).with_target_version(config.min_version); + let parsed = ruff_python_parser::parse_unchecked(source, options); + let Some(first) = parsed + .unsupported_syntax_errors() + .iter() + .find(|error| error.kind == ruff_python_parser::UnsupportedSyntaxErrorKind::Destructuring) + else { + return Ok(()); + }; + Err(TranspileError { + kind: TranspileErrorKind::InvalidSyntax, + message: first.to_string(), + output_range: None, + by_range: Some(first.range), + }) +} + /// Splice `preamble` into `body` where generated lines belong: after the module /// docstring and any `from __future__ import`, each of which is only valid /// where it already is. @@ -2829,109 +2892,6 @@ mod cross_file { ); } - /// Locate an interpreter for the cross-module runtime checks. - fn python() -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3", "python"].map(String::from)); - candidates.into_iter().find(|py| { - std::process::Command::new(py) - .args(["-c", ""]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) - } - - /// a target new enough that `override` resolves in the stdlib rather than - /// through `typing_extensions`, which the test environment need not have - fn runtime_config() -> Config { - Config { - min_version: ruff_python_ast::PythonVersion::PY313, - ..Config::test_default() - } - } - - /// Transpile every module of `project` into a temp directory and run `main`. - /// - /// Asserting the emitted *text* of each file in isolation is not enough for - /// anything cross-module: a conformance registers in one module and is read - /// in another, so only executing both together proves the two halves meet. - /// Two text-asserting tests once passed while the program they described - /// raised `AttributeError`. - fn run_project(project: &Project, config: &Config) -> Result { - let Some(python) = python() else { - return Ok("SKIPPED".to_owned()); - }; - let dir = tempfile::tempdir().expect("temp dir"); - for (path, _) in &project.files { - let out = transpile_file(project, path, config); - let name = path.trim_start_matches('/').replace(".by", ".py"); - let target = dir.path().join(&name); - if let Some(parent) = target.parent() { - std::fs::create_dir_all(parent).expect("mkdir"); - } - std::fs::write(&target, out).expect("write"); - } - let output = std::process::Command::new(&python) - .arg("main.py") - .current_dir(dir.path()) - .output() - .expect("spawn python"); - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) - } else { - Err(String::from_utf8_lossy(&output.stderr).to_string()) - } - } - - /// the feature's headline case: the interface comes from one module, the - /// conformance from another, and the consumer sees both only through its - /// imports. this must *run*, not merely lower - #[test] - fn an_imported_conformance_dispatches_at_runtime() { - let project = project_db(&[ - ("/iface.by", "protocol Show:\n def show(self) -> str\n"), - ( - "/adapters.by", - "from iface import Show\n\nextension str(Show):\n override def show(self) -> str:\n return \"OWN:\" + self\n", - ), - ( - "/main.by", - "import adapters\nfrom iface import Show\n\ndef render(value: Show) -> str:\n return value.show()\n\nprint(render(\"hi\"))\n", - ), - ]); - match run_project(&project, &runtime_config()) { - Ok(out) if out == "SKIPPED" => {} - Ok(out) => assert_eq!(out, "OWN:hi"), - Err(err) => panic!("the imported conformance did not dispatch:\n{err}"), - } - } - - /// the module that declares the protocol-typed function is *upstream* of the - /// conformance by construction, so it can never see one. it still has to - /// emit a dispatch - #[test] - fn a_conformance_declared_downstream_of_its_use_still_dispatches() { - let project = project_db(&[ - ( - "/lib.by", - "protocol Show:\n def show(self) -> str\n\ndef render(value: Show) -> str:\n return value.show()\n", - ), - ( - "/main.by", - "from lib import Show, render\n\nextension str(Show):\n override def show(self) -> str:\n return \"OWN:\" + self\n\nprint(render(\"hi\"))\n", - ), - ]); - match run_project(&project, &runtime_config()) { - Ok(out) if out == "SKIPPED" => {} - Ok(out) => assert_eq!(out, "OWN:hi"), - Err(err) => panic!("a downstream conformance did not dispatch:\n{err}"), - } - } - /// a project with no conformance anywhere is byte-for-byte unaffected. /// /// whether a requirement dispatches is a whole-program question — a diff --git a/crates/by_transforms/src/runtime/_by_runtime.py b/crates/by_transforms/src/runtime/_by_runtime.py index a4ed952803..e6df6451ef 100644 --- a/crates/by_transforms/src/runtime/_by_runtime.py +++ b/crates/by_transforms/src/runtime/_by_runtime.py @@ -397,6 +397,22 @@ def _by_rebind(_by_fn): # --- string templates and graphemes ---------------------------------------- +# a t-string field's source text, as the interpreter running it reports one +# +# a lowered field's text keeps the whitespace before the `=`, conversion, format spec +# or `}` that ends the field, which is what python reports from 3.14.8 on. 3.14.0 to +# 3.14.7 dropped it (python/cpython#154719), so there it is dropped here too, or a +# lowered field would report different text from an untouched field beside it. an +# interpreter older than 3.14 has no t-strings and so no form of its own, and keeps +# the newer one +def _by_field_text(_text): + from sys import version_info + + if (3, 14) <= version_info < (3, 14, 8): + return _text.rstrip() + return _text + + # PEP 750 `Template` / `Interpolation` polyfill for runtimes before 3.14 # # matches the `string.templatelib` shape a tag relies on: `Template.strings` @@ -404,10 +420,13 @@ def _by_rebind(_by_fn): # `Template.interpolations` is the replacement fields, and `Template.values` # is their evaluated values. iterating a `Template` yields the segments and # interpolations interleaved in source order, the same as the stdlib type +# +# the lowering builds every `_Interpolation` with its field's source text in the form +# python 3.14.8 reports, which `_by_field_text` matches to the interpreter class _Interpolation: def __init__(self, value, expression, conversion=None, format_spec=""): self.value = value - self.expression = expression + self.expression = _by_field_text(expression) self.conversion = conversion self.format_spec = format_spec @@ -451,31 +470,28 @@ def __iter__(self): # `None` a field nothing lowered, which keeps the interpolation python built and so # stays byte-identical to what python would have reported # -# the author's text keeps the whitespace before the `=`, conversion, format spec or -# `}` that ends the field, which is what python reports from 3.14.8 on. 3.14.0 to -# 3.14.7 dropped it (python/cpython#154719), so there it is dropped here too, or a -# lowered field would report different text from an untouched field beside it +# the author's text is written as python 3.14.8 reports it, which `_by_field_text` +# matches to the interpreter running it # # the import is made here rather than at the top of the file because this module is # shared with runtimes that have no `string.templatelib`. only a t-string can reach # this helper, and a t-string needs 3.14 def _by_template(_t, _exprs): from string.templatelib import Interpolation, Template - from sys import version_info - _drops_trailing_space = version_info < (3, 14, 8) _strings = _t.strings _parts = [] for _index, _field in enumerate(_t.interpolations): _parts.append(_strings[_index]) _expr = _exprs[_index] - if _expr is not None and _drops_trailing_space: - _expr = _expr.rstrip() _parts.append( _field if _expr is None else Interpolation( - _field.value, _expr, _field.conversion, _field.format_spec + _field.value, + _by_field_text(_expr), + _field.conversion, + _field.format_spec, ) ) _parts.append(_strings[-1]) @@ -814,7 +830,10 @@ def _by_specialize(alias, origin, depth=0): if not isinstance(klass, type): return None args = getattr(alias, "__args__", ()) - params = getattr(klass, "__type_params__", ()) + # a class the pep 695 polyfill wrote declares its parameters through + # `Generic[...]`, which records them in `__parameters__` and leaves + # `__type_params__` empty, or absent before 3.12 + params = getattr(klass, "__type_params__", ()) or getattr(klass, "__parameters__", ()) if not args: defaulted = _by_type_param_defaults(params) if defaulted is not params: diff --git a/crates/by_transforms/src/transforms/annotation.rs b/crates/by_transforms/src/transforms/annotation.rs index 77d759a584..71a33c160a 100644 --- a/crates/by_transforms/src/transforms/annotation.rs +++ b/crates/by_transforms/src/transforms/annotation.rs @@ -45,10 +45,11 @@ pub(crate) struct TupleLiteralType<'src> { /// dedicated passes' edits inside it are dropped — without this an element /// would reach the output as surface syntax leaves: RefCell>, - /// the operations `symbolic_type_op` folded, with their rendered types. our - /// replacement covers a folded operation nested inside a tuple type, so it - /// has to carry the fold rather than re-emit the operation - symbolic_substitutions: &'src [(TextRange, String)], + /// the type expressions already lowered to a fixed spelling — an operation + /// `symbolic_type_op` folded, a synthesized type's class. our replacement + /// covers one nested inside a tuple type, so it has to carry that spelling + /// rather than re-emit the source + type_substitutions: &'src [(TextRange, String)], edits: Vec, templates: Vec<(TextRange, Vec)>, } @@ -60,10 +61,10 @@ impl<'src> TupleLiteralType<'src> { quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: &Config, - symbolic_substitutions: &'src [(TextRange, String)], + type_substitutions: &'src [(TextRange, String)], ) -> Self { let mut leaves = CallableSyntax::new(source, written, quotes, config).with_types(types); - for (range, rendered) in symbolic_substitutions { + for (range, rendered) in type_substitutions { leaves.add_substitution(*range, rendered.clone()); } Self { @@ -73,7 +74,7 @@ impl<'src> TupleLiteralType<'src> { min_version: config.min_version, needs_unpack_import: Cell::new(false), leaves: RefCell::new(leaves), - symbolic_substitutions, + type_substitutions, edits: Vec::new(), templates: Vec::new(), } @@ -83,8 +84,15 @@ impl<'src> TupleLiteralType<'src> { /// star form is PEP 646, python 3.11; below that it is a *syntax* error /// rather than something the `__future__` import could defer, so a tuple /// type has to say `Unpack[...]` instead. - fn unpack(&self, inner: &str) -> String { - if self.min_version >= PythonVersion::PY311 { + /// + /// `operand` is the type the element unpacks, where the source names one. + /// python carries out a star by iterating its operand, which the + /// `typing_extensions.TypeAliasType` a type alias is below 3.12 cannot do, + /// so a starred alias says `Unpack[...]` until 3.12. + fn unpack(&self, inner: &str, operand: Option<&Expr>) -> String { + let iterable = self.min_version >= PythonVersion::PY312 + || !operand.is_some_and(|operand| self.types.is_type_alias(operand)); + if self.min_version >= PythonVersion::PY311 && iterable { format!("*{inner}") } else { self.needs_unpack_import.set(true); @@ -174,7 +182,7 @@ impl<'src> TupleLiteralType<'src> { // folded to Expr::BinOp(b) if let Some((_, rendered)) = self - .symbolic_substitutions + .type_substitutions .iter() .find(|(range, _)| *range == b.range()) => { @@ -302,15 +310,15 @@ impl<'src> TupleLiteralType<'src> { let inner_src = self .transform_annotation(&unpacked.value) .unwrap_or_else(|| self.fallback_src(&unpacked.value)); - return self.unpack(&inner_src); + return self.unpack(&inner_src, Some(&unpacked.value)); } let value_src = self .transform_annotation(&named.value) .unwrap_or_else(|| self.fallback_src(&named.value)); - return self.unpack(&format!( - "{}[{value_src}, ...]", - self.written.builtin("tuple") - )); + return self.unpack( + &format!("{}[{value_src}, ...]", self.written.builtin("tuple")), + None, + ); } self.transform_annotation(&named.value) .unwrap_or_else(|| self.fallback_src(&named.value)) @@ -325,12 +333,12 @@ impl<'src> TupleLiteralType<'src> { .transform_annotation(&s.value) .unwrap_or_else(|| self.fallback_src(&s.value)); if parameter_shape { - self.unpack(&format!( - "{}[{value_src}, ...]", - self.written.builtin("tuple") - )) + self.unpack( + &format!("{}[{value_src}, ...]", self.written.builtin("tuple")), + None, + ) } else { - self.unpack(&value_src) + self.unpack(&value_src, Some(&s.value)) } } // a plain element type — also lower a nested `?` (`(int, str?)`), @@ -442,14 +450,14 @@ impl TypeAwarePass for TupleLiteralTypePass<'_> { } fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let symbolic_substitutions = ctx.symbolic_substitutions.clone(); + let type_substitutions = ctx.type_substitutions.clone(); let mut inner = TupleLiteralType::new( self.source, self.written, self.quotes, types, &self.config, - &symbolic_substitutions, + &type_substitutions, ); walk_type_positions(stmts, Some(types), &mut inner); ctx.template_edits.append(&mut inner.templates); @@ -578,6 +586,37 @@ mod tests { ); } + /// python carries out a star by iterating what it unpacks, and the + /// `typing_extensions.TypeAliasType` a type alias is below 3.12 cannot be + /// iterated, so a starred alias is spelled `Unpack[...]` on 3.11 + #[test] + fn unpacked_type_alias_before_312() { + check_at( + PythonVersion::PY311, + indoc! {" + type Pair = (int, str) + a: (bool, *Pair) + "}, + indoc! {" + from typing import Unpack + from typing_extensions import TypeAliasType + Pair = TypeAliasType(\"Pair\", tuple[int, str]) + a: tuple[bool, Unpack[Pair]] + "}, + ); + check_at( + PythonVersion::PY312, + indoc! {" + type Pair = (int, str) + a: (bool, *Pair) + "}, + indoc! {" + type Pair = tuple[int, str] + a: tuple[bool, *Pair] + "}, + ); + } + #[test] fn lone_unpacked_element_without_a_comma() { check_at(PythonVersion::PY311, "a: (*A)\n", "a: tuple[*A]\n"); diff --git a/crates/by_transforms/src/transforms/anon_named_tuple.rs b/crates/by_transforms/src/transforms/anon_named_tuple.rs index 76c99f6252..ab9c0eebb9 100644 --- a/crates/by_transforms/src/transforms/anon_named_tuple.rs +++ b/crates/by_transforms/src/transforms/anon_named_tuple.rs @@ -4,10 +4,11 @@ //! Each unique shape (field-name + field-type-source tuple) is hoisted to a //! single class definition and reused across all occurrences. Identical shapes //! across the module collapse to one class so structural equivalence is -//! preserved at the type level. -//! -//! a plain tuple literal the checker reads as an instance of an anonymous named -//! tuple — one standing wherever one is expected — is constructed as its class +//! preserved at the type level. The type forms are named by +//! [`synthesized_types`](super::synthesized_types), with the other kinds of type +//! that lower to a class; this pass writes those names, the value forms +//! `(name=v, ...)`, and a call of the class around each plain tuple literal the +//! checker took for one. //! //! Example: //! ```by @@ -36,15 +37,14 @@ use std::fmt::Write as _; use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; -use ruff_python_ast::{Expr, Stmt}; +use ruff_python_ast::Expr; +use ruff_python_ast::visitor::{Visitor, walk_expr}; use ruff_text_size::Ranged; use ty_python_semantic::AnonNamedTupleId; -use crate::transforms::callable::CallableSyntax; -use crate::transforms::generics::PolyfilledTypeParams; use crate::transforms::quotes::WrittenQuotes; use crate::transforms::repeated_underscore::WrittenNames; +use crate::transforms::synthesized_types::{Namer, SynthesizedKind, SynthesizedTypes}; use crate::type_info::{AnonNamedTupleConstruction, AnonNamedTupleField, TypeInfo}; /// Synthetic field name for the i-th positional element of a mixed @@ -76,18 +76,18 @@ fn first_duplicate_name(fields: &[(String, String)]) -> Option<&str> { /// One anonymous named tuple shape: ordered list of `(field_name, type_source)`. #[derive(Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -struct Shape { +pub(crate) struct Shape { fields: Vec<(String, String)>, } impl Shape { /// the name of the class declaring this shape, one the module has no other use of - fn class_name(&self, written: WrittenNames) -> String { + pub(crate) fn class_name(&self, written: WrittenNames) -> String { written.synthesized_class("_AnonNamedTuple_", self) } /// the class declaring this shape as `name`, `typing.NamedTuple` written as `named_tuple` - fn class_def(&self, name: &str, named_tuple: &str) -> String { + pub(crate) fn class_def(&self, name: &str, named_tuple: &str) -> String { let mut out = format!("class {name}({named_tuple}):\n"); for (field_name, field_type) in &self.fields { let _ = writeln!(out, " {field_name}: {field_type}"); @@ -96,6 +96,93 @@ impl Shape { } } +/// The shape of a type-form anonymous named tuple (`(name: T, ...)`), each field's type +/// lowered through the synthesized type namer, which writes a nested typed dict, inline +/// protocol or named tuple as its class. Positional fields (bare type expressions like `int` in +/// `(int, name: str)`) are assigned synthetic names `arg0`, `arg1`, … `NamedTuple` disallows +/// leading-underscore field names, so `argN` is used. +/// +/// Returns `Err(message)` on hard structural failure (duplicate field +/// name; collision between a user-named field and a synthetic positional +/// name). Returns `Ok(None)` only when this isn't an anon-NT type tuple +/// at all (so the caller can ignore it). +pub(crate) fn type_shape( + namer: &mut Namer, + tuple: &ruff_python_ast::ExprTuple, +) -> Result, String> { + if !tuple.is_anon_named_tuple { + return Ok(None); + } + let mut fields = Vec::with_capacity(tuple.elts.len()); + for (i, elt) in tuple.elts.iter().enumerate() { + let (name, type_source) = match elt { + Expr::Named(named) => { + let Expr::Name(name_expr) = named.target.as_ref() else { + return Err(format!( + "anonymous named tuple field name must be an identifier, got `{}`", + namer.src(named.target.range()) + )); + }; + let name = name_expr.id.as_str().to_owned(); + (name, field_type(namer, &named.value)) + } + other => (synth_pos_name(i), field_type(namer, other)), + }; + fields.push((name, type_source)); + } + if let Some(dup) = first_duplicate_name(&fields) { + return Err(format!( + "anonymous named tuple `{}` has duplicate field name `{}` \ + (positional fields are assigned synthetic names `arg0`, `arg1`, …; \ + rename the colliding field)", + namer.src(tuple.range()), + dup, + )); + } + Ok(Some(Shape { fields })) +} + +/// A field's type as python source. +/// +/// The class body is hoisted to module scope, ahead of everything the module binds, the generics +/// polyfill's declarations among them, so a field naming one of those — a type parameter read +/// by the name the polyfill declares it under, a class defined further down — is a forward +/// reference there, and the annotation is written as a string. +fn field_type(namer: &mut Namer, expr: &Expr) -> String { + let lowered = namer.lower(expr); + let renamed = namer.read_type_parameters(expr.range(), &lowered); + let forward = renamed != lowered || reads_a_module_binding(namer.types(), expr); + if forward && !lowered.is_empty() { + namer.quotes.quote(&renamed) + } else { + renamed + } +} + +/// whether `expr` reads a name the module binds +fn reads_a_module_binding(types: &dyn TypeInfo, expr: &Expr) -> bool { + struct Finder<'a> { + types: &'a dyn TypeInfo, + found: bool, + } + impl<'ast> Visitor<'ast> for Finder<'_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + if let Expr::Name(name) = expr + && self.types.is_bound_globally(&name.id) + { + self.found = true; + } + walk_expr(self, expr); + } + } + let mut finder = Finder { + types, + found: false, + }; + finder.visit_expr(expr); + finder.found +} + pub(crate) struct AnonNamedTuple<'src> { source: &'src str, written: WrittenNames<'src>, @@ -103,23 +190,18 @@ pub(crate) struct AnonNamedTuple<'src> { quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, - /// One lowerer drives every field type in the run, so its imports and the - /// classes it synthesizes can be collected at the end. A hoisted class body - /// leaves the statement the field type was written in, so no sibling - /// transform's edit reaches it — the field type has to be lowered here or - /// the surface syntax lands in the output verbatim - pub(crate) callable: CallableSyntax<'src>, + /// every type form's class, named together with the other kinds of synthesized type, whose + /// shapes a value form or a coerced tuple of the same shape reuses + synthesized: std::rc::Rc, pub(crate) edits: Vec, /// Insertion-ordered map of shape → synthesized class name. Visitor /// processes nested anonymous-named-tuple expressions before their /// containing expression (post-order), so a class is defined before /// any later class that references it as a field type shapes: indexmap::IndexMap, - /// Source range of every anonymous-named-tuple expression we've assigned - /// a synthesized class to (both top-level rewrites and nested ones inside - /// other anon-NT field types). Used when rendering a class body to - /// substitute inner anon-NT source spans with their class names so the - /// emitted `NamedTuple` definition is valid Python. + /// Source range of every synthesized type, of whichever kind, and of every value form, + /// with the class it names. A value form's arguments are copied from source, so a type + /// among them is written as its class. /// /// Stored as a `Vec` (`TextRange` isn't `Ord`) and walked linearly when /// rendering. Inserts append; `render_subbed_value` skips entries that don't @@ -142,19 +224,7 @@ pub(crate) struct AnonNamedTuple<'src> { /// The imports of the `typing` names a field type reads, collected the same way typing_imports: std::cell::RefCell>, /// whether python evaluates the synthesized class's field annotations as it is made - annotations_evaluated: bool, - /// Stack of typevar rename maps (`T` → `_T`) accumulated as the visitor - /// descends into PEP 695 generics. Synthesized `NamedTuple` class bodies - /// are hoisted to module scope, but the original source text references - /// the *unmangled* typevar names. Applying these renames at field-type - /// rendering keeps the synthesized class's field annotations pointing at - /// the same module-level `_T = TypeVar("_T")` that the generics polyfill - /// emits, instead of leaving an unbound `T`. - /// - /// A list kept as native PEP 695 syntax isn't renamed, so it pushes no frame. - typevar_rename_stack: Vec>, - /// the name the generics polyfill declares each type parameter under - polyfilled: PolyfilledTypeParams, + pub(crate) annotations_evaluated: bool, /// the class emitted for each anonymous named tuple the checker identifies, so /// that every type form it reads as one type, and every plain tuple literal it /// reads as an instance of it, names one class @@ -177,125 +247,67 @@ impl<'src> AnonNamedTuple<'src> { quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: crate::Config, - stmts: &[Stmt], + synthesized: std::rc::Rc, ) -> Self { Self { - polyfilled: PolyfilledTypeParams::decide(source, written, stmts, &config), source, written, quotes, types, - callable: CallableSyntax::new(source, written, quotes, &config).with_types(types), config, edits: Vec::new(), - shapes: indexmap::IndexMap::new(), - range_to_class: Vec::new(), + shapes: synthesized.named_tuples().clone(), + // every synthesized type, of whichever kind, is written as its class wherever source + // text this copies into the output holds one + range_to_class: synthesized.substitutions().collect(), + identity_classes: synthesized.named_tuple_identities().collect(), + synthesized, range_to_value_render: Vec::new(), needs_import: false, type_only_imports: std::cell::RefCell::default(), typing_imports: std::cell::RefCell::default(), annotations_evaluated: false, - typevar_rename_stack: Vec::new(), - identity_classes: HashMap::new(), constructions: Vec::new(), errors: Vec::new(), } } - /// Returns the synthesized class definitions to prepend to the module body, - /// in deterministic order. Each shape's field types are rendered from the - /// original source text with any nested anonymous-named-tuple regions - /// substituted with their synthesized class names so the emitted - /// `NamedTuple` definitions are valid Python. + /// Returns the class definitions of the shapes only a value form or a coerced tuple has, in + /// deterministic order: the synthesized type namer declares every type form's class. pub(crate) fn class_defs(&self) -> String { let mut out = String::new(); let named_tuple = self.written.imported("typing", "NamedTuple"); for (shape, name) in &self.shapes { + if self.synthesized.named_tuples().contains_key(shape) { + continue; + } out.push_str(&shape.class_def(name, &named_tuple)); out.push('\n'); } out } - fn src(&self, range: ruff_text_size::TextRange) -> &str { - &self.source[usize::from(range.start())..usize::from(range.end())] - } - - /// Lower a field's type expression to python source. - /// - /// The synthesized class body is hoisted out of the statement the field type - /// was written in, so a sibling transform's edit never reaches it. Driving - /// the shared type-expression lowerer here is what keeps `dynamic`, `T?`, - /// `typeof x`, a callable arrow and a nested anonymous named tuple from - /// reaching the output as surface syntax. - fn render_field_type(&mut self, expr: &Expr) -> String { - let lowered = self.callable.lower_type_expr(expr); - let renamed = self.apply_typevar_renames(&lowered); - // the class is hoisted ahead of everything the module binds, a renamed typevar - // among them, so a field naming one is a forward reference there and is quoted - // as one - let forward = renamed != lowered || self.reads_a_module_binding(expr); - if forward && !lowered.is_empty() { - self.quotes.quote(&renamed) + /// `annotation`, a field's type as the class declares it, written as a string where it reads + /// a name the module binds: the class is declared ahead of everything the module binds, so + /// such a name is not bound yet where the class's field annotations are read, as it is made + fn declared_field(&self, annotation: String) -> String { + let reads_a_module_binding = crate::type_info::names_read(&annotation) + .is_some_and(|names| names.iter().any(|name| self.types.is_bound_globally(name))); + if reads_a_module_binding { + self.quotes.quote(&annotation) } else { - renamed + annotation } } - /// whether `expr` reads a name the module binds - fn reads_a_module_binding(&self, expr: &Expr) -> bool { - struct Finder<'a> { - types: &'a dyn TypeInfo, - found: bool, - } - impl<'ast> Visitor<'ast> for Finder<'_> { - fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Name(name) = expr - && self.types.is_bound_globally(&name.id) - { - self.found = true; - } - walk_expr(self, expr); - } - } - let mut finder = Finder { - types: self.types, - found: false, - }; - finder.visit_expr(expr); - finder.found - } - - /// Active typevar rename map, flattened from the stack with inner scopes - /// shadowing outer ones. Empty when not inside a generic scope or when - /// `min_version` is 3.12+ (where PEP 695 stays native and no renames - /// are emitted). - fn active_typevar_renames(&self) -> HashMap { - let mut out = HashMap::new(); - for frame in &self.typevar_rename_stack { - out.extend(frame.clone()); - } - out - } - - /// Substitute every word-boundary occurrence of a typevar name in - /// `text` with its mangled module-scope counterpart. Operates on the - /// rendered field-type string so the synthesized `NamedTuple` class - /// references the renamed `_T` instead of the original `T`. - fn apply_typevar_renames(&self, text: &str) -> String { - let renames = self.active_typevar_renames(); - let borrowed: HashMap<&str, &str> = renames - .iter() - .map(|(from, to)| (from.as_str(), to.as_str())) - .collect(); - super::source_util::rename_identifiers(text, &borrowed) + fn src(&self, range: ruff_text_size::TextRange) -> &str { + &self.source[usize::from(range.start())..usize::from(range.end())] } - /// Render `range`'s source with every anonymous named tuple inside it - /// substituted: a value form by its constructor call, a type form by its - /// class name. Used when an outer anon-NT *value* form is rendering its - /// argument list — a nested value form there must call its inner - /// constructor, not just reference the inner class. + /// Render `range`'s source with each value form inside it written as its constructor call, + /// and each synthesized type as its class. Used when an outer anon-NT *value* form + /// is rendering its argument list — a nested value form there must call + /// its inner constructor, not just reference the inner class. fn render_subbed_value(&self, range: ruff_text_size::TextRange) -> String { // Combine value-form constructor calls (preferred) with class-name // subs for any nested *type* form in the same range. Value renders @@ -339,54 +351,8 @@ impl<'src> AnonNamedTuple<'src> { out } - /// Build a shape from a type-form anonymous named tuple - /// (`(name: T, ...)`). Field types come from source text. Positional - /// fields (bare type expressions like `int` in `(int, name: str)`) are - /// assigned synthetic names `arg0`, `arg1`, … `NamedTuple` disallows - /// leading-underscore field names, so `argN` is used. - /// - /// Returns `Err(message)` on hard structural failure (duplicate field - /// name; collision between a user-named field and a synthetic positional - /// name). Returns `Ok(None)` only when this isn't an anon-NT type tuple - /// at all (so the caller can ignore it). - fn extract_type_shape( - &mut self, - tuple: &ruff_python_ast::ExprTuple, - ) -> Result, String> { - if !tuple.is_anon_named_tuple { - return Ok(None); - } - let mut fields = Vec::with_capacity(tuple.elts.len()); - for (i, elt) in tuple.elts.iter().enumerate() { - let (name, type_source) = match elt { - Expr::Named(named) => { - let Expr::Name(name_expr) = named.target.as_ref() else { - return Err(format!( - "anonymous named tuple field name must be an identifier, got `{}`", - self.src(named.target.range()) - )); - }; - let name = name_expr.id.as_str().to_owned(); - (name, self.render_field_type(&named.value)) - } - other => (synth_pos_name(i), self.render_field_type(other)), - }; - fields.push((name, type_source)); - } - if let Some(dup) = first_duplicate_name(&fields) { - return Err(format!( - "anonymous named tuple `{}` has duplicate field name `{}` \ - (positional fields are assigned synthetic names `arg0`, `arg1`, …; \ - rename the colliding field)", - self.src(tuple.range()), - dup, - )); - } - Ok(Some(Shape { fields })) - } - /// Build a shape from a value-form anonymous named tuple - /// (`(name=v, ...)`). Same conventions as `extract_type_shape`; field + /// (`(name=v, ...)`). Same conventions as [`type_shape`]; field /// types come from ty's promoted (literal-stripped) inference. /// /// Returns `Err(message)` on hard structural failure, and on a field whose value has no @@ -448,7 +414,7 @@ impl<'src> AnonNamedTuple<'src> { self.typing_imports .borrow_mut() .extend(d.typing_imports_in(self.written)); - d.annotation_in(self.written, self.annotations_evaluated) + self.declared_field(d.annotation_in(self.written, self.annotations_evaluated)) }; fields.push((name, type_display)); } @@ -464,42 +430,25 @@ impl<'src> AnonNamedTuple<'src> { Ok(Some(Shape { fields })) } - fn class_name_for( - &mut self, - shape: Shape, - source_range: ruff_text_size::TextRange, - id: Option, - ) -> String { - let name = if let Some(existing) = id.and_then(|id| self.identity_classes.get(&id)) { - existing.clone() - } else { - let name = self.shape_class_name(shape); - if let Some(id) = id { - self.identity_classes.insert(id, name.clone()); - } - name - }; - // Track this anon-NT's source range → class name so any *outer* anon-NT + fn class_name_for(&mut self, shape: Shape, source_range: ruff_text_size::TextRange) -> String { + let name = self.shape_class_name(shape); + // Track this anon-NT's source range → class name so any *outer* value form // that contains this one can substitute its source span when rendering - // its own field types in the synthesized class body. the lowerer needs - // the same table so an outer field type resolves a nested anon-NT at any - // depth, not only when it is the whole field type + // its own arguments self.range_to_class.push((source_range, name.clone())); - self.callable.add_substitution(source_range, name.clone()); name } + /// a type form is replaced with the class the synthesized type namer gave it, which also + /// reported any shape that could not be declared fn rewrite_type_form(&mut self, tuple: &ruff_python_ast::ExprTuple) { - let shape = match self.extract_type_shape(tuple) { - Ok(Some(shape)) => shape, - Ok(None) => return, - Err(msg) => { - self.errors.push(msg); - return; - } + let Some(class_name) = self + .synthesized + .class_at(tuple.range(), SynthesizedKind::NamedTuple) + .map(str::to_owned) + else { + return; }; - let id = self.types.anon_named_tuple_id(tuple); - let class_name = self.class_name_for(shape, tuple.range(), id); self.needs_import = true; self.edits.push(Fix::safe_edit(Edit::range_replacement( class_name, @@ -528,9 +477,7 @@ impl<'src> AnonNamedTuple<'src> { }; value_args.push(self.render_subbed_value(value_expr.range())); } - // a value form is left out of the checker's identities: its class is the one - // its own field values make, which a type form's need not be - let class_name = self.class_name_for(shape, tuple.range(), None); + let class_name = self.class_name_for(shape, tuple.range()); self.needs_import = true; let constructor_call = format!("{class_name}({})", value_args.join(", ")); self.range_to_value_render @@ -541,31 +488,6 @@ impl<'src> AnonNamedTuple<'src> { ))); } - /// Push the typevar rename frame the generics polyfill gives a `[T, U, ...]` - /// type-parameter list. Returns whether a frame was pushed so the caller - /// knows to pop. No-op (returns false) for a list kept as native syntax - fn push_typevar_scope_from(&mut self, tp: Option<&ruff_python_ast::TypeParams>) -> bool { - let Some(frame) = tp.and_then(|tp| self.polyfilled.hoisted_renames(tp)) else { - return false; - }; - if frame.is_empty() { - return false; - } - self.typevar_rename_stack.push(frame); - true - } - - /// Look up the synthesized class name for a nested anon-NT value tuple - /// that's already been processed. Relies on the post-order visit: - /// children are rewritten first, so by the time the outer extracts its - /// shape, the inner's class name is already in `range_to_class`. - fn shape_for_value_tuple(&self, inner: &ruff_python_ast::ExprTuple) -> Option { - self.range_to_class - .iter() - .find(|(r, _)| *r == inner.range()) - .map(|(_, n)| n.clone()) - } - /// the name of the class declaring `shape`, one class per shape fn shape_class_name(&mut self, shape: Shape) -> String { if let Some(existing) = self.shapes.get(&shape) { @@ -632,7 +554,9 @@ impl<'src> AnonNamedTuple<'src> { self.typing_imports .borrow_mut() .extend(spelled.typing_imports_in(self.written)); - spelled.annotation_in(self.written, self.annotations_evaluated) + self.declared_field( + spelled.annotation_in(self.written, self.annotations_evaluated), + ) } AnonNamedTupleField::NamedTuple(nested) => self.construction_class(nested)?, AnonNamedTupleField::Unspellable => return Err(name.clone()), @@ -643,6 +567,17 @@ impl<'src> AnonNamedTuple<'src> { self.identity_classes.insert(target.id, class_name.clone()); Ok(class_name) } + + /// Look up the synthesized class name for a nested anon-NT value tuple + /// that's already been processed. Relies on the post-order visit: + /// children are rewritten first, so by the time the outer extracts its + /// shape, the inner's class name is already in `range_to_class`. + fn shape_for_value_tuple(&self, inner: &ruff_python_ast::ExprTuple) -> Option { + self.range_to_class + .iter() + .find(|(r, _)| *r == inner.range()) + .map(|(_, n)| n.clone()) + } } /// a plain tuple literal to build as an anonymous named tuple @@ -653,19 +588,6 @@ struct PendingConstruction { } impl<'ast> Visitor<'ast> for AnonNamedTuple<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - let type_params = match stmt { - Stmt::FunctionDef(func) => func.type_params.as_deref(), - Stmt::ClassDef(cls) => cls.type_params.as_deref(), - _ => None, - }; - let pushed = self.push_typevar_scope_from(type_params); - walk_stmt(self, stmt); - if pushed { - self.typevar_rename_stack.pop(); - } - } - fn visit_expr(&mut self, expr: &'ast Expr) { if let Expr::Tuple(tuple) = expr { if tuple.is_anon_named_tuple || tuple.is_anon_named_tuple_value { @@ -723,18 +645,10 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { Some(super::ast_driver::Lowering::AnonNamedTuple) } - /// a named tuple type it replaces moves into a class, whose field types the shared - /// type-expression lowerer prints, a `typeof` among them + /// a named tuple type it replaces moves into a class, whose field types the synthesized + /// type namer prints fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[ - super::ast_driver::Lowering::Callable, - super::ast_driver::Lowering::OptionalType, - super::ast_driver::Lowering::LiteralType, - super::ast_driver::Lowering::JustFloat, - super::ast_driver::Lowering::DynamicKeyword, - super::ast_driver::Lowering::FloatConst, - super::ast_driver::Lowering::Typeof, - ] + super::synthesized_types::CLASS_BODY } fn run( @@ -749,7 +663,7 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { self.quotes, types, self.config.clone(), - stmts, + ctx.synthesized.clone(), ); inner.annotations_evaluated = ctx.annotations_evaluated; for stmt in stmts { @@ -762,18 +676,13 @@ impl super::ast_driver::TypeAwarePass for AnonNamedTuplePass<'_> { if inner.needs_import { ctx.required_imports .push(self.written.import_from("typing", &["NamedTuple"])); - // the field lowerer's own classes come first: a field type may name - // one. they go in as a single entry so the sort below cannot part - // them from the classes that reference them - let defs = format!("{}{}", inner.callable.class_defs(), inner.class_defs()); + // one entry, so the sort of the imports cannot part the classes from each other + let defs = inner.class_defs(); if !defs.is_empty() { let trimmed = defs.trim_end_matches('\n'); ctx.required_imports.push(format!("{trimmed}\n")); } } - let (imports, helpers) = inner.callable.take_requirements(); - ctx.required_imports.extend(imports); - ctx.runtime.extend(helpers); ctx.type_only_imports.extend(inner.type_only_imports.take()); ctx.required_imports.extend(inner.typing_imports.take()); for fix in std::mem::take(&mut inner.edits) { @@ -888,6 +797,14 @@ mod tests { ); } + /// a type alias's own parameter is declared by the polyfill below the class, so the field + /// reads it by that name, as a forward reference + #[test] + fn a_field_of_a_generic_alias_reads_the_declared_name() { + let out = transpile("type H[T] = (p: T, q: int)\n", &Config::test_default()).unwrap(); + assert!(out.contains(" p: \"_T\"\n q: int\n"), "got:\n{out}"); + } + /// the synthesized class body leaves the statement the field type was /// written in, so nothing else lowers it — `dynamic` reaching the output /// verbatim is a `NameError` at import, and a wrapped type is a syntax error @@ -906,6 +823,34 @@ mod tests { ); } + /// the class is declared ahead of everything the module binds, so a field reading a class + /// or an import of the module, wherever it is bound, is written as a string — a value form's + /// field as a type form's. a builtin is bound there + #[test] + fn a_field_naming_a_binding_of_the_module_is_quoted() { + let out = transpile( + indoc! {" + from fractions import Fraction + + class Earlier: ... + + def f(p: (a: Earlier, b: Later, c: Fraction, d: int)) -> None: ... + + v = (e=Earlier(), g=Fraction(1), n=1) + + class Later: ... + "}, + &Config::test_default(), + ) + .unwrap(); + for field in [ + " a: \"Earlier\"\n b: \"Later\"\n c: \"Fraction\"\n d: int\n", + " e: \"Earlier\"\n g: \"Fraction\"\n n: int\n", + ] { + assert!(out.contains(field), "expected {field:?}, got: {out}"); + } + } + /// `typeof` is lowered by an AST pass, inside the construct this pass replaces /// whole — the cleanup run then lowers what is left in the output, and must /// not emit the class a second time @@ -1538,6 +1483,57 @@ mod tests { assert_eq!(occurrences, 1, "expected single class def, got: {out}"); } + /// a plain tuple the checker took for an anonymous named tuple is one wherever it stands: a + /// call's argument, a parameter's default, an assignment to a declared name, a dict's value, + /// under a union. left a plain tuple, `x.name` is an `AttributeError` on a file the checker + /// passed + #[test] + fn a_plain_tuple_is_coerced_wherever_the_checker_expects_the_named_tuple() { + let out = transpile( + indoc! {" + def foo(x: (name: str, age: int) = (\"d\", 0)) -> None: ... + foo((\"asdf\", 1)) + foo(x=(\"b\", 2)) + y: (name: str, age: int) + y = (\"c\", 3) + z: dict[str, (name: str, age: int)] = {\"k\": (\"e\", 4)} + w: (name: str, age: int) | None = (\"f\", 5) + "}, + &Config::test_default(), + ) + .unwrap(); + for coerced in [ + "x = _AnonNamedTuple_7bfb4772(\"d\", 0)", + "foo(_AnonNamedTuple_7bfb4772(\"asdf\", 1))", + "foo(x=_AnonNamedTuple_7bfb4772(\"b\", 2))", + "y = _AnonNamedTuple_7bfb4772(\"c\", 3)", + "{\"k\": _AnonNamedTuple_7bfb4772(\"e\", 4)}", + "= _AnonNamedTuple_7bfb4772(\"f\", 5)", + ] { + assert!(out.contains(coerced), "{coerced} in:\n{out}"); + } + } + + /// an assignment's target is no value, whatever the value assigned to it is + #[test] + fn a_tuple_target_is_not_coerced() { + let out = transpile( + indoc! {" + def f(p: (a: int, b: int)) -> int: + (x, y) = p + for (u, v) in [p]: + pass + return x + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains(" (x, y) = p\n") && out.contains(" for (u, v) in [p]:\n"), + "got: {out}" + ); + } + #[test] fn a_plain_tuple_argument_is_built_as_its_parameter() { let out = transpile( @@ -1573,6 +1569,20 @@ mod tests { assert!(out.contains("take(_AnonNamedTuple_"), "got: {out}"); } + /// a plain tuple the checker did not take for an anonymous named tuple stays one + #[test] + fn a_plain_tuple_the_checker_does_not_coerce_stays_plain() { + let out = transpile( + indoc! {" + def foo(x: tuple[str, int]) -> None: ... + foo((\"asdf\", 1)) + "}, + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains("foo((\"asdf\", 1))"), "got: {out}"); + } + #[test] fn value_form_construction() { let out = transpile("a = (name=\"asdf\", age=20)\n", &Config::test_default()).unwrap(); diff --git a/crates/by_transforms/src/transforms/ast_driver.rs b/crates/by_transforms/src/transforms/ast_driver.rs index 8a86698bf7..7001cc1073 100644 --- a/crates/by_transforms/src/transforms/ast_driver.rs +++ b/crates/by_transforms/src/transforms/ast_driver.rs @@ -48,9 +48,10 @@ use super::{ overload, parametric_is, postfix_await, propagate, properties, protocol_type, raises_clause, reified_class, reified_generic, repeated_underscore, return_value_use, runtime_union, sentinel, some_ctor, soundness, statement_expression, static_resource, string_tag, super_keyword, - symbolic_type_op, template_expression, template_type, top_star, trailing_lambda, tuple_index, - type_fn, type_is, type_reification, typed_dict_literal, typed_lambda, typeof_keyword, - unique_loop_bindings, unpack, use_site_variance, visibility_rename, + symbolic_type_op, synthesized_types, template_expression, template_type, top_star, + trailing_lambda, tuple_index, type_fn, type_is, type_reification, typed_dict_literal, + typed_lambda, typeof_keyword, unique_loop_bindings, unpack, use_site_variance, + visibility_rename, }; use crate::Config; use crate::source_map::Replacement; @@ -210,11 +211,16 @@ pub(crate) struct PassContext { /// type-expression lowerer rewrites as part of that form. Every other `typeof` /// is lowered in the syntax tree, and only there pub(crate) structural_typeof_ranges: Vec, - /// The same operations as `(range, rendered)` pairs. A pass that replaces a - /// whole statement subsumes any fold inside it — skipping is not enough, the - /// rendered text has to be spliced into the replacement or the operation is - /// re-emitted from source and reaches the runtime - pub(crate) symbolic_substitutions: Vec<(TextRange, String)>, + /// Every type expression already lowered to a fixed spelling, as `(range, rendered)` + /// pairs: the operations above, and each type the + /// [synthesized type namer](super::synthesized_types) gave a class. A pass that replaces a + /// type or a statement whole subsumes any of them inside it — skipping is not enough, the + /// rendered text has to be spliced into the replacement or the source is re-emitted and + /// reaches the runtime + pub(crate) type_substitutions: Vec<(TextRange, String)>, + /// The class each synthesized type of the module names, named in one walk so that each + /// kind can name the others inside it + pub(crate) synthesized: std::rc::Rc, /// where python refuses an assignment expression in the module the type-aware /// passes walk, which every pass binding a temporary asks pub(crate) walrus_regions: super::walrus_scope::WalrusRegions, @@ -236,6 +242,8 @@ pub(crate) enum Lowering { /// the whitespace the keyword blanking collapses. only spacing is at stake, so any /// edit it lands inside may leave it out KeywordPadding, + /// `literal str`, written `LiteralString` + LiteralString, LiteralType, LocalOnce, MatchType, @@ -1329,6 +1337,10 @@ pub(crate) fn run_against_source<'a>( config.min_version >= ruff_python_ast::PythonVersion::PY38, ); let written = written.with_lowerings(&underscore_lowerings); + ctx.errors.extend(repeated_underscore::refused_reads( + parsed_handle.suite(), + &semantic_model, + )); // resolve symbolic operations in type positions (`1 + 1` → `Literal[2]`) // up front, from the original parse where `typeof` operands are still @@ -1358,9 +1370,40 @@ pub(crate) fn run_against_source<'a>( ); let symbolic_imports = std::mem::take(&mut symbolic_folds.imports); ctx.claimed_type_op_ranges = symbolic_folds.claimed_ranges(); - ctx.symbolic_substitutions = symbolic_folds.substitutions(); + ctx.type_substitutions = symbolic_folds.substitutions(); + ctx.type_substitutions + .extend(literal_string_rewrites.substitutions()); let symbolic_pass = symbolic_type_op::SymbolicTypeOp::new(symbolic_folds); + // every class a type expression lowers to — a typed dict literal, an anonymous named tuple, + // an inline protocol — is named in one walk, innermost first, since each kind declares the + // others nested inside it as its field types. the passes lowering each kind write the names, + // and so does every other lowering that prints a type with one inside it + let synthesized = synthesized_types::SynthesizedTypes::name_all( + source_ref, + written, + quotes, + &semantic_model, + config, + parsed_handle.suite(), + &ctx.type_substitutions, + &ctx.claimed_type_op_ranges, + ); + ctx.type_substitutions.extend(synthesized.substitutions()); + ctx.required_imports + .extend(synthesized.imports.iter().cloned()); + if !synthesized.declarations.is_empty() { + // one entry, so the sort of the imports cannot part a class from the ones it reads. one + // trailing newline is kept, for the blank line between the classes and the module + ctx.required_imports.push(format!( + "{}\n", + synthesized.declarations.trim_end_matches('\n') + )); + } + ctx.runtime.extend(synthesized.helpers.iter().copied()); + ctx.errors.extend(synthesized.errors.iter().cloned()); + ctx.synthesized = std::rc::Rc::new(synthesized); + // a `typeof` nested under a structural type-form (`&` / `or` / `not` / an // arrow) belongs to the type-expression lowerer's wide edit; the fold // skips those so its statement re-render doesn't drop that edit @@ -1416,7 +1459,7 @@ pub(crate) fn run_against_source<'a>( let top_star_pass = top_star::TopStar::new(written); let identity_swap_pass = identity_swap::IdentitySwap::new(source_ref); let compat_pass = compat::CompatRewrite::new(source_ref, config.clone()); - let string_tag_pass = string_tag::StringTagPass::new(source_ref, config.clone()); + let string_tag_pass = string_tag::StringTagPass::new(source_ref, &comments, config.clone()); let dedent_string_pass = dedent_string::DedentString::new(source_ref); let super_keyword_pass = super_keyword::SuperKeyword::new(written); let postfix_await_pass = postfix_await::PostfixAwait::new(source_ref); @@ -1472,14 +1515,7 @@ pub(crate) fn run_against_source<'a>( config.clone(), unpack::collect_type_subscripts(parsed_handle.suite(), &semantic_model), ); - let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass::new( - source_ref, - written, - quotes, - config.clone(), - parsed_handle.suite(), - &semantic_model, - ); + let typed_dict_literal_pass = typed_dict_literal::TypedDictLiteralPass; let just_float_pass = just_float::JustFloatPass::new(written); let float_const_pass = float_const::FloatConstPass::new(); let kw_subscript_pass = @@ -1507,8 +1543,7 @@ pub(crate) fn run_against_source<'a>( literal_types::LiteralTypePass::new(source_ref, written, config.float_literals); let callable_pass = callable::CallableSyntaxPass::new(source_ref, written, quotes, config.clone()); - let protocol_type_pass = - protocol_type::ProtocolTypePass::new(source_ref, written, quotes, config.clone()); + let protocol_type_pass = protocol_type::ProtocolTypePass; let coalesce_text_pass = coalesce::NoneCoalescePass::new(source_ref, &comments); let force_unwrap_pass = force_unwrap::ForceUnwrapPass::new(source_ref); let flexible_keyword_pass = flexible_keyword::FlexibleKeywordPass; @@ -1884,7 +1919,10 @@ pub(crate) fn run_against_source<'a>( for (range, replacement) in literal_string_rewrites.edits() { ctx.text_edits.push((range, replacement)); } - authorship.finished(Author::driver("literal string", None), &ctx); + authorship.finished( + Author::driver("literal string", Some(Lowering::LiteralString)), + &ctx, + ); if literal_string_rewrites.needs_import { ctx.required_imports .push(written.import_from("typing", &["LiteralString"])); @@ -1952,6 +1990,7 @@ pub(crate) fn run_against_source<'a>( // reason the `=` fields above are let template_wraps = template_expression::claim( source_ref, + &comments, parsed_handle.suite(), &rewriting, &fields_replaced_in_tree, @@ -2605,16 +2644,6 @@ mod subsumed_lowerings { /// the declared pairs shown written somewhere other than a case here, by the declaring /// pass's name: the test of the pass's own that shows it, or why it holds const SHOWN_ELSEWHERE: &[(&str, Lowering, &str)] = &[ - ( - "TupleLiteralTypePass", - Lowering::SymbolicTypeOp, - "nested_in_subscript", - ), - ( - "CallableSyntaxPass", - Lowering::SymbolicTypeOp, - "tuple_repetition_unpacked_in_tuple_type", - ), ( "GenericPolyfillPass", Lowering::MatchType, @@ -2727,6 +2756,13 @@ mod subsumed_lowerings { Lowering::OptionalType, "an_optional_target_tests_for_none_too", ), + ( + "TypedDictLiteralPass", + Lowering::LiteralString, + "by construction: a dict literal type's values are not read as type expressions, so \ + no `literal` modifier is parsed in one, and the class body prints its fields through \ + the same lowerer as the other synthesized types", + ), // the runtime-union lowering runs only below 3.10, where the cases above never go ( "CallableSyntaxPass", @@ -2782,6 +2818,18 @@ mod subsumed_lowerings { Some("(int) ->"), ), (Lowering::Typeof, "typeof y", "TypeOf[y]", Some("typeof")), + ( + Lowering::SymbolicTypeOp, + "1 + 1", + "Literal[2]", + Some("1 + 1"), + ), + ( + Lowering::LiteralString, + "literal str", + "LiteralString", + Some("literal "), + ), ]; leaves .into_iter() @@ -2847,6 +2895,8 @@ mod subsumed_lowerings { Lowering::DynamicKeyword, Lowering::FloatConst, Lowering::Typeof, + Lowering::SymbolicTypeOp, + Lowering::LiteralString, ]; // the class is hoisted ahead of the module's own bindings, so a field reading one — // the `y` a `typeof` names — is written as a forward reference @@ -2877,6 +2927,25 @@ mod subsumed_lowerings { |leaf| format!("def f(x: {{\"a\": {leaf}}}) -> None: ..."), |lowered| format!("a: \"{lowered}\""), )); + // a static cast keeps its target as written, so the target's own lowering passes through + // it. a checked one writes a check in its place, which stands for the lowered type too + cases.extend(leaves( + "CheckedCastPass", + super::super::callable::TYPE_EXPRESSION, + |leaf| format!("def f(x: object) -> None:\n z = x cast {leaf}"), + |lowered| format!("z = cast({lowered}, x)"), + )); + cases.push(case( + "CheckedCastPass", + &[ + Lowering::AnonNamedTuple, + Lowering::ProtocolType, + Lowering::TypedDictLiteral, + ], + "def f(x: object) -> None:\n a = x cast (p: int)\n b = x cast protocol(n: int)\n c = x cast {\"k\": int}\n", + &["a = cast(_AnonNamedTuple_", "b = cast(_Protocol_", "c = cast(_TypedDict_"], + &["(p: int", "protocol(n", "{\"k\""], + )); cases.push(case( "CallableSyntaxPass", &[Lowering::ProtocolType], @@ -2884,6 +2953,57 @@ mod subsumed_lowerings { &["Callable[[_Protocol_"], &["protocol("], )); + // each kind of synthesized type, nested in a construct that writes a type whole, is + // written as its class there + let synthesized = [ + Lowering::AnonNamedTuple, + Lowering::ProtocolType, + Lowering::TypedDictLiteral, + ]; + let unwritten = ["(p: int", "protocol(n", "{\"k\""]; + cases.push(case( + "TupleLiteralTypePass", + &synthesized, + "def f(x: (int, (p: int), protocol(n: int), {\"k\": int})) -> None: ...\n", + &[ + "tuple[int, _AnonNamedTuple_", + ", _Protocol_", + ", _TypedDict_", + ], + &unwritten, + )); + cases.push(case( + "CallableSyntaxPass", + &[Lowering::AnonNamedTuple, Lowering::TypedDictLiteral], + "def f(x: ((p: int), {\"k\": int}) -> str) -> None: ...\n", + &["Callable[[_AnonNamedTuple_", ", _TypedDict_"], + &unwritten, + )); + cases.push(case( + "AnonNamedTuplePass", + &synthesized, + "def f(x: (a: (p: int), b: protocol(n: int), c: {\"k\": int})) -> None: ...\n", + &["a: _AnonNamedTuple_", "b: _Protocol_", "c: _TypedDict_"], + &unwritten, + )); + cases.push(case( + "ProtocolTypePass", + &synthesized, + "def f(x: protocol(a: (p: int); b: protocol(n: int); c: {\"k\": int})) -> None: ...\n", + &[ + "a: \"_AnonNamedTuple_", + "b: \"_Protocol_", + "c: \"_TypedDict_", + ], + &unwritten, + )); + cases.push(case( + "TypedDictLiteralPass", + &synthesized, + "def f(x: {\"a\": (p: int), \"b\": protocol(n: int), \"c\": {\"k\": int}}) -> None: ...\n", + &["a: \"_AnonNamedTuple_", "b: \"_Protocol_", "c: \"_TypedDict_"], + &unwritten, + )); cases } @@ -2903,6 +3023,13 @@ mod subsumed_lowerings { &["TypeAliasType(\"Point\", _AnonNamedTuple_"], &["(x: int"], ), + case( + "GenericPolyfillPass", + &[Lowering::LiteralString], + "class Box[T: literal str, U in (literal str, int)]: ...\n", + &["bound=LiteralString", "TypeVar(\"_U\", LiteralString, int)"], + &["literal "], + ), case( "GenericPolyfillPass", &[Lowering::VarianceStrip], diff --git a/crates/by_transforms/src/transforms/callable.rs b/crates/by_transforms/src/transforms/callable.rs index 80cd6ea3a6..a449688814 100644 --- a/crates/by_transforms/src/transforms/callable.rs +++ b/crates/by_transforms/src/transforms/callable.rs @@ -24,8 +24,9 @@ use std::hash::Hash; use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::helpers::{ - ReturnGuardForm, callable_parameter_label, is_classvar_marker_id, is_final_marker_id, - is_let_marker_id, predicate_guard, + ReturnGuardForm, any_over_expr, callable_parameter_label, is_classvar_marker_id, + is_final_marker_id, is_let_marker_id, predicate_guard, type_modifier_marker, + use_site_variance_marker, }; use ruff_python_ast::{Expr, ExprCallableType, Operator, PythonVersion, Stmt, UnaryOp}; use ruff_text_size::{Ranged, TextRange}; @@ -40,8 +41,9 @@ use crate::config::{Config, FloatLiteralLowering}; use crate::type_info::{TypeInfo, UnpackedKwargsLowering}; /// the lowerings [`CallableSyntax::lower_type_expr`] writes itself, wherever it prints a type -/// expression, a symbolic fold among them when it is handed the fold as a substitution. a -/// pass that prints one through it in place of the source writes these too +/// expression, a symbolic fold and a synthesized type's class among them when it is handed the +/// [`type_substitutions`](super::ast_driver::PassContext::type_substitutions). a pass that +/// prints one through it in place of the source writes these too pub(crate) const TYPE_EXPRESSION: &[Lowering] = &[ Lowering::Callable, Lowering::OptionalType, @@ -50,6 +52,10 @@ pub(crate) const TYPE_EXPRESSION: &[Lowering] = &[ Lowering::DynamicKeyword, Lowering::FloatConst, Lowering::SymbolicTypeOp, + Lowering::LiteralString, + Lowering::AnonNamedTuple, + Lowering::ProtocolType, + Lowering::TypedDictLiteral, ]; #[expect( @@ -107,6 +113,9 @@ pub(crate) struct CallableSyntax<'src> { /// substitution reaches every depth of the recursion below rather than only /// a subtree whose range matches exactly substitutions: Vec<(TextRange, String)>, + /// set while a leaf holding a use-site variance keyword or a type modifier is written out, + /// so each one is written as the type it marks. see [`Self::lower_leaf`] + writing_markers: bool, } #[derive(Clone, Debug, Eq, PartialEq, Hash)] @@ -149,6 +158,7 @@ impl<'src> CallableSyntax<'src> { protocol_class_defs: String::new(), extra_imports: Vec::new(), substitutions: Vec::new(), + writing_markers: false, } } @@ -160,7 +170,7 @@ impl<'src> CallableSyntax<'src> { } /// The name a substitution assigned to exactly `range`, if any. - fn substitution_for(&self, range: TextRange) -> Option<&str> { + pub(crate) fn substitution_for(&self, range: TextRange) -> Option<&str> { self.substitutions .iter() .find(|(candidate, _)| *candidate == range) @@ -283,6 +293,21 @@ impl<'src> CallableSyntax<'src> { if let Some(name) = self.substitution_for(expr.range()) { return name.to_owned(); } + // a use-site variance keyword or a type modifier has no python spelling, and is written + // as the type it marks — `literal str`, which has one, is a substitution. the driver + // collapses the blanks a keyword was replaced with in the source it keeps, but a leaf + // copied from the source here would keep them, and a string annotation cannot start with + // them. the markers are written out only here, in text this lowerer is writing anyway: + // asked whether a type needs rewriting at all, a marker alone answers no, so a type + // with nothing else to lower keeps its source and its own collapsed keyword + if !self.writing_markers && contains_keyword_marker(expr) { + self.writing_markers = true; + let written = self.rewrite(expr); + self.writing_markers = false; + if let Some(written) = written { + return written; + } + } // a leaf that *contains* an already-lowered subtree is rendered by // sweeping the substitutions over its source instead: the per-leaf // composers below would re-render the subtree from source and leak the @@ -823,6 +848,14 @@ impl<'src> CallableSyntax<'src> { )) } + // see `lower_leaf` + Expr::Subscript(_) + if self.writing_markers + && let Some(inner) = keyword_marker_inner(expr) => + { + Some(self.rewrite_or_leaf(inner)) + } + Expr::Subscript(s) => { // `Annotated[T, meta…]` — only the first slice element is a type // position; the rest is arbitrary metadata and must stay verbatim @@ -1001,6 +1034,18 @@ pub(crate) fn lower_type_expr_full( .map(|(text, _)| text) } +/// the type a use-site variance keyword or a type modifier marks, when `expr` is one +fn keyword_marker_inner(expr: &Expr) -> Option<&Expr> { + use_site_variance_marker(expr) + .map(|(_, inner)| inner) + .or_else(|| type_modifier_marker(expr).map(|(_, inner)| inner)) +} + +/// whether `expr` holds a use-site variance keyword or a type modifier anywhere +fn contains_keyword_marker(expr: &Expr) -> bool { + any_over_expr(expr, |node| keyword_marker_inner(node).is_some()) +} + /// if `expr` is `Subscript(Name("__let__"|"__classvar__"|"__final__"), slice)`, /// returns the slice fn synthetic_let_slice(expr: &Expr) -> Option<&Expr> { @@ -1065,6 +1110,11 @@ impl crate::transforms::type_expr_walker::TypeExprVisitor for CallableSyntax<'_> { return crate::transforms::type_expr_walker::Recurse::Stop; } + // a type another lowering replaces whole — a fold, a synthesized type's class — is + // that lowering's own edit, which a wider edit of ours writes where it covers it + if self.substitution_for(expr.range()).is_some() { + return crate::transforms::type_expr_walker::Recurse::Stop; + } // a generic's type arguments are lowered one by one, each where it stands, so a // pass that writes one of them whole, a forward reference quoted where python reads // it lazily, wraps that argument's lowering rather than landing inside a rewrite of @@ -1157,7 +1207,7 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { } /// a type expression it replaces is printed by the shared lowerer, which writes every - /// leaf inside it too, an inline protocol as the name of the class it hoists to, a + /// leaf inside it too, a synthesized type as the name of the class it hoists to, a /// union the runtime evaluates in the spelling the target can run, and an operation /// `symbolic_type_op` folded as the fold fn subsumes(&self) -> &'static [Lowering] { @@ -1169,8 +1219,11 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { Lowering::DynamicKeyword, Lowering::FloatConst, Lowering::ProtocolType, + Lowering::AnonNamedTuple, + Lowering::TypedDictLiteral, Lowering::RuntimeUnion, Lowering::SymbolicTypeOp, + Lowering::LiteralString, ] } @@ -1183,9 +1236,9 @@ impl TypeAwarePass for CallableSyntaxPass<'_> { .with_types(types) .with_claimed_ranges(&claimed) .with_owned_typeof(&structural_typeof); - // a fold inside a callable type this pass replaces is written as the fold; one - // it does not replace keeps its own edit - for (range, rendered) in &ctx.symbolic_substitutions { + // a fold or a synthesized type inside a callable type this pass replaces is written as + // the fold or the class; one it does not replace keeps its own edit + for (range, rendered) in &ctx.type_substitutions { inner.add_substitution(*range, rendered.clone()); } crate::transforms::type_expr_walker::walk_type_positions_skipping( diff --git a/crates/by_transforms/src/transforms/checked_cast.rs b/crates/by_transforms/src/transforms/checked_cast.rs index 513d345aee..a1488915fb 100644 --- a/crates/by_transforms/src/transforms/checked_cast.rs +++ b/crates/by_transforms/src/transforms/checked_cast.rs @@ -274,8 +274,10 @@ impl TypeAwarePass for CheckedCastPass<'_> { super::ast_driver::Lowering::DynamicKeyword, super::ast_driver::Lowering::FloatConst, super::ast_driver::Lowering::JustFloat, + super::ast_driver::Lowering::LiteralString, super::ast_driver::Lowering::LiteralType, super::ast_driver::Lowering::OptionalType, + super::ast_driver::Lowering::ProtocolType, super::ast_driver::Lowering::SymbolicTypeOp, super::ast_driver::Lowering::TemplateType, super::ast_driver::Lowering::TupleLiteralType, @@ -351,6 +353,42 @@ mod tests { assert!(out.contains("def _try_cast"), "got:\n{out}"); } + /// below 3.10 a union written with `|` is lowered to `Union[...]` wherever python evaluates + /// it, a cast target among them. a target the cast checks is written as the check, which + /// stands for the whole union, and one it keeps passes the lowered union through + #[test] + fn a_union_target_below_3_10() { + let config = Config { + min_version: PythonVersion::PY39, + ..Config::test_default() + }; + let out = transpile( + indoc::indoc! {" + from typing import Protocol + + class Get[T](Protocol): + def get(self) -> T: ... + + def f(a: object): + b = a cast? list[int] | str + c = a cast? Get[int] | str + d = a cast! int | str + "}, + &config, + ) + .unwrap(); + for check in [ + "b = _try_cast_pred(a, lambda _by_cast_value: ", + "c = _try_cast_pred(a, lambda _by_cast_value: ", + ] { + assert!(out.contains(check), "got:\n{out}"); + } + assert!( + out.contains("d = _checked_cast(a, (int, str))"), + "got:\n{out}" + ); + } + #[test] fn checked_cast_to_union() { let out = check("def f(a: object):\n b = a cast! int | str\n"); diff --git a/crates/by_transforms/src/transforms/decl_site_variance.rs b/crates/by_transforms/src/transforms/decl_site_variance.rs index c6484db3ec..34ff57dea8 100644 --- a/crates/by_transforms/src/transforms/decl_site_variance.rs +++ b/crates/by_transforms/src/transforms/decl_site_variance.rs @@ -1,8 +1,8 @@ //! AST pass: strips the def-site keywords a type-parameter declaration writes -//! ahead of its name — `out` / `in` / `in out` in `class C[out T]`, and -//! `reified` in `def f[reified T]`. both are preserved on the AST node and -//! consumed by ty's type checker directly — this only deletes surface bytes so -//! output is valid Python. +//! ahead of its name — `out` / `in` / `in out` in `class C[out T]` and +//! `type A[out T] = ...`, and `reified` in `def f[reified T]`. both are +//! preserved on the AST node and consumed by ty's type checker directly — this +//! only deletes surface bytes so output is valid Python. //! //! also strips the lower end of a basedpython bound range (`class C[T: int..object]`), which //! ty enforces from the AST node; python bounds only have an upper end @@ -123,6 +123,7 @@ impl<'ast> Visitor<'ast> for State<'_> { let type_params = match stmt { Stmt::ClassDef(c) => c.type_params.as_deref(), Stmt::FunctionDef(f) => f.type_params.as_deref(), + Stmt::TypeAlias(a) => a.type_params.as_deref(), _ => None, }; @@ -202,6 +203,31 @@ mod tests { check_py312("class C[T]: ...\n", "class C[T]: ...\n"); } + #[test] + fn strips_on_type_alias() { + check_py312( + "type Pinned[in out T] = list[T]\ntype Alias[out T: int..object] = list[T]\n", + "type Pinned[T] = list[T]\ntype Alias[T: object] = list[T]\n", + ); + } + + #[test] + fn strips_on_type_alias_under_polyfill() { + let config = Config { + min_version: PythonVersion::PY311, + ..Config::test_default() + }; + assert_eq!( + transpile("type Alias[out T] = list[T]\n", &config).unwrap(), + indoc! {" + from typing import TypeVar + from typing_extensions import TypeAliasType + _T = TypeVar(\"_T\", covariant=True) + Alias = TypeAliasType(\"Alias\", list[_T], type_params=(_T,)) + "}, + ); + } + #[test] fn strips_lower_bound_end() { check_py312( diff --git a/crates/by_transforms/src/transforms/destructure.rs b/crates/by_transforms/src/transforms/destructure.rs index 61c6fd408d..86d0292a38 100644 --- a/crates/by_transforms/src/transforms/destructure.rs +++ b/crates/by_transforms/src/transforms/destructure.rs @@ -797,7 +797,6 @@ fn is_name_fragment(subject: &[Fragment]) -> bool { #[cfg(test)] mod tests { use indoc::indoc; - use ruff_python_ast::PythonVersion; use crate::{Config, transpile}; @@ -1293,19 +1292,4 @@ mod tests { assert!(!out.contains("??"), "got:\n{out}"); assert!(out.contains(" z = "), "got:\n{out}"); } - - /// the `match` a destructuring lowers to is itself lowered for a target - /// that predates it, so the construct reaches every version the polyfill - /// covers rather than being an error below 3.10 - #[test] - fn lowers_below_python_310() { - let config = Config { - min_version: PythonVersion::PY39, - ..Config::test_default() - }; - let out = transpile("let (a, b) := (1, 2)\n", &config).unwrap(); - assert!(!out.contains("match "), "got:\n{out}"); - assert!(out.contains("a := "), "binds the captures, got:\n{out}"); - assert!(out.contains("b := "), "binds the captures, got:\n{out}"); - } } diff --git a/crates/by_transforms/src/transforms/except_polyfill.rs b/crates/by_transforms/src/transforms/except_polyfill.rs new file mode 100644 index 0000000000..7cedb3eec2 --- /dev/null +++ b/crates/by_transforms/src/transforms/except_polyfill.rs @@ -0,0 +1,154 @@ +//! Lowers the bracketless `except` of PEP 758 for a target before python 3.14. +//! +//! python 3.14 lets a handler that catches several types leave out the parentheses +//! around them, as long as it binds nothing with `as`. before that, the types have to +//! be written as a parenthesized tuple, which is what the handler means either way: +//! +//! ```text +//! except TimeoutError, ConnectionError: except (TimeoutError, ConnectionError): +//! ``` +//! +//! the same goes for `except*`. like the `match` lowering, this runs over the finished +//! python, so a handler another lowering writes is lowered by the same code as one the +//! author wrote. only the parentheses are added, so every line keeps its place + +use ruff_python_ast::visitor::{Visitor, walk_except_handler}; +use ruff_python_ast::{ExceptHandler, Expr, PySourceType, PythonVersion}; +use ruff_text_size::{Ranged, TextSize}; + +use crate::Config; + +pub(crate) fn lower(source: String, config: &Config) -> String { + if config.min_version >= PythonVersion::PY314 { + return source; + } + let parsed = ruff_python_parser::parse_unchecked_source(&source, PySourceType::Python); + let mut bracketless = Bracketless(Vec::new()); + bracketless.visit_body(parsed.suite()); + if bracketless.0.is_empty() { + return source; + } + + let mut insertions: Vec<(TextSize, char)> = bracketless + .0 + .into_iter() + .flat_map(|(start, end)| [(start, '('), (end, ')')]) + .collect(); + insertions.sort_by_key(|(at, _)| *at); + let mut out = String::with_capacity(source.len() + insertions.len()); + let mut at = 0usize; + for (position, paren) in insertions { + let position = usize::from(position); + out.push_str(&source[at..position]); + out.push(paren); + at = position; + } + out.push_str(&source[at..]); + out +} + +/// the start and end of the types of every handler that writes them without parentheses +struct Bracketless(Vec<(TextSize, TextSize)>); + +impl<'a> Visitor<'a> for Bracketless { + fn visit_except_handler(&mut self, handler: &'a ExceptHandler) { + let ExceptHandler::ExceptHandler(handler_node) = handler; + if let Some(Expr::Tuple(types)) = handler_node.type_.as_deref() + && !types.parenthesized + { + self.0.push((types.start(), types.end())); + } + walk_except_handler(self, handler); + } +} + +#[cfg(test)] +mod tests { + use indoc::indoc; + use ruff_python_ast::PythonVersion; + + use crate::{Config, transpile}; + + fn at(min_version: PythonVersion, source: &str) -> String { + let config = Config { + min_version, + ..Config::test_default() + }; + transpile(source, &config).unwrap() + } + + #[test] + fn parenthesizes_the_types_before_3_14() { + let out = at( + PythonVersion::PY313, + indoc! {" + try: + pass + except ValueError, TypeError: + pass + "}, + ); + assert!( + out.contains("except (ValueError, TypeError):\n"), + "got:\n{out}" + ); + } + + #[test] + fn parenthesizes_an_except_star_before_3_14() { + let out = at( + PythonVersion::PY311, + indoc! {" + try: + pass + except* ValueError, TypeError: + pass + "}, + ); + assert!( + out.contains("except* (ValueError, TypeError):\n"), + "got:\n{out}" + ); + } + + #[test] + fn leaves_a_nested_handler_and_a_parenthesized_one_in_place() { + let out = at( + PythonVersion::PY310, + indoc! {" + try: + pass + except (KeyError, IndexError): + try: + pass + except ValueError, TypeError: + pass + "}, + ); + assert!( + out.contains("except (KeyError, IndexError):\n"), + "got:\n{out}" + ); + assert!( + out.contains(" except (ValueError, TypeError):\n"), + "got:\n{out}" + ); + } + + #[test] + fn keeps_the_bracketless_form_from_3_14() { + let out = at( + PythonVersion::PY314, + indoc! {" + try: + pass + except ValueError, TypeError: + pass + "}, + ); + assert!( + out.contains("except ValueError, TypeError:\n"), + "got:\n{out}" + ); + } +} diff --git a/crates/by_transforms/src/transforms/generics.rs b/crates/by_transforms/src/transforms/generics.rs index 6be0dc104f..59011215d1 100644 --- a/crates/by_transforms/src/transforms/generics.rs +++ b/crates/by_transforms/src/transforms/generics.rs @@ -5,6 +5,7 @@ use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::visitor::{Visitor, walk_body, walk_expr, walk_stmt}; use ruff_python_ast::{Expr, Stmt, StmtClassDef, StmtFunctionDef, StmtTypeAlias, TypeParam}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use ty_python_semantic::types::NameResolution; use crate::config::Config; use crate::transforms::callable::lower_type_expr_full; @@ -61,9 +62,14 @@ pub(crate) struct GenericPolyfill<'src> { /// (`class Owner[T]: def narrow[U: T]`), and that name is mangled by the enclosing /// list, not by the one being processed enclosing_renames: Vec>, - /// `(range, rendered)` for every symbolic fold in the module. A fold inside a - /// statement this pass replaces wholesale is dropped unless spliced in here - symbolic_substitutions: Vec<(TextRange, String)>, + /// `(range, rendered)` for every type expression already lowered to a fixed spelling: a + /// symbolic fold, a synthesized type's class. One inside a statement this pass replaces + /// wholesale is dropped unless spliced in here + type_substitutions: Vec<(TextRange, String)>, + /// the span of every type that lowers to a synthesized class, whose body reads each type + /// parameter by the name this pass declares it under already — see + /// [`Namer::read_type_parameters`](super::synthesized_types::Namer::read_type_parameters) + synthesized: Vec, /// text edits earlier passes already emitted. one of these covering a type /// expression hides the source bytes our typevar rename would have patched /// — the arrow lowering's `(T) -> R` → `Callable[[T], R]` is the common case @@ -211,15 +217,14 @@ struct ProcessedTypeParams { } impl<'src> GenericPolyfill<'src> { - #[expect(clippy::too_many_arguments)] + /// the pass over a module, taking what the passes before it wrote from `ctx` fn new( source: &'src str, written: WrittenNames<'src>, quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, config: Config, - symbolic_substitutions: Vec<(TextRange, String)>, - pending_edits: Vec<(TextRange, String)>, + ctx: &super::ast_driver::PassContext, decided: PolyfilledTypeParams, ) -> Self { let constructors = Constructors::new(written, &config, &decided); @@ -239,8 +244,13 @@ impl<'src> GenericPolyfill<'src> { generic_class_renames: HashMap::new(), enclosing_renames: Vec::new(), private_aliases: HashMap::new(), - symbolic_substitutions, - pending_edits, + type_substitutions: ctx.type_substitutions.clone(), + synthesized: ctx + .synthesized + .substitutions() + .map(|(range, _)| range) + .collect(), + pending_edits: ctx.text_edits.clone(), superseded: Vec::new(), statement_prefixes: Vec::new(), } @@ -248,7 +258,8 @@ impl<'src> GenericPolyfill<'src> { /// take over every rewrite that already exists inside `range`, so that a /// replacement covering `range` can carry them: the symbolic folds - /// (`Dim + 1` → `int`) and the text edits earlier passes queued + /// (`Dim + 1` → `int`), the synthesized types' classes, and the text edits earlier passes + /// queued /// (`kw_subscript`'s positional rebuild, `unpack`'s `Unpack[…]`). without /// the second group a keyword subscript or a variadic inside a polyfilled /// `type` alias value or a type-parameter bound reaches the output in its @@ -260,7 +271,7 @@ impl<'src> GenericPolyfill<'src> { /// span would delete those edits without reinstating them fn subsume_within(&mut self, range: TextRange) -> Vec<(TextRange, String)> { let mut within: Vec<(TextRange, String)> = self - .symbolic_substitutions + .type_substitutions .iter() .filter(|(folded, _)| range.contains_range(*folded)) .cloned() @@ -269,14 +280,18 @@ impl<'src> GenericPolyfill<'src> { // `Unpack[` replacement plus a `]` insertion — so it has to come along // too, or the subsumed rewrite arrives half-applied for (pending, text) in &self.pending_edits { - if range.contains_range(*pending) - && !within - .iter() - .any(|(seen, seen_text)| seen == pending && seen_text == text) + if !range.contains_range(*pending) { + continue; + } + // a pending edit writing what a substitution already carries — a synthesized type + // replaced by its class — is carried by the substitution, and superseded all the same + if !within + .iter() + .any(|(seen, seen_text)| seen == pending && seen_text == text) { within.push((*pending, text.clone())); - self.superseded.push(*pending); } + self.superseded.push(*pending); } within } @@ -336,6 +351,28 @@ impl<'src> GenericPolyfill<'src> { } } + /// rename the type parameters `renames` maps wherever `expr` reads them, as edits + fn rename_expr(&mut self, expr: &Expr, renames: &HashMap) { + rename_in_expr( + expr, + renames, + Some(self.types), + &self.synthesized, + &mut self.edits, + ); + } + + /// [`Self::rename_expr`] over every expression of `stmt`, at any depth + fn rename_stmt(&mut self, stmt: &Stmt, renames: &HashMap) { + Rename { + renames, + types: Some(self.types), + written_whole: &self.synthesized, + edits: &mut self.edits, + } + .visit_stmt(stmt); + } + /// pre-scan for the module's `private type` aliases so a reference inside a /// later alias's value can be renamed as the value is re-rendered. like the /// module's other private symbols, they are the ones declared outside any class @@ -392,7 +429,7 @@ impl<'src> GenericPolyfill<'src> { /// Lower one element of a parameter-shape tuple to a Python type /// expression suitable for inclusion inside `tuple[...]`. Mirrors the /// rules in `annotation.rs::lower_tuple_element` - fn lower_param_shape_elt(&self, elt: &Expr) -> String { + fn lower_param_shape_elt(&mut self, elt: &Expr) -> String { match elt { Expr::Named(named) => { if let Expr::Starred(starred) = named.target.as_ref() { @@ -402,10 +439,10 @@ impl<'src> GenericPolyfill<'src> { return format!( "*{}[{}, ...]", self.written.builtin("tuple"), - self.src(named.value.range()) + self.lowered_type(&named.value) ); } - self.src(named.value.range()).to_owned() + self.lowered_type(&named.value) } Expr::Starred(s) => { if matches!(s.value.as_ref(), Expr::Starred(_)) { @@ -414,10 +451,10 @@ impl<'src> GenericPolyfill<'src> { format!( "*{}[{}, ...]", self.written.builtin("tuple"), - self.src(s.value.range()) + self.lowered_type(&s.value) ) } - _ => self.src(elt.range()).to_owned(), + _ => self.lowered_type(elt), } } @@ -427,21 +464,30 @@ impl<'src> GenericPolyfill<'src> { (start, indent) } - /// `default`, a type parameter's default, as the python the `default=` argument of its - /// declaration is written with. `visible` renames the type parameters in scope - fn default_arg(&mut self, default: &Expr, visible: &HashMap) -> String { - let rendered = lower_type_expr_full( + /// `expr`, a type written as an argument of a type parameter's declaration — its default, + /// one of its constraints — as python spells it. `visible` renames the type parameters in + /// scope + fn type_argument(&mut self, expr: &Expr, visible: &HashMap) -> String { + apply_renames_to_rendered(&self.lowered_type(expr), visible) + } + + /// `expr`, a type this pass writes in place of its source, as python spells it, carrying + /// whatever the lowerings inside it wrote + /// + /// this commits as [`Self::subsume_within`] does: the caller writes the result over the + /// span of `expr` + fn lowered_type(&mut self, expr: &Expr) -> String { + lower_type_expr_full( self.source, self.written, self.quotes, self.types, - default, - &self.subsume_within(default.range()), + expr, + &self.subsume_within(expr.range()), &self.config, RootKind::Evaluated, ) - .unwrap_or_else(|| self.src(default.range()).to_owned()); - apply_renames_to_rendered(&rendered, visible) + .unwrap_or_else(|| self.src(expr.range()).to_owned()) } /// the `default=` argument of a `ParamSpec`: a list of types, `...`, or another parameter @@ -455,11 +501,11 @@ impl<'src> GenericPolyfill<'src> { let elements: Vec = list .elts .iter() - .map(|element| self.default_arg(element, visible)) + .map(|element| self.type_argument(element, visible)) .collect(); format!("[{}]", elements.join(", ")) } - other => self.default_arg(other, visible), + other => self.type_argument(other, visible), } } @@ -545,17 +591,17 @@ impl<'src> GenericPolyfill<'src> { // In basedpython, `T: (int, str)` means bound=(int, str), not // positional constraints — the mapping is written with `in` if tv.is_type_mapping { - let inner = match bound.as_ref() { + let members: Vec = match bound.as_ref() { Expr::Tuple(t) if t.parenthesized => t .elts .iter() - .map(|e| self.src(e.range())) - .collect::>() - .join(", "), - other => self.src(other.range()).to_owned(), + .map(|member| self.type_argument(member, &visible)) + .collect(), + other => vec![self.type_argument(other, &visible)], }; + let inner = members.join(", "); if !inner.is_empty() { - extra_args.push(apply_renames_to_rendered(&inner, &visible)); + extra_args.push(inner); } } else { // basedpython parameter-shape tuple bound — lower to @@ -606,7 +652,7 @@ impl<'src> GenericPolyfill<'src> { if let Some(default) = &tv.default { extra_args.push(format!( "default={}", - through_classes(self.default_arg(default, &visible)) + through_classes(self.type_argument(default, &visible)) )); } @@ -653,7 +699,7 @@ impl<'src> GenericPolyfill<'src> { Expr::Starred(starred) => starred.value.as_ref(), other => other, }; - let rendered = self.default_arg(unpacked, &visible); + let rendered = self.type_argument(unpacked, &visible); if matches!(default, Expr::Starred(_)) { self.needed_imports.unpack = true; format!("{}[{rendered}]", self.constructors.unpack) @@ -824,7 +870,7 @@ impl<'src> GenericPolyfill<'src> { // as individual edits — this lets literal_types and auto_quote also // emit their own non-overlapping edits on the same expressions. for base_expr in &args.args { - rename_in_expr(base_expr, &rename_map, &mut self.edits); + self.rename_expr(base_expr, &rename_map); } if args.args.is_empty() && args.keywords.is_empty() { // empty `()` → replace with `(Generic[_T])`; 2-char range, safe @@ -879,7 +925,7 @@ impl<'src> GenericPolyfill<'src> { // Rename type param references in class body. let mark = self.edits.len(); for stmt in &class.body { - rename_in_stmt(stmt, &rename_map, &mut self.edits); + self.rename_stmt(stmt, &rename_map); } self.reconcile_pending(&rename_map, mark); rename_map @@ -898,7 +944,7 @@ impl<'src> GenericPolyfill<'src> { }; if let Some(rename_map) = self.generic_class_renames.get(base.id.as_str()).cloned() { for stmt in &class.body { - rename_in_stmt(stmt, &rename_map, &mut self.edits); + self.rename_stmt(stmt, &rename_map); } } } @@ -953,21 +999,21 @@ impl<'src> GenericPolyfill<'src> { .chain(func.parameters.kwonlyargs.iter()); for param in all_params { if let Some(ann) = ¶m.parameter.annotation { - rename_in_expr(ann, &rename_map, &mut self.edits); + self.rename_expr(ann, &rename_map); } } if let Some(vararg) = &func.parameters.vararg && let Some(ann) = &vararg.annotation { - rename_in_expr(ann, &rename_map, &mut self.edits); + self.rename_expr(ann, &rename_map); } if let Some(kwarg) = &func.parameters.kwarg && let Some(ann) = &kwarg.annotation { - rename_in_expr(ann, &rename_map, &mut self.edits); + self.rename_expr(ann, &rename_map); } if let Some(ret) = &func.returns { - rename_in_expr(ret, &rename_map, &mut self.edits); + self.rename_expr(ret, &rename_map); } // the body reads a parameter under its own name, which a type parameter of that name // does not reach: the parameter is bound in the body's scope, the type parameter in @@ -979,7 +1025,7 @@ impl<'src> GenericPolyfill<'src> { self.reconcile_pending(&rename_map, mark); let mark = self.edits.len(); for stmt in &func.body { - rename_in_stmt(stmt, &body_renames, &mut self.edits); + self.rename_stmt(stmt, &body_renames); } self.reconcile_pending(&body_renames, mark); // what the rest of the body sees, a nested definition among it @@ -1077,17 +1123,19 @@ impl<'src> GenericPolyfill<'src> { // silently losing the other. let folded = self.subsume_within(alias.value.range()); let mut value_renames: Vec = Vec::new(); - rename_in_expr(&alias.value, &rename_map, &mut value_renames); + // a rename inside a folded operation went with the operand it renamed — + // `type X[T: A] = T.a` folds to `int`, which mentions no `T` to rename + let folded_ranges: Vec = folded.iter().map(|(range, _)| *range).collect(); + rename_in_expr( + &alias.value, + &rename_map, + None, + &folded_ranges, + &mut value_renames, + ); let renames = value_renames .iter() .flat_map(ruff_diagnostics::Fix::edits) - // a rename inside a folded operation went with the operand it renamed — - // `type X[T: A] = T.a` folds to `int`, which mentions no `T` to rename - .filter(|edit| { - !folded - .iter() - .any(|(range, _)| range.contains_range(edit.range())) - }) .map(|edit| (edit.range(), edit.content().unwrap_or_default().to_owned())); let substitutions: Vec<(TextRange, String)> = folded.iter().cloned().chain(renames).collect(); @@ -1182,7 +1230,7 @@ impl GenericPolyfill<'_> { } _ => { parts.push(match visible { - Some(visible) => self.default_arg(elt, visible), + Some(visible) => self.type_argument(elt, visible), None => self.src(elt.range()).to_owned(), }); } @@ -1332,135 +1380,81 @@ fn apply_renames_to_rendered(rendered: &str, renames: &HashMap) out } -fn rename_in_expr(expr: &Expr, renames: &HashMap, edits: &mut Vec) { - match expr { - Expr::Name(n) => { - if let Some(new) = renames.get(n.id.as_str()) { - edits.push(Fix::safe_edit(Edit::range_replacement( - new.clone(), - n.range(), - ))); - } - } - Expr::Subscript(s) => { - rename_in_expr(&s.value, renames, edits); - rename_in_expr(&s.slice, renames, edits); - } - Expr::Attribute(a) => rename_in_expr(&a.value, renames, edits), - Expr::Tuple(t) => t - .elts - .iter() - .for_each(|e| rename_in_expr(e, renames, edits)), - Expr::List(l) => l - .elts - .iter() - .for_each(|e| rename_in_expr(e, renames, edits)), - Expr::BinOp(b) => { - rename_in_expr(&b.left, renames, edits); - rename_in_expr(&b.right, renames, edits); - } - Expr::Call(c) => { - rename_in_expr(&c.func, renames, edits); - c.arguments - .args - .iter() - .for_each(|a| rename_in_expr(a, renames, edits)); - } - Expr::UnaryOp(u) => rename_in_expr(&u.operand, renames, edits), - Expr::Starred(s) => rename_in_expr(&s.value, renames, edits), - // an arrow callable `(**P) -> None` lowers to `Callable[P, None]` via a template edit - // that passes its operand source through, so a rename on the inner name still lands - Expr::CallableType(c) => { - if let Some(receiver) = &c.receiver { - rename_in_expr(receiver, renames, edits); - } - c.args - .iter() - .for_each(|a| rename_in_expr(a, renames, edits)); - rename_in_expr(&c.returns, renames, edits); - } - _ => {} +/// rename every name in `expr` that `renames` maps, as an edit +/// +/// with `types`, the renames are a generic's type parameters, and a name ty finds bound +/// elsewhere — declared `global`, or bound by a scope of its own — keeps the author's spelling. +/// without, every spelling `renames` maps is renamed +/// +/// nothing inside a span of `written_whole` is renamed: what is written there in its place +/// reads the names the polyfill declares already, or none at all +fn rename_in_expr( + expr: &Expr, + renames: &HashMap, + types: Option<&dyn TypeInfo>, + written_whole: &[TextRange], + edits: &mut Vec, +) { + Rename { + renames, + types, + written_whole, + edits, } + .visit_expr(expr); } -fn rename_in_stmt(stmt: &Stmt, renames: &HashMap, edits: &mut Vec) { - match stmt { - Stmt::AnnAssign(a) => { - rename_in_expr(&a.annotation, renames, edits); - if let Some(v) = &a.value { - rename_in_expr(v, renames, edits); - } - } - Stmt::FunctionDef(f) => { - for p in f - .parameters - .posonlyargs - .iter() - .chain(f.parameters.args.iter()) - .chain(f.parameters.kwonlyargs.iter()) - { - if let Some(ann) = &p.parameter.annotation { - rename_in_expr(ann, renames, edits); - } - } - for variadic in [ - f.parameters.vararg.as_deref(), - f.parameters.kwarg.as_deref(), - ] - .into_iter() - .flatten() - { - if let Some(ann) = &variadic.annotation { - rename_in_expr(ann, renames, edits); - } - } - if let Some(ret) = &f.returns { - rename_in_expr(ret, renames, edits); - } - for s in &f.body { - rename_in_stmt(s, renames, edits); - } - } - Stmt::Return(r) => { - if let Some(v) = &r.value { - rename_in_expr(v, renames, edits); - } - } - Stmt::Assign(a) => { - for t in &a.targets { - rename_in_expr(t, renames, edits); - } - rename_in_expr(&a.value, renames, edits); +struct Rename<'a> { + renames: &'a HashMap, + types: Option<&'a dyn TypeInfo>, + written_whole: &'a [TextRange], + edits: &'a mut Vec, +} + +impl<'ast> Visitor<'ast> for Rename<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + // a nested class that introduces its *own* type params is polyfilled independently + // and may shadow the name, so it is left to that. one without, an enum's variant + // class say, reads the enclosing class's parameters in its bases and field + // annotations, and after the polyfill those resolve to the mangled `TypeVar` bound + // at module scope + if let Stmt::ClassDef(class) = stmt + && class.type_params.is_some() + { + return; } - Stmt::Expr(e) => rename_in_expr(&e.value, renames, edits), - Stmt::If(i) => { - rename_in_expr(&i.test, renames, edits); - for s in &i.body { - rename_in_stmt(s, renames, edits); - } - for clause in &i.elif_else_clauses { - for s in &clause.body { - rename_in_stmt(s, renames, edits); - } - } + walk_stmt(self, stmt); + } + + /// a nested function's own type-parameter list is written by its own polyfill, which + /// reads the parameters of the lists around it under their mangled names already + fn visit_type_params(&mut self, _type_params: &'ast ruff_python_ast::TypeParams) {} + + fn visit_expr(&mut self, expr: &'ast Expr) { + if self + .written_whole + .iter() + .any(|range| range.contains_range(expr.range())) + { + return; } - // descend into a nested class (e.g. an enum's nested variant classes) - // so the enclosing class's type-param references in its bases and field - // annotations are renamed too — after the polyfill the mangled `TypeVar` - // is bound at module scope, so the nested reference resolves to it. a - // nested class that introduces its *own* type params is polyfilled - // independently and may shadow the name, so skip it - Stmt::ClassDef(c) if c.type_params.is_none() => { - if let Some(args) = &c.arguments { - for base in &args.args { - rename_in_expr(base, renames, edits); - } - } - for s in &c.body { - rename_in_stmt(s, renames, edits); - } + let Expr::Name(name) = expr else { + walk_expr(self, expr); + return; + }; + if let Some(new) = self.renames.get(name.id.as_str()) + && !self.types.is_some_and(|types| { + matches!( + types.name_resolution(name), + Some(NameResolution::DeclaredGlobal | NameResolution::Local) + ) + }) + { + self.edits.push(Fix::safe_edit(Edit::range_replacement( + new.clone(), + name.range(), + ))); } - _ => {} } } @@ -2091,14 +2085,15 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { /// a header or alias it polyfills is re-printed with its type parameters renamed, and it /// re-renders what the lowerings inside wrote rather than their source — the symbolic - /// folds, the private alias names, the tuple, named tuple and match types, and the variance - /// keywords as `TypeVar` arguments + /// folds, the `LiteralString` a `literal str` is written as, the private alias names, the + /// tuple, named tuple and match types, and the variance keywords as `TypeVar` arguments fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { &[ super::ast_driver::Lowering::TupleLiteralType, super::ast_driver::Lowering::AnonNamedTuple, super::ast_driver::Lowering::MatchType, super::ast_driver::Lowering::SymbolicTypeOp, + super::ast_driver::Lowering::LiteralString, super::ast_driver::Lowering::Modifiers, super::ast_driver::Lowering::VisibilityRename, super::ast_driver::Lowering::VarianceStrip, @@ -2117,8 +2112,7 @@ impl super::ast_driver::TypeAwarePass for GenericPolyfillPass<'_> { self.quotes, types, self.config.clone(), - ctx.symbolic_substitutions.clone(), - ctx.text_edits.clone(), + ctx, PolyfilledTypeParams::decide(self.source, self.written, stmts, &self.config), ); inner.collect_private_aliases(stmts); @@ -2505,6 +2499,83 @@ mod tests { ); } + /// a type parameter read anywhere in the body is renamed, however deep the statement it + /// stands in: a class body runs its annotations as the class is made, so one missed in a + /// loop there is a `NameError` on import + #[test] + fn a_type_parameter_is_renamed_in_every_compound_statement() { + check( + indoc! {" + class A[T]: + for i in range(1): + a: T + while False: + b: T + with open(\"f\") as f: + c: T + try: + d: T + except Exception: + e: T + else: + g: T + finally: + h: T + match 1: + case 1: + k: T + def method(self) -> None: + for i in range(1): + m: list[T] = [] + "}, + indoc! {" + from typing import TypeVar, Generic + _T = TypeVar(\"_T\") + class A(Generic[_T]): + for i in range(1): + a: _T + while False: + b: _T + with open(\"f\") as f: + c: _T + try: + d: _T + except Exception: + e: _T + else: + g: _T + finally: + h: _T + match 1: + case 1: + k: _T + def method(self) -> None: + for i in range(1): + m: list[_T] = [] + "}, + ); + } + + /// a folded operation is written as the type it folds to, which reads the parameters the + /// operation read, so they are renamed in it too + #[test] + fn a_type_parameter_is_renamed_inside_a_fold() { + check_at( + indoc! {" + def pair_up[T, U](values: (T, U) * int) -> (U, T) * int: + return values + "}, + indoc! {" + from typing import TypeVar + _T = TypeVar(\"_T\") + _U = TypeVar(\"_U\") + def pair_up(values: tuple[_T | _U, ...]) -> tuple[_U | _T, ...]: + return values + "}, + PythonVersion::PY311, + ); + } + #[test] fn rendered_rename_skips_string_and_attribute() { let mut renames = std::collections::HashMap::new(); @@ -2801,6 +2872,46 @@ mod tests { ); } + /// a method that declares the type parameter's name `global` reads the module's binding, and + /// one that binds the name itself reads its own, so neither is renamed to the `TypeVar` + #[test] + fn a_name_bound_elsewhere_keeps_its_spelling() { + check( + indoc! {" + T = \"module level\" + + class Box[T]: + def module(self) -> str: + global T + return T + + def own(self) -> int: + T = 1 + return T + + def parameter(self, x: T) -> T: + return x + "}, + indoc! {" + from typing import TypeVar, Generic + T = \"module level\" + + _T = TypeVar(\"_T\") + class Box(Generic[_T]): + def module(self) -> str: + global T + return T + + def own(self) -> int: + T = 1 + return T + + def parameter(self, x: _T) -> _T: + return x + "}, + ); + } + /// a definition written into an `if` does not run when the `if` does not, so a function the /// module declares later gets a definition of its own, under a name that shadows neither. /// a method's is written ahead of its class, at module scope, where everything that reads @@ -3156,6 +3267,65 @@ mod tests { ); } + /// `literal str` is the one type modifier python can spell, as `LiteralString`, so a + /// polyfilled type parameter carries it in its bound, its tuple bound and its constraints, + /// as the native spelling does from 3.12. the other modifiers have no spelling and erase, + /// there as everywhere: a constraint copied from the source keeps the padding the keyword + /// was blanked to, and a tuple bound, which is written out, is written without it + #[test] + fn a_literal_string_modifier_reaches_a_polyfilled_type_parameter() { + check_at( + indoc! {" + def bounded[T: literal str](t: T) -> T: + return t + def in_a_tuple[T: (literal str, literal int)](t: T) -> T: + return t + def constrained[T in (literal str, literal int)](t: T) -> T: + return t + class C[T: literal str]: ... + type Alias[T: literal str] = list[T] + "}, + indoc! {" + from typing import LiteralString, TypeVar, Generic + from typing_extensions import TypeAliasType + _T = TypeVar(\"_T\", bound=LiteralString) + def bounded(t: _T) -> _T: + return t + _T_1 = TypeVar(\"_T_1\", bound=tuple[LiteralString, int]) + def in_a_tuple(t: _T_1) -> _T_1: + return t + _T_2 = TypeVar(\"_T_2\", LiteralString, int) + def constrained(t: _T_2) -> _T_2: + return t + class C(Generic[_T]): ... + Alias = TypeAliasType(\"Alias\", list[_T], type_params=(_T,)) + "}, + PythonVersion::PY311, + ); + } + + /// the native spelling, where a tuple bound is written `tuple[...]` and the type mapping + /// with python's own `:` + #[test] + fn a_literal_string_modifier_reaches_a_native_tuple_bound_and_constraints() { + check_at( + indoc! {" + def in_a_tuple[T: (literal str, literal int)](t: T) -> T: + return t + def constrained[T in (literal str, literal int)](t: T) -> T: + return t + "}, + indoc! {" + from typing import LiteralString + def in_a_tuple[T: tuple[LiteralString, int]](t: T) -> T: + return t + def constrained[T: (LiteralString, int)](t: T) -> T: + return t + "}, + PythonVersion::PY312, + ); + } + #[test] fn type_mapping_on_type_alias_lowered_on_312() { check_at( diff --git a/crates/by_transforms/src/transforms/if_let.rs b/crates/by_transforms/src/transforms/if_let.rs index da173834f2..95c993e173 100644 --- a/crates/by_transforms/src/transforms/if_let.rs +++ b/crates/by_transforms/src/transforms/if_let.rs @@ -290,7 +290,6 @@ mod tests { use indoc::indoc; use crate::{Config, transpile}; - use ruff_python_ast::PythonVersion; fn check(input: &str) -> String { transpile(input, &Config::test_default()).unwrap() @@ -516,26 +515,4 @@ mod tests { "got:\n{out}" ); } - - /// the `match` this lowering emits is itself lowered for a target that - /// predates it, so an `if let` reaches every version the polyfill covers - #[test] - fn lowers_below_python_310() { - let config = Config { - min_version: PythonVersion::PY39, - ..Config::test_default() - }; - let out = transpile( - indoc! {" - opt: int | None = 1 - if let int(x) := opt: - print(x) - "}, - &config, - ) - .unwrap(); - assert!(!out.contains("match "), "got:\n{out}"); - assert!(out.contains("x := "), "binds the capture, got:\n{out}"); - assert!(out.contains("print(x)"), "keeps the body, got:\n{out}"); - } } diff --git a/crates/by_transforms/src/transforms/init_method.rs b/crates/by_transforms/src/transforms/init_method.rs index fca51952b4..e360e6cebd 100644 --- a/crates/by_transforms/src/transforms/init_method.rs +++ b/crates/by_transforms/src/transforms/init_method.rs @@ -75,7 +75,7 @@ impl TypeAwarePass for InitMethod<'_> { written: self.written, quotes: self.quotes, types, - symbolic_substitutions: ctx.symbolic_substitutions.clone(), + type_substitutions: ctx.type_substitutions.clone(), module_private: types.private_module_symbols().into_iter().collect(), config: &self.config, edits: RefCell::new(Vec::new()), @@ -147,10 +147,10 @@ struct State<'src> { /// where a synthesized protocol's member types are written as strings quotes: &'src WrittenQuotes, types: &'src dyn TypeInfo, - /// `(range, rendered)` for every symbolic fold in the module. the `__init__` - /// line is fresh output, so a fold inside a parameter annotation is dropped - /// unless it is spliced in here - symbolic_substitutions: Vec<(TextRange, String)>, + /// `(range, rendered)` for every type expression already lowered to a fixed spelling: a + /// symbolic fold, a synthesized type's class. the `__init__` line is fresh output, so one + /// inside a parameter annotation is dropped unless it is spliced in here + type_substitutions: Vec<(TextRange, String)>, /// the module-level names the module declares `private` module_private: HashSet, config: &'src Config, @@ -232,7 +232,7 @@ impl State<'_> { }; renames.visit_expr(ann); let substitutions: Vec<(TextRange, String)> = self - .symbolic_substitutions + .type_substitutions .iter() .filter(|(range, _)| ann.range().contains_range(*range)) .cloned() diff --git a/crates/by_transforms/src/transforms/lazy_import.rs b/crates/by_transforms/src/transforms/lazy_import.rs index 1d1bf452d4..f7b406fe6f 100644 --- a/crates/by_transforms/src/transforms/lazy_import.rs +++ b/crates/by_transforms/src/transforms/lazy_import.rs @@ -93,6 +93,10 @@ pub(crate) struct LazyImport<'src> { /// (`class Character(str)`) that the grapheme accessors construct, so the /// preamble must define the real class rather than the marker pub(crate) needs_character_class: bool, + /// the names `JustFloat` / `JustComplex` are bound to at runtime: the builtin `float` / + /// `complex`, under a name of its own where the module binds the builtin's + builtin_float: String, + builtin_complex: String, } impl<'src> LazyImport<'src> { @@ -103,8 +107,11 @@ impl<'src> LazyImport<'src> { eager_names: &[String], deferrable: HashSet, holder: &'src str, + written: super::repeated_underscore::WrittenNames, ) -> Self { Self { + builtin_float: written.builtin("float"), + builtin_complex: written.builtin("complex"), source, at_module_level: true, eager: eager.to_vec(), @@ -284,7 +291,15 @@ impl<'src> LazyImport<'src> { return; } - if self.deferral == Deferral::Keyword { + // `ty_extensions` exists only for a checker, so its names are bound to their runtime + // stand-ins below however the other imports are deferred: a `lazy` import of it fails + // as soon as anything reads one of them — `typing.get_type_hints` on an annotation + let is_ty_extensions = node.level == 0 + && node + .module + .as_ref() + .is_some_and(|m| m.id.as_str() == "ty_extensions"); + if self.deferral == Deferral::Keyword && !is_ty_extensions { if is_future || is_star { if node.is_lazy { self.strip_lazy_keyword(node.range()); @@ -363,8 +378,8 @@ impl<'src> LazyImport<'src> { // runtime value is the builtin, so `get_type_hints` (and thus // pydantic / dataclasses schema generation) resolves them match name { - "JustFloat" => lines.push(format!("{bind} = float")), - "JustComplex" => lines.push(format!("{bind} = complex")), + "JustFloat" => lines.push(format!("{bind} = {}", self.builtin_float)), + "JustComplex" => lines.push(format!("{bind} = {}", self.builtin_complex)), _ => { self.needs_ty_ext_marker = true; lines.push(format!("{bind} = _TyExtMarker")); @@ -986,6 +1001,32 @@ mod tests { ); } + /// `ty_extensions` exists only for a checker, so where the other imports are made `lazy` + /// its names are still bound to their stand-ins: a deferred import of it fails the first + /// time an annotation naming one is read + #[test] + fn ty_ext_names_are_bound_where_imports_are_keyword_lazy() { + let out = transpile("a: float\nb: not int\n", &cfg_315()).unwrap(); + assert!( + out.contains("JustFloat = float") + && out.contains("Not = _TyExtMarker") + && !out.contains("ty_extensions"), + "got:\n{out}" + ); + } + + /// where the module binds `float` itself, `JustFloat` is bound to the builtin, read under + /// a name of its own + #[test] + fn ty_ext_just_float_binds_to_the_builtin_the_module_rebinds() { + let out = transpile_polyfill("float = int\nfrom ty_extensions import JustFloat\n"); + assert!( + out.contains("from builtins import float as float2\n") + && out.contains("JustFloat = float2\n"), + "got:\n{out}" + ); + } + #[test] fn ty_ext_other_markers_stay_marker() { // a genuinely type-only marker (`Not`, from `not int`) has no runtime diff --git a/crates/by_transforms/src/transforms/literal_string.rs b/crates/by_transforms/src/transforms/literal_string.rs index 369a7eb17f..2a6ba2641b 100644 --- a/crates/by_transforms/src/transforms/literal_string.rs +++ b/crates/by_transforms/src/transforms/literal_string.rs @@ -19,6 +19,13 @@ //! Reading the *marker's* inferred type, rather than matching the spelling of //! the type it wraps, is what makes this agree with the checker by construction: //! a shadowed `str` never reduces to `LiteralString`, so it never rewrites here. +//! +//! A lowering that writes a type whole — a tuple type, a callable arrow, a +//! polyfilled type parameter's bound — prints the type from the blanked source, +//! where a marker's keyword is gone. Each rewrite is therefore also a +//! [type substitution](super::ast_driver::PassContext::type_substitutions), which +//! such a lowering writes in the marker's place, and the walk that finds them goes +//! on into those types rather than stopping at them. use ruff_python_ast::helpers::{TypeModifier, type_modifier_marker}; use ruff_python_ast::{Expr, Stmt}; @@ -47,6 +54,11 @@ impl LiteralStringRewrites { .map(|range| (*range, self.literal_string.clone())) } + /// the same rewrites, as the substitutions a lowering that writes a type whole applies + pub(crate) fn substitutions(&self) -> impl Iterator + '_ { + self.edits() + } + /// whether `range` falls inside a rewritten marker. The blanking pass emits /// a collapse edit for every marker keyword; for a marker rewritten here /// that edit is subsumed, and letting both through would leave two plain @@ -83,6 +95,11 @@ struct Collector<'a> { } impl TypeExprVisitor for Collector<'_> { + /// a type one lowering writes whole writes the substitutions found inside it + fn descends_into_wholly_lowered(&self) -> bool { + true + } + fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { if let Some((TypeModifier::Literal, _)) = type_modifier_marker(expr) && self.types.is_literal_string(expr) @@ -150,6 +167,27 @@ mod tests { ); } + /// a type another lowering writes whole — a tuple type, a callable arrow — writes the + /// `LiteralString` inside it too + #[test] + fn inside_a_type_written_whole() { + check( + "a: (literal str, int)\n", + indoc! {" + from typing_extensions import LiteralString + a: tuple[LiteralString, int] + "}, + ); + check( + "def f(g: (literal str) -> literal str) -> None: ...\n", + indoc! {" + from typing import Callable + from typing_extensions import LiteralString + def f(g: Callable[[LiteralString], LiteralString]) -> None: ... + "}, + ); + } + #[test] fn other_modifiers_still_erase() { // only `literal str` has a stdlib spelling; the rest are compile-time diff --git a/crates/by_transforms/src/transforms/match_polyfill.rs b/crates/by_transforms/src/transforms/match_polyfill.rs index 980369ecf6..37a00f53db 100644 --- a/crates/by_transforms/src/transforms/match_polyfill.rs +++ b/crates/by_transforms/src/transforms/match_polyfill.rs @@ -2,9 +2,10 @@ //! //! `match` is python 3.10 syntax, so a target below that cannot even parse a //! file containing one. Every `match` in the emitted python — the ones the -//! author wrote, and the ones other lowerings produce (`let` destructuring, -//! `if let`, statement expressions, enum exhaustiveness) — is rewritten here -//! into an `if`/`elif` chain whose conditions do the matching: +//! author wrote, and the ones statement expressions and enum exhaustiveness +//! produce — is rewritten here into an `if`/`elif` chain whose conditions do +//! the matching. A destructuring (`let`, `if let`) is written as a `match` too, +//! but the parser holds it to 3.10, so none reaches this lowering: //! //! ```text //! match point: if [__by_match_0__ := (point)]: diff --git a/crates/by_transforms/src/transforms/mod.rs b/crates/by_transforms/src/transforms/mod.rs index ceb26d7734..657d0dc057 100644 --- a/crates/by_transforms/src/transforms/mod.rs +++ b/crates/by_transforms/src/transforms/mod.rs @@ -25,6 +25,7 @@ pub(crate) mod dynamic_keyword; pub(crate) mod empty_declarations; pub(crate) mod enums; pub(crate) mod erased_union; +pub(crate) mod except_polyfill; pub(crate) mod export_import; pub(crate) mod extension; pub(crate) mod flexible_keyword; @@ -81,6 +82,7 @@ pub(crate) mod static_resource; pub(crate) mod string_tag; pub(crate) mod super_keyword; pub(crate) mod symbolic_type_op; +pub(crate) mod synthesized_types; pub(crate) mod template_expression; pub(crate) mod template_type; pub(crate) mod top_star; diff --git a/crates/by_transforms/src/transforms/mutable_defaults.rs b/crates/by_transforms/src/transforms/mutable_defaults.rs index 62d649ee76..a9dcfaf081 100644 --- a/crates/by_transforms/src/transforms/mutable_defaults.rs +++ b/crates/by_transforms/src/transforms/mutable_defaults.rs @@ -66,7 +66,7 @@ use ruff_python_ast::{Expr, ParameterWithDefault, Stmt, StmtFunctionDef}; use ruff_text_size::{Ranged, TextRange, TextSize}; use super::ast_driver::{Fragment, PassContext, TypeAwarePass}; -use super::repeated_underscore::{WrittenNames, reads_underscore, rebound_underscore}; +use super::repeated_underscore::{WrittenNames, rebound_underscore}; use super::source_util::{ PrologueStatement, body_prologue, first_body_statement, python_string_literal, standalone_expression, @@ -94,10 +94,11 @@ pub(crate) enum Guard { /// the name the builtin `TypeError` is written under type_error: String, }, - /// bind `_` to the parameter that was the first `_`, for a body that reads it. a + /// bind `_` to the parameter that was the first `_`, for a body that refers to it. a /// repeated `_` that takes its name from an overridden method leaves no parameter - /// named `_`, and a read of `_` is the first one's value — after its default is - /// settled, so this comes after the guards above + /// named `_`, and a read of `_` is the first one's value, a `del _` deletes it and a + /// nested `nonlocal _` finds it — after its default is settled, so this comes after the + /// guards above Rebind { parameter: String }, } @@ -426,7 +427,7 @@ pub(crate) fn parameter_guards( } if !is_stub && let Some(parameter) = rebound_underscore(params, written_names) - && reads_underscore(&f.body) + && types.refers_to_own_underscore(f) { guards.push(Guard::Rebind { parameter: parameter.to_string(), diff --git a/crates/by_transforms/src/transforms/protocol_type.rs b/crates/by_transforms/src/transforms/protocol_type.rs index 30d1d7710a..0bb249257f 100644 --- a/crates/by_transforms/src/transforms/protocol_type.rs +++ b/crates/by_transforms/src/transforms/protocol_type.rs @@ -31,39 +31,32 @@ use std::fmt::Write as _; use std::hash::Hash; -use std::collections::HashMap; - -use indexmap::IndexMap; -use ruff_diagnostics::{Edit, Fix}; use ruff_python_ast::{Expr, ExprProtocolType, Stmt}; -use ruff_text_size::{Ranged, TextRange}; +use ruff_text_size::Ranged; -use crate::config::Config; use crate::type_info::TypeInfo; use super::ast_driver::{PassContext, TypeAwarePass}; -use super::callable::CallableSyntax; -use super::quotes::WrittenQuotes; use super::repeated_underscore::{WrittenNames, protocol_method_receiver}; -use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions_skipping}; +use super::synthesized_types::{Namer, SynthesizedKind}; /// One inline protocol shape: the rendered class-body lines, in source order. /// /// Rendered text rather than the AST is what identifies a shape, so two /// occurrences that spell the same members the same way share one class. #[derive(Clone, Debug, Eq, PartialEq, Hash)] -struct Shape { +pub(crate) struct Shape { members: Vec, } impl Shape { /// the name of the class declaring this shape, one the module has no other use of - fn class_name(&self, written: WrittenNames) -> String { + pub(crate) fn class_name(&self, written: WrittenNames) -> String { written.synthesized_class("_Protocol_", self) } /// the class declaring this shape as `name`, `typing.Protocol` written as `protocol` - fn class_def(&self, name: &str, protocol: &str) -> String { + pub(crate) fn class_def(&self, name: &str, protocol: &str) -> String { let mut out = format!("class {name}({protocol}):\n"); if self.members.is_empty() { out.push_str(" pass\n"); @@ -76,445 +69,106 @@ impl Shape { } } -pub(crate) struct ProtocolTypePass<'src> { - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - config: crate::Config, -} - -impl<'src> ProtocolTypePass<'src> { - pub(crate) fn new( - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - config: crate::Config, - ) -> Self { - Self { - source, - written, - quotes, - config, - } - } -} - -struct ProtocolTypeLowering<'src> { - source: &'src str, - written: WrittenNames<'src>, - /// where the member types are written as strings - quotes: &'src WrittenQuotes, - /// One lowerer drives every member type in the run, so its imports and the - /// `_Callable_*` classes it synthesizes can be collected at the end. The - /// `callable` pass deliberately does not descend into an inline protocol, - /// so nothing else emits them - callable: CallableSyntax<'src>, - edits: Vec, - errors: Vec, - /// Insertion-ordered so a class is emitted before any later class whose - /// members mention it (the visitor rewrites nested protocols first) - shapes: IndexMap, - /// Typevar renames in effect at each generic scope. Python < 3.12 has no - /// native PEP 695, so the generics polyfill renames `T` to `_T` at module - /// scope — a hoisted class body naming the original would resolve to nothing - typevar_scopes: Vec<(TextRange, HashMap)>, - needs_import: bool, -} - -impl<'src> ProtocolTypeLowering<'src> { - fn new( - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - types: &'src dyn TypeInfo, - claimed: &'src [TextRange], - config: &Config, - ) -> Self { - Self { - source, - written, - quotes, - callable: CallableSyntax::new(source, written, quotes, config) - .with_types(types) - .with_claimed_ranges(claimed), - edits: Vec::new(), - errors: Vec::new(), - shapes: IndexMap::new(), - typevar_scopes: Vec::new(), - needs_import: false, - } - } - - fn class_defs(&self) -> String { - let mut out = String::new(); - let protocol = self.written.imported("typing", "Protocol"); - for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name, &protocol)); - out.push('\n'); - } - out - } - - /// Lower a member's type expression to Python source. - /// - /// A nested inline protocol resolves to the class it was assigned — at any - /// depth, since the substitution is registered on the shared lowerer, whose - /// recursion reaches every leaf. - fn render_type(&mut self, expr: &Expr) -> String { - let rendered = self.callable.lower_type_expr(expr); - self.apply_typevar_renames(expr.range(), &rendered) - } - - /// Substitute every whole-word typevar reference in `rendered` with the - /// mangled module-scope name the generics polyfill gives it, for the scopes - /// enclosing `range`. - fn apply_typevar_renames(&self, range: TextRange, rendered: &str) -> String { - let mut renames: HashMap<&str, &str> = HashMap::new(); - for (scope, frame) in &self.typevar_scopes { - if scope.contains_range(range) { - renames.extend(frame.iter().map(|(k, v)| (k.as_str(), v.as_str()))); +/// The shape of an inline protocol, each member's type written as python, a nested inline +/// protocol or any other type the walk named before it as its class — at any depth, since the +/// name is a substitution on the shared lowerer, whose recursion reaches every leaf. +pub(crate) fn shape(namer: &mut Namer, protocol: &ExprProtocolType) -> Shape { + let mut members = Vec::with_capacity(protocol.members.len()); + for member in &protocol.members { + match member { + Expr::Named(named) => { + let Some(name) = named.target.as_name_expr() else { + continue; + }; + let ty = render_type(namer, &named.value); + let ty = namer.quotes.quote(&ty); + // a member that only has to be readable is a getter-only property, + // which is how a protocol class body says read-only + members.push(if named.is_let { + let property = namer.written.builtin("property"); + format!( + "@{property}\n def {name}(self) -> {ty}: ...", + name = name.id + ) + } else { + format!("{name}: {ty}", name = name.id) + }); } - } - if renames.is_empty() { - return rendered.to_owned(); - } - super::source_util::rename_identifiers(rendered, &renames) - } - - /// Render a method member as a `def` line, splitting off the receiver — - /// which is a parameter name rather than a type, and which the parser - /// marked as a label to keep it out of name resolution. - fn render_method(&mut self, method: &ruff_python_ast::ExprProtocolMethod) -> Option { - let Expr::CallableType(signature) = method.signature.as_ref() else { - return None; - }; - let receiver = protocol_method_receiver(signature).map(|name| name.id.as_str()); - let params = self.callable.render_protocol_params( - signature, - true, - receiver.unwrap_or("self"), - // a protocol method's receiver is its `self` parameter, never an implicit one - None, - ); - let returns = self.callable.render_return(signature); - let line = format!( - "def {name}({params}) -> {returns}: ...", - name = method.name.id, - returns = self.quotes.quote(&returns), - ); - Some(self.apply_typevar_renames(method.range(), &line)) - } - - fn extract_shape(&mut self, protocol: &ExprProtocolType) -> Shape { - let mut members = Vec::with_capacity(protocol.members.len()); - for member in &protocol.members { - match member { - Expr::Named(named) => { - let Some(name) = named.target.as_name_expr() else { - continue; - }; - let ty = self.quotes.quote(&self.render_type(&named.value)); - // a member that only has to be readable is a getter-only property, - // which is how a protocol class body says read-only - members.push(if named.is_let { - let property = self.written.builtin("property"); - format!( - "@{property}\n def {name}(self) -> {ty}: ...", - name = name.id - ) - } else { - format!("{name}: {ty}", name = name.id) - }); - } - Expr::ProtocolMethod(method) => match self.render_method(method) { - Some(rendered) => members.push(rendered), - None => self.errors.push(format!( - "inline protocol method `{}` has no parameter list", - method.name.id - )), - }, - // `**Kwargs` contributes no members: the pack's fields are only - // known once the enclosing generic is specialized, and python - // erases type arguments anyway. ty splices them in statically - Expr::Starred(_) => {} - // the parser only ever produces the three shapes above; this - // keeps a future one from being silently dropped - other => self.errors.push(format!( - "invalid inline protocol member `{}`", - self.source - .get(std::ops::Range::::from(other.range())) - .unwrap_or("") + Expr::ProtocolMethod(method) => match render_method(namer, method) { + Some(rendered) => members.push(rendered), + None => namer.errors.push(format!( + "inline protocol method `{}` has no parameter list", + method.name.id )), - } - } - Shape { members } - } - - fn rewrite(&mut self, protocol: &ExprProtocolType) { - let shape = self.extract_shape(protocol); - let name = if let Some(existing) = self.shapes.get(&shape) { - existing.clone() - } else { - let name = shape.class_name(self.written); - self.shapes.insert(shape, name.clone()); - name - }; - // register on the shared lowerer so an enclosing member type resolves the - // nested protocol wherever it sits — inside a subscript, a union, a - // callable arrow — not only when it is the whole member type - self.callable - .add_substitution(protocol.range(), name.clone()); - self.needs_import = true; - self.edits.push(Fix::safe_edit(Edit::range_replacement( - name, - protocol.range(), - ))); - } - - /// Rewrite every inline protocol inside `expr`, innermost first, so an - /// enclosing protocol's members can name the class a nested one produced. - fn visit_nested(&mut self, expr: &Expr) { - match expr { - Expr::ProtocolType(protocol) => { - for member in &protocol.members { - self.visit_nested(member); - } - self.rewrite(protocol); - } - Expr::ProtocolMethod(method) => self.visit_nested(&method.signature), - Expr::CallableType(callable) => { - for arg in &callable.args { - self.visit_nested(arg); - } - self.visit_nested(&callable.returns); - } - Expr::Named(named) => self.visit_nested(&named.value), - Expr::BinOp(binop) => { - self.visit_nested(&binop.left); - self.visit_nested(&binop.right); - } - Expr::BoolOp(boolop) => { - for value in &boolop.values { - self.visit_nested(value); - } - } - Expr::UnaryOp(unary) => self.visit_nested(&unary.operand), - Expr::Subscript(subscript) => self.visit_nested(&subscript.slice), - Expr::Starred(starred) => self.visit_nested(&starred.value), - Expr::Tuple(tuple) => { - for elt in &tuple.elts { - self.visit_nested(elt); - } - } - Expr::List(list) => { - for elt in &list.elts { - self.visit_nested(elt); - } - } - _ => {} - } - } -} - -impl TypeExprVisitor for ProtocolTypeLowering<'_> { - fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { - // `visit_nested` is a deep rewriter that already knows every container - // an inline protocol can hide in, so the walker must not descend and - // process the same node twice - self.visit_nested(expr); - Recurse::Stop - } -} - -/// Lowers inline protocols that appear in a value position (`x = protocol(a: -/// int)`). `protocol(...)` is never valid python wherever it appears, so it has -/// to be replaced everywhere, not just in the annotation slots the type-position -/// walker visits. -struct ValueProtocolWalker<'a, 'src> { - inner: &'a mut ProtocolTypeLowering<'src>, -} - -impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for ValueProtocolWalker<'_, '_> { - fn visit_expr(&mut self, expr: &'ast Expr) { - if matches!(expr, Expr::ProtocolType(_)) { - self.inner.visit_nested(expr); - return; - } - ruff_python_ast::visitor::walk_expr(self, expr); - } -} - -/// Collects the typevar renames the PEP 695 polyfill applies, keyed by the -/// range of the generic scope that declares them. -struct TypevarScopeWalker<'a> { - polyfilled: &'a super::generics::PolyfilledTypeParams, - scopes: &'a mut Vec<(TextRange, HashMap)>, -} - -impl TypevarScopeWalker<'_> { - fn record(&mut self, range: TextRange, type_params: Option<&ruff_python_ast::TypeParams>) { - if let Some(frame) = type_params.and_then(|tp| self.polyfilled.hoisted_renames(tp)) - && !frame.is_empty() - { - self.scopes.push((range, frame)); - } - } -} - -impl<'ast> ruff_python_ast::visitor::Visitor<'ast> for TypevarScopeWalker<'_> { - fn visit_stmt(&mut self, stmt: &'ast Stmt) { - match stmt { - Stmt::ClassDef(class) => self.record(class.range(), class.type_params.as_deref()), - Stmt::FunctionDef(function) => { - self.record(function.range(), function.type_params.as_deref()); - } - _ => {} + }, + // `**Kwargs` contributes no members: the pack's fields are only + // known once the enclosing generic is specialized, and python + // erases type arguments anyway. ty splices them in statically + Expr::Starred(_) => {} + // the parser only ever produces the three shapes above; this + // keeps a future one from being silently dropped + other => namer.errors.push(format!( + "invalid inline protocol member `{}`", + namer.src(other.range()) + )), } - ruff_python_ast::visitor::walk_stmt(self, stmt); } + Shape { members } } -/// Drive the lowering over `stmts`, visiting both the type positions the walker -/// knows about and any inline protocol left in a value position. -fn lower<'src>( - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - types: &'src dyn TypeInfo, - claimed: &'src [TextRange], - stmts: &[Stmt], - config: &crate::Config, -) -> ProtocolTypeLowering<'src> { - let mut inner = ProtocolTypeLowering::new(source, written, quotes, types, claimed, config); - let polyfilled = super::generics::PolyfilledTypeParams::decide(source, written, stmts, config); - { - let mut walker = TypevarScopeWalker { - polyfilled: &polyfilled, - scopes: &mut inner.typevar_scopes, - }; - for stmt in stmts { - ruff_python_ast::visitor::Visitor::visit_stmt(&mut walker, stmt); - } - } - walk_type_positions_skipping(stmts, Some(types), claimed, &mut inner); - let mut walker = ValueProtocolWalker { inner: &mut inner }; - for stmt in stmts { - ruff_python_ast::visitor::Visitor::visit_stmt(&mut walker, stmt); - } - inner +/// Lower a member's type expression to Python source. +fn render_type(namer: &mut Namer, expr: &Expr) -> String { + let rendered = namer.lower(expr); + namer.read_type_parameters(expr.range(), &rendered) } -/// The preamble and edits needed to lower every inline protocol still present in -/// post-transform output. -/// -/// A node an AST pass rewrote is printed from the AST, which can re-emit the -/// surface `protocol(...)` syntax after this pass's own text edits were computed -/// — so the driver runs the lowering again over the spliced output to catch them. -/// Returns `None` when nothing is left to lower. -pub(crate) fn cleanup( - source: &str, - written: WrittenNames, - quotes: &WrittenQuotes, - types: &dyn TypeInfo, - stmts: &[Stmt], - config: &crate::Config, -) -> Result, String)>, String> { - let mut inner = lower(source, written, quotes, types, &[], stmts, config); - if let Some(error) = inner.errors.first() { - return Err(error.clone()); - } - if !inner.needs_import { - return Ok(None); - } - // the earlier run over the pre-splice source already emitted a class (and - // its imports) for the protocol whose edit was then dropped, so only what - // the spliced output is actually missing gets prepended - let mut preamble = String::new(); - let source_lines: Vec<&str> = source.lines().collect(); - // matched as a run of whole lines: a bare `contains` would let - // `from typing import ProtocolFoo` suppress the `Protocol` import we need - let push_missing = |preamble: &mut String, entry: &str| { - let entry_lines: Vec<&str> = entry.lines().collect(); - let present = !entry_lines.is_empty() - && source_lines - .windows(entry_lines.len()) - .any(|window| window == entry_lines.as_slice()); - if !present { - preamble.push_str(entry); - preamble.push('\n'); - } +/// Render a method member as a `def` line, splitting off the receiver — +/// which is a parameter name rather than a type, and which the parser +/// marked as a label to keep it out of name resolution. +fn render_method( + namer: &mut Namer, + method: &ruff_python_ast::ExprProtocolMethod, +) -> Option { + let Expr::CallableType(signature) = method.signature.as_ref() else { + return None; }; - push_missing(&mut preamble, &written.import_from("typing", &["Protocol"])); - let (imports, helpers) = inner.callable.take_requirements(); - for line in imports - .into_iter() - .chain(crate::runtime_entries(config, &helpers, source)) - { - push_missing(&mut preamble, line.trim_end_matches('\n')); - } - for defs in [inner.callable.class_defs().to_owned(), inner.class_defs()] { - for class_def in defs.split_inclusive("\n\n") { - push_missing(&mut preamble, class_def.trim_end_matches('\n')); - } - } - Ok(Some((inner.edits, preamble))) + let receiver = protocol_method_receiver(signature).map(|name| name.id.as_str()); + let params = namer.callable.render_protocol_params( + signature, + true, + receiver.unwrap_or("self"), + // a protocol method's receiver is its `self` parameter, never an implicit one + None, + ); + let returns = namer.callable.render_return(signature); + let line = format!( + "def {name}({params}) -> {returns}: ...", + name = method.name.id, + returns = namer.quotes.quote(&returns), + ); + Some(namer.read_type_parameters(method.range(), &line)) } -impl TypeAwarePass for ProtocolTypePass<'_> { +/// replaces each inline protocol with the `Protocol` class it names, wherever it stands +pub(crate) struct ProtocolTypePass; + +impl TypeAwarePass for ProtocolTypePass { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::ProtocolType) } - /// an inline protocol it replaces moves into a class, whose member types the shared - /// type-expression lowerer prints, a `typeof` among them + /// an inline protocol it replaces moves into a class, whose member types the synthesized + /// type namer prints fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[ - super::ast_driver::Lowering::Callable, - super::ast_driver::Lowering::OptionalType, - super::ast_driver::Lowering::LiteralType, - super::ast_driver::Lowering::JustFloat, - super::ast_driver::Lowering::DynamicKeyword, - super::ast_driver::Lowering::FloatConst, - super::ast_driver::Lowering::Typeof, - ] + super::synthesized_types::CLASS_BODY } - fn run(&self, stmts: &[Stmt], types: &dyn TypeInfo, ctx: &mut PassContext) { - let claimed = ctx.claimed_type_op_ranges.clone(); - let mut inner = lower( - self.source, - self.written, - self.quotes, - types, - &claimed, - stmts, - &self.config, - ); - - if inner.needs_import { - ctx.required_imports - .push(self.written.import_from("typing", &["Protocol"])); - // synthesized class defs are raw multi-line python source; push - // them as a single non-`from` line so `merge_from_imports` leaves - // them untouched and they land in the preamble verbatim. the - // member lowerer's own classes come first: an inline protocol's - // member may name one - let defs = format!("{}{}", inner.callable.class_defs(), inner.class_defs()); - if !defs.is_empty() { - ctx.required_imports - .push(defs.trim_end_matches('\n').to_owned()); - } - } - let (imports, helpers) = inner.callable.take_requirements(); - ctx.required_imports.extend(imports); - ctx.runtime.extend(helpers); - for fix in inner.edits { - for edit in fix.edits() { - ctx.text_edits - .push((edit.range(), edit.content().unwrap_or_default().to_owned())); - } + fn run(&self, _stmts: &[Stmt], _types: &dyn TypeInfo, ctx: &mut PassContext) { + // the class each one names, and the declarations, come from the namer's one walk over + // the module, which named every kind of synthesized type together + for (range, name) in ctx.synthesized.of_kind(SynthesizedKind::Protocol) { + ctx.text_edits.push((range, name)); } - ctx.errors.extend(inner.errors); } } @@ -553,25 +207,26 @@ mod tests { class _Protocol_66269af6(Protocol): a: "int" def m(self) -> "str": ... + def f(x: _Protocol_66269af6) -> None: ... "#}, ); } + /// a member's type carries a type modifier as any other type does. `final` has no python + /// spelling and is written as the type it marks, without the blanks the keyword was replaced + /// with in the source, which a quoted type cannot start with #[test] - fn a_let_member_is_a_getter_only_property() { - check( - indoc! {" - def f(x: protocol(let a: int; b: str)) -> None: ... - "}, - indoc! {r#" - from typing import Protocol - class _Protocol_4e1bfe56(Protocol): - @property - def a(self) -> "int": ... - b: "str" - def f(x: _Protocol_4e1bfe56) -> None: ... - "#}, + fn a_type_modifier_on_a_member() { + let out = transpile( + "def f(x: protocol(a: final int; def m(self) -> final str)) -> None: ...\n", + &Config::test_default(), + ) + .unwrap(); + assert!(out.contains(" a: \"int\"\n"), "got: {out}"); + assert!( + out.contains(" def m(self) -> \"str\": ...\n"), + "got: {out}" ); } @@ -590,6 +245,7 @@ mod tests { from typing import Protocol class _Protocol_2abdf9ec(Protocol): a: "int" + from typing import Protocol def f(x: _Protocol_2abdf9ec) -> None: ... @@ -624,6 +280,7 @@ mod tests { from typing import Protocol class _Protocol_6a4f11ec(Protocol): x: "int" + a: _Protocol_6a4f11ec b: _Protocol_6a4f11ec "#}, @@ -643,6 +300,7 @@ mod tests { from typing import Protocol class _Protocol_58c79e45(Protocol): pass + class A[**Kwargs]: def get(self) -> _Protocol_58c79e45: ... "}, @@ -662,6 +320,7 @@ mod tests { class _Protocol_7ea2762f(Protocol): inner: "_Protocol_6a4f11ec" + a: _Protocol_7ea2762f "#}, ); @@ -691,6 +350,7 @@ mod tests { class _Protocol_7729cacb(Protocol): f: "Callable[[_Protocol_9ac2fcf2], str]" + a: _Protocol_c1aa572a c: _Protocol_534e77b4 d: _Protocol_7729cacb @@ -712,6 +372,7 @@ mod tests { class _Protocol_72797bc7(Protocol): a: "_T" def m(self, x: "_T") -> "_T": ... + _T = TypeVar("_T") class A(Generic[_T]): def get(self) -> _Protocol_72797bc7: ... @@ -765,6 +426,7 @@ mod tests { from typing import Protocol, TypeVar, Generic class _Protocol_fd816244(Protocol): a: "_T_1" + _T = TypeVar("_T", bound=int) class A(Generic[_T]): ... _T_1 = TypeVar("_T_1") @@ -774,6 +436,34 @@ mod tests { ); } + #[test] + fn a_let_member_is_a_getter_only_property() { + check( + indoc! {" + def f(x: protocol(let a: int; b: str)) -> None: ... + "}, + indoc! {r#" + from typing import Protocol + class _Protocol_4e1bfe56(Protocol): + @property + def a(self) -> "int": ... + b: "str" + + def f(x: _Protocol_4e1bfe56) -> None: ... + "#}, + ); + } + + /// the synthesized class is written into the preamble and names `Protocol` as it is + /// defined, so that import stays above it even when the module imports `Protocol` itself — + /// folding it into the module's own line would leave the class with no such name + /// a type alias's own parameter is declared by the polyfill too + #[test] + fn typevar_member_of_a_generic_alias() { + let out = transpile("type G[T] = protocol(a: T)\n", &Config::test_default()).unwrap(); + assert!(out.contains(" a: \"_T\"\n"), "got:\n{out}"); + } + /// a defaulted list is polyfilled on 3.12, which has native syntax for one without a /// default #[test] @@ -788,6 +478,7 @@ mod tests { from typing_extensions import TypeVar class _Protocol_ea950896(Protocol): a: "_T" + _T = TypeVar("_T", default=int) class A(Generic[_T]): def get(self) -> _Protocol_ea950896: ... @@ -805,6 +496,7 @@ mod tests { from typing import Protocol class _Protocol_84b8ccdc(Protocol): def f(self, x: "int", /, *args: "str", **kw: "int") -> "str": ... + a: _Protocol_84b8ccdc "#}, ); @@ -825,6 +517,7 @@ mod tests { x: "int | None" y: "Intersection[int, str]" def f(self) -> "Callable[[int], str]": ... + a: _Protocol_366f1172 "#}, ); @@ -846,6 +539,7 @@ mod tests { class _Protocol_ade07c7b(Protocol): a: "TypeOf[d]" def m(self, n: "int") -> "str": ... + d: dict[str, int] = {} x: _Protocol_ade07c7b = ... "#}, diff --git a/crates/by_transforms/src/transforms/repeated_underscore.rs b/crates/by_transforms/src/transforms/repeated_underscore.rs index a4e87ee078..529e175422 100644 --- a/crates/by_transforms/src/transforms/repeated_underscore.rs +++ b/crates/by_transforms/src/transforms/repeated_underscore.rs @@ -10,8 +10,9 @@ //! //! which names and where the `/` goes is ty's answer, the one its signature of the //! definition is built from (`ty_python_semantic::types::repeated_underscore`), so a -//! call ty accepts is one the python accepts. a shape ty refuses is refused here too. -//! references to `_` inside the body are left alone and resolve to the first parameter +//! call ty accepts is one the python accepts. a shape ty refuses is refused here too, and +//! so is a read of `_` that may be one of the repeated parameters: python binds the name to +//! the first of them and ty to the last use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -90,6 +91,52 @@ pub(crate) fn collect(suite: &[Stmt], types: &dyn TypeInfo, slash: bool) -> Unde collector.lowerings } +/// a refusal for every read of `_` in `suite` that may read one of a parameter list's +/// repeated `_`s — ty's answer, the one it reports, so the two cannot disagree. python binds +/// `_` to the first of them and ty to the last, so the read has no one meaning to lower +pub(crate) fn refused_reads(suite: &[Stmt], types: &dyn TypeInfo) -> Vec { + struct Reads<'a> { + types: &'a dyn TypeInfo, + refusals: Vec, + } + impl Reads<'_> { + fn ask(&mut self, name: &ruff_python_ast::ExprName) { + if name.id == "_" + && let Some(read) = self.types.repeated_underscore_read(name) + { + self.refusals + .push(format!("{}: {}", read.message(), read.help())); + } + } + } + impl Visitor<'_> for Reads<'_> { + fn visit_stmt(&mut self, stmt: &Stmt) { + // `_ += 1` reads `_` before it binds it + if let Stmt::AugAssign(node) = stmt + && let Expr::Name(name) = node.target.as_ref() + { + self.ask(name); + } + visit_walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &Expr) { + if let Expr::Name(name) = expr + && name.ctx.is_load() + { + self.ask(name); + } + visit_walk_expr(self, expr); + } + } + let mut reads = Reads { + types, + refusals: Vec::new(), + }; + reads.visit_body(suite); + reads.refusals +} + /// ty's answer for every parameter list in `suite` that repeats `_`, for a native build of /// the module `model` is of. that build runs on the interpreter it is compiled for, which /// has the `/` @@ -839,36 +886,6 @@ pub(crate) fn rebound_underscore(parameters: &Parameters, written: WrittenNames) .filter(|name| name != "_") } -/// whether `body` reads `_`, anywhere in it -pub(crate) fn reads_underscore(body: &[Stmt]) -> bool { - struct Reads { - found: bool, - } - impl Visitor<'_> for Reads { - fn visit_stmt(&mut self, stmt: &Stmt) { - if let Stmt::AugAssign(node) = stmt - && matches!(node.target.as_ref(), Expr::Name(name) if name.id == "_") - { - self.found = true; - } - visit_walk_stmt(self, stmt); - } - - fn visit_expr(&mut self, expr: &Expr) { - if let Expr::Name(name) = expr - && name.id == "_" - && !name.ctx.is_store() - { - self.found = true; - } - visit_walk_expr(self, expr); - } - } - let mut reads = Reads { found: false }; - reads.visit_body(body); - reads.found -} - pub(crate) struct RepeatedUnderscore<'src> { source: &'src str, written: WrittenNames<'src>, @@ -1212,8 +1229,8 @@ mod tests { #[test] fn two_underscores() { check( - "def f(_, _):\n print(_)\n", - "def f(_, _2, /):\n print(_)\n", + "def f(_, _):\n print(1)\n", + "def f(_, _2, /):\n print(1)\n", ); } @@ -1221,8 +1238,8 @@ mod tests { #[test] fn three_underscores() { check( - "def g(_, _, _, x):\n return _\n", - "def g(_, _2, _3, /, x):\n return _\n", + "def g(_, _, _, x):\n return x\n", + "def g(_, _2, _3, /, x):\n return x\n", ); } @@ -1234,8 +1251,8 @@ mod tests { #[test] fn nested_function() { check( - "def outer(_, _):\n def inner(_, _):\n return _\n return _\n", - "def outer(_, _2, /):\n def inner(_, _2, /):\n return _\n return _\n", + "def outer(_, _):\n def inner(_, _):\n return 1\n return inner\n", + "def outer(_, _2, /):\n def inner(_, _2, /):\n return 1\n return inner\n", ); } @@ -1277,8 +1294,8 @@ mod tests { #[test] fn vararg_underscore() { check( - "def f(a, _, *_):\n return _\n", - "def f(a, _, *_2):\n return _\n", + "def f(a, _, *_):\n return a\n", + "def f(a, _, *_2):\n return a\n", ); } @@ -1346,8 +1363,8 @@ mod tests { #[test] fn existing_collision() { check( - "def f(_, _2, _, /):\n return _\n", - "def f(_, _2, _3, /):\n return _\n", + "def f(_, _2, _, /):\n return _2\n", + "def f(_, _2, _3, /):\n return _2\n", ); } @@ -1408,10 +1425,10 @@ mod tests { ); } - /// no parameter is named `_` once they take the base's names, so a body that reads `_` - /// is handed the first of them, as it was before + /// no parameter is named `_` once they take the base's names, so a body that unbinds + /// `_` is handed the first of them to unbind #[test] - fn an_override_that_reads_underscore_binds_it() { + fn an_override_that_deletes_underscore_binds_it() { check( indoc! {" class A: @@ -1420,7 +1437,8 @@ mod tests { class B(A): override def f(self, _: int, _: int) -> int: - return _ + del _ + return 1 "}, indoc! {" from typing_extensions import override @@ -1432,35 +1450,75 @@ mod tests { @override def f(self, x: int, y: int) -> int: _ = x - return _ + del _ + return 1 + "}, + ); + } + + /// a nested function that declares `_` `nonlocal` refers to the override's `_` even when + /// it only writes it, so the override binds `_` for it to find + #[test] + fn an_override_whose_nested_function_declares_underscore_nonlocal_binds_it() { + check( + indoc! {" + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + override def f(self, _: int, _: int) -> int: + def g(): + nonlocal _ + _ = 1 + g() + return 1 + "}, + indoc! {" + from typing_extensions import override + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + @override + def f(self, x: int, y: int) -> int: + _ = x + def g(): + nonlocal _ + _ = 1 + g() + return 1 "}, ); } /// a base that repeats `_` numbers its own, and the override takes those names. its - /// first is `_`, which the body's `_` already reads + /// first is `_`, so nothing has to bind `_` in the body #[test] fn an_override_of_numbered_underscores_keeps_them() { check( indoc! {" class A: def f(self, _: int, _: int) -> int: - return _ + return 1 class B(A): override def f(self, _: int, _: int) -> int: - return _ + del _ + return 2 "}, indoc! {" from typing_extensions import override class A: def f(self, _: int, _2: int, /) -> int: - return _ + return 1 class B(A): @override def f(self, _: int, _2: int, /) -> int: - return _ + del _ + return 2 "}, ); } @@ -1477,7 +1535,7 @@ mod tests { class B(A): override def f(self, _: int, _: int) -> int: - return _ + return 1 "}, indoc! {" from typing_extensions import override @@ -1488,8 +1546,7 @@ mod tests { class B(A): @override def f(self, x: int, _: int) -> int: - _ = x - return _ + return 1 "}, ); } @@ -1514,4 +1571,103 @@ mod tests { parameter takes its name from the method it overrides; rename the `x` the body reads" ); } + + /// python binds `_` to the first of the parameters and ty to the last, so a read the + /// parameters may reach is refused, through ty's answer, wherever it is + #[test] + fn a_read_of_a_repeated_underscore_is_refused() { + let refusal = "reading a repeated `_` parameter of `f`: several parameters are named `_`, \ + so a read cannot say which: give the one the body reads a name of its own"; + assert_eq!( + refused("def f(_: int, _: str) -> str:\n return _\n"), + refusal + ); + assert_eq!( + refused("def f(_: int, _: int) -> None:\n _ += 1\n"), + refusal + ); + assert_eq!( + refused(indoc! {" + def f(_: int, _: int) -> None: + def g() -> int: + return _ + "}), + refusal + ); + assert_eq!( + refused("f = lambda _, _: _\n"), + "reading a repeated `_` parameter of a lambda: several parameters are named `_`, \ + so a read cannot say which: give the one the body reads a name of its own" + ); + assert_eq!( + refused(indoc! {" + class A: + def f(self, x: int, y: int) -> int: + return x + + class B(A): + override def f(self, _: int, _: int) -> int: + return _ + "}), + refusal + ); + } + + /// a read only a binding the body makes reaches reads that binding, and `del _` reads + /// no value, so neither is refused + #[test] + fn a_read_of_a_rebound_underscore_is_kept() { + check( + "def f(_: int, _: int) -> int:\n _ = 5\n return _\n", + "def f(_: int, _2: int, /) -> int:\n _ = 5\n return _\n", + ); + check( + "def f(_: int, _: int) -> None:\n del _\n", + "def f(_: int, _2: int, /) -> None:\n del _\n", + ); + } + + /// none of the parameters declares the `_` the body binds, so every way of writing it is + /// left as written: a local the python binds over the first parameter + #[test] + fn a_write_to_a_repeated_underscore_is_kept() { + check( + indoc! {" + def f(_: int, _: str) -> int: + _ = 1 + a, _ = 1, 2 + for _ in range(3): + pass + with open(\"x\") as _: + pass + if (_ := 3) > 0: + return _ + def g() -> int: + nonlocal _ + _ = 7 + return _ + return g() + + h = lambda _, _: (_ := 9) + "}, + indoc! {" + def f(_: int, _2: str, /) -> int: + _ = 1 + a, _ = 1, 2 + for _ in range(3): + pass + with open(\"x\") as _: + pass + if (_ := 3) > 0: + return _ + def g() -> int: + nonlocal _ + _ = 7 + return _ + return g() + + h = lambda _, _2, /: (_ := 9) + "}, + ); + } } diff --git a/crates/by_transforms/src/transforms/rerender.rs b/crates/by_transforms/src/transforms/rerender.rs index c4e47db861..99718241a6 100644 --- a/crates/by_transforms/src/transforms/rerender.rs +++ b/crates/by_transforms/src/transforms/rerender.rs @@ -1846,7 +1846,7 @@ mod tests { fn a_default_guard_names_the_renamed_parameter() { let out = check(indoc! {" def f(_: list[int] = [], _: list[int] = []) -> int: - return len(_) + return 0 "}); assert!(out.contains("if _ is _MISSING"), "got:\n{out}"); assert!( diff --git a/crates/by_transforms/src/transforms/string_tag.rs b/crates/by_transforms/src/transforms/string_tag.rs index 486d2783df..404cd9ebda 100644 --- a/crates/by_transforms/src/transforms/string_tag.rs +++ b/crates/by_transforms/src/transforms/string_tag.rs @@ -31,6 +31,7 @@ use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; use ruff_python_ast::{ ConversionFlag, Expr, ExprCall, InterpolatedStringElement, ModModule, PythonVersion, Stmt, }; +use ruff_python_trivia::CommentRanges; use ruff_text_size::{Ranged, TextRange}; use crate::Config; @@ -41,12 +42,17 @@ use super::template_expression; pub(crate) struct StringTagPass<'src> { source: &'src str, + comments: &'src CommentRanges, config: Config, } impl<'src> StringTagPass<'src> { - pub(crate) fn new(source: &'src str, config: Config) -> Self { - Self { source, config } + pub(crate) fn new(source: &'src str, comments: &'src CommentRanges, config: Config) -> Self { + Self { + source, + comments, + config, + } } } @@ -55,6 +61,7 @@ impl AstPass for StringTagPass<'_> { let native = self.config.min_version >= PythonVersion::PY314; let mut state = State { source: self.source, + comments: self.comments, native, text_edits: Vec::new(), template_edits: Vec::new(), @@ -74,6 +81,7 @@ impl AstPass for StringTagPass<'_> { struct State<'src> { source: &'src str, + comments: &'src CommentRanges, native: bool, text_edits: Vec<(TextRange, String)>, template_edits: Vec<(TextRange, Vec)>, @@ -156,7 +164,11 @@ impl State<'_> { frags.push(Fragment::Src(interp.expression.range())); frags.push(Fragment::Lit(format!( ", {}", - string_repr(template_expression::author_text(self.source, interp)) + string_repr(&template_expression::field_text( + self.source, + self.comments, + interp + )) ))); let conversion = conversion_arg(interp.conversion); let format_spec = interp diff --git a/crates/by_transforms/src/transforms/super_keyword.rs b/crates/by_transforms/src/transforms/super_keyword.rs index a17281ef43..e3b39914a3 100644 --- a/crates/by_transforms/src/transforms/super_keyword.rs +++ b/crates/by_transforms/src/transforms/super_keyword.rs @@ -4,7 +4,18 @@ //! - `super.x` → `super().x` //! - `super[T].x` → `super(.__mro__[.__mro__.index() - 1], ).x` //! where `` is the enclosing class name and `` is the first -//! parameter name of the enclosing method. +//! parameter of the function or lambda the keyword is written in — the +//! receiver zero-argument `super()` reads there, and the one the checker types. +//! +//! either form is the keyword only as an attribute's base. anywhere else `super` and +//! `super[T]` are the name and its subscript, as python reads them. +//! +//! the keyword is always the builtin `super`, so in a module that binds the name `super` +//! itself — a parameter, a global — the call reads the builtin under a name of its own +//! ([`WrittenNames::builtin`]). zero-argument `super()` only works under the name `super`, +//! which is what has the compiler give the method its `__class__` cell, so the bare form +//! is then spelled out with the arguments it stands for: `__class__`, which itself has the +//! compiler make the cell, and the method's first parameter. use std::cell::RefCell; @@ -29,23 +40,31 @@ impl AstPass for SuperKeyword<'_> { fn run(&self, module: &mut ModModule, ctx: &mut PassContext) { let state = State { written: self.written, + builtin_super: self.written.builtin("super"), edits: RefCell::new(Vec::new()), class_stack: RefCell::new(Vec::new()), self_stack: RefCell::new(Vec::new()), + errors: RefCell::new(Vec::new()), }; let mut visitor = &state; for stmt in &module.body { visitor.visit_stmt(stmt); } ctx.text_edits.extend(state.edits.into_inner()); + ctx.errors.extend(state.errors.into_inner()); } } struct State<'src> { written: WrittenNames<'src>, + /// the name the builtin `super` is read by + builtin_super: String, edits: RefCell>, class_stack: RefCell>, - self_stack: RefCell>, + /// the receiver of each function or lambda the walk is inside, innermost last: the name its + /// first positional parameter is bound to, or `None` for one with no such parameter + self_stack: RefCell>>, + errors: RefCell>, } /// the name the receiver is bound to in the python, which a repeated `_` may have changed @@ -61,6 +80,48 @@ fn first_param_name( .map(|p| crate::python_parameter_name(params, &p.parameter, written).to_string()) } +/// the name a lambda's receiver is bound to in the python: its first positional parameter +fn first_lambda_param_name( + lambda: &ruff_python_ast::ExprLambda, + written: WrittenNames, +) -> Option { + let params = lambda.parameters.as_deref()?; + params + .posonlyargs + .first() + .or_else(|| params.args.first()) + .map(|p| crate::python_parameter_name(params, &p.parameter, written).to_string()) +} + +/// whether `base`, an attribute's base, is the keyword: `super` or `super[T]` +fn is_super_keyword(base: &Expr) -> bool { + let name = match base { + Expr::Subscript(subscript) => subscript.value.as_ref(), + other => other, + }; + matches!(name, Expr::Name(name) if name.id.as_str() == "super") +} + +impl State<'_> { + /// the receiver of the function or lambda the walk is inside, when it has one + fn receiver(&self) -> Option { + self.self_stack.borrow().last().cloned().flatten() + } + + /// the builtin `super` called with the arguments the bare keyword stands for + fn bare(&self) -> String { + if self.builtin_super == "super" { + return "super()".to_owned(); + } + match self.receiver() { + Some(receiver) => format!("{}(__class__, {receiver})", self.builtin_super), + // a function with no parameter to read the receiver from fails as zero-argument + // `super()` does there, raising `RuntimeError` + None => format!("{}()", self.builtin_super), + } + } +} + impl<'ast> Visitor<'ast> for &State<'_> { fn visit_stmt(&mut self, stmt: &'ast Stmt) { match stmt { @@ -72,21 +133,44 @@ impl<'ast> Visitor<'ast> for &State<'_> { self.class_stack.borrow_mut().pop(); } Stmt::FunctionDef(f) => { - let pushed = first_param_name(f, self.written); - if let Some(name) = &pushed { - self.self_stack.borrow_mut().push(name.clone()); - } + self.self_stack + .borrow_mut() + .push(first_param_name(f, self.written)); walk_stmt(self, stmt); - if pushed.is_some() { - self.self_stack.borrow_mut().pop(); - } + self.self_stack.borrow_mut().pop(); } _ => walk_stmt(self, stmt), } } fn visit_expr(&mut self, expr: &'ast Expr) { - if let Expr::Subscript(s) = expr + // a lambda reads `super`'s receiver from its own first parameter, as a `def` does + if let Expr::Lambda(lambda) = expr { + self.self_stack + .borrow_mut() + .push(first_lambda_param_name(lambda, self.written)); + walk_expr(self, expr); + self.self_stack.borrow_mut().pop(); + return; + } + // an attribute of the keyword is never assigned or deleted: a `super()` object has no + // attributes of its own, and does not pass a write on to the class it reads past, so the + // python the keyword lowers to raises `AttributeError` whatever the attribute + if let Expr::Attribute(attribute) = expr + && matches!(attribute.ctx, ExprContext::Store | ExprContext::Del) + && is_super_keyword(&attribute.value) + { + self.errors.borrow_mut().push(format!( + "`{}` cannot be assigned or deleted: a `super()` object has no attributes of its \ + own to write", + render_expr(expr) + )); + return; + } + // the targeted keyword is the base of an attribute access, `super[T].x`. anywhere else + // `super[T]` is the subscript of the name, as the checker reads it + if let Expr::Attribute(attribute) = expr + && let Expr::Subscript(s) = attribute.value.as_ref() && let Expr::Name(n) = s.value.as_ref() && n.id.as_str() == "super" { @@ -96,15 +180,21 @@ impl<'ast> Visitor<'ast> for &State<'_> { .last() .cloned() .unwrap_or_default(); - let self_name = self - .self_stack - .borrow() - .last() - .cloned() - .unwrap_or_else(|| "self".to_owned()); + let Some(self_name) = self.receiver() else { + // written as an argument the receiver would have to be some other binding of + // the name, an enclosing function's, which is not the one python reads + self.errors.borrow_mut().push(format!( + "`{}` reads its receiver from the first parameter of the function it is \ + written in, and it has none", + render_expr(&attribute.value) + )); + return; + }; let target_src = render_expr(&s.slice); - let replacement = - format!("super({cls}.__mro__[{cls}.__mro__.index({target_src}) - 1], {self_name})"); + let replacement = format!( + "{}({cls}.__mro__[{cls}.__mro__.index({target_src}) - 1], {self_name})", + self.builtin_super + ); self.edits.borrow_mut().push((s.range(), replacement)); return; } @@ -115,13 +205,15 @@ impl<'ast> Visitor<'ast> for &State<'_> { self.visit_arguments(&call.arguments); return; } - if let Expr::Name(n) = expr + // the keyword is the base of an attribute access, `super.x`, where the checker reads + // it as one. read anywhere else, `super` is the name it is bound to, as it is there + if let Expr::Attribute(attribute) = expr + && let Expr::Name(n) = attribute.value.as_ref() && n.id.as_str() == "super" && matches!(n.ctx, ExprContext::Load) { - self.edits - .borrow_mut() - .push((n.range(), "super()".to_owned())); + self.edits.borrow_mut().push((n.range(), self.bare())); + return; } walk_expr(self, expr); } @@ -261,6 +353,128 @@ mod tests { "}); } + /// the keyword is the builtin wherever the module binds the name to something else + #[test] + fn a_module_binding_super_still_calls_the_builtin() { + check( + indoc! {" + super = 5 + + class C(A, B): + def f(this): + super.x + super[B].y + "}, + indoc! {" + from builtins import super as super2 + super = 5 + + class C(A, B): + def f(this): + super2(__class__, this).x + super2(C.__mro__[C.__mro__.index(B) - 1], this).y + "}, + ); + } + + /// read where it is not an attribute's base, `super` is the name the checker reads it as: + /// the builtin class, or the parameter that binds the name + #[test] + fn super_read_as_a_value_is_the_name() { + check( + indoc! {" + class A(B): + def f(self, super: B): + return super.g(super) + def h(self): + return super + "}, + indoc! {" + from builtins import super as super2 + class A(B): + def f(self, super: B): + return super2(__class__, self).g(super) + def h(self): + return super + "}, + ); + } + + /// `super[T]` is the keyword only as an attribute's base. anywhere else it is the subscript + /// of the name, as the checker reads it + #[test] + fn a_subscript_of_super_that_is_no_attribute_base_is_the_name() { + unchanged(indoc! {" + super = {1: \"one\"} + + class B(A): + def f(self): + return super[1] + "}); + } + + /// a lambda passes its own first parameter as the receiver, as zero-argument `super()` + /// written in it would, not the method's + #[test] + fn a_lambda_passes_its_own_receiver() { + check( + indoc! {" + super = 5 + + class C(A, B): + def f(this): + return lambda other: super.x + super[B].y + "}, + indoc! {" + from builtins import super as super2 + super = 5 + + class C(A, B): + def f(this): + return lambda other: super2(__class__, other).x + super2(C.__mro__[C.__mro__.index(B) - 1], other).y + "}, + ); + } + + /// with no parameter there is no receiver to write, and an enclosing function's would be + /// another one than python reads + #[test] + fn a_targeted_super_with_no_receiver_is_refused() { + let error = transpile( + indoc! {" + class C(A, B): + def f(self): + return lambda: super[B].y + "}, + &Config::test_default(), + ) + .unwrap_err(); + assert!( + error.contains("reads its receiver from the first parameter"), + "got: {error}" + ); + } + + /// a `super()` object has no attributes of its own to write, so a store or a deletion + /// through the keyword is refused rather than written as one that raises + #[test] + fn an_attribute_of_super_written_or_deleted_is_refused() { + for statement in [ + "super.x = 1", + "super.x += 1", + "del super.x", + "super[B].x = 1", + "for super.x in []: pass", + ] { + let source = format!("class C(A, B):\n def f(self):\n {statement}\n"); + let error = transpile(&source, &Config::test_default()).unwrap_err(); + assert!( + error.contains("cannot be assigned or deleted"), + "{statement}: {error}" + ); + } + } + #[test] fn python_unchanged() { unchanged("class A:\n def f(self):\n super().x\n"); diff --git a/crates/by_transforms/src/transforms/symbolic_type_op.rs b/crates/by_transforms/src/transforms/symbolic_type_op.rs index 506c5d241c..a2425dc1c5 100644 --- a/crates/by_transforms/src/transforms/symbolic_type_op.rs +++ b/crates/by_transforms/src/transforms/symbolic_type_op.rs @@ -32,9 +32,7 @@ use ruff_text_size::{Ranged, TextRange}; use super::ast_driver::{AstPass, PassContext}; use super::literal_types::float_literal_spelling; use super::repeated_underscore::WrittenNames; -use super::type_expr_walker::{ - Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions, -}; +use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; use crate::PythonVersion; use crate::config::FloatLiteralLowering; use crate::type_info::TypeInfo; @@ -115,6 +113,12 @@ struct FoldCollector<'a> { } impl TypeExprVisitor for FoldCollector<'_> { + /// a type one lowering writes whole writes the folds found inside it too: `(str, *(int, + /// str) * n)`, `((int, str) * 2) -> str`, `{"a": 1 + 1}` + fn descends_into_wholly_lowered(&self) -> bool { + true + } + fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { // which nodes are symbolic operations to fold: // - binary ops other than `|` (union) and `&` (intersection), which @@ -158,19 +162,6 @@ impl TypeExprVisitor for FoldCollector<'_> { // type parameter's bound — python cannot express the dependency on `T`, and // the bound's member type is the guarantee every specialization satisfies Expr::Attribute(_) => self.types.is_attribute_type(expr), - // the shared walker stops at a tuple type, which `annotation` lowers as a - // whole, carrying the folds found inside it: `(str, *(int, str) * n)` - Expr::Tuple(tuple) if !tuple.is_anon_named_tuple => { - for element in &tuple.elts { - let element = match element { - Expr::Starred(starred) => starred.value.as_ref(), - Expr::Named(named) => named.value.as_ref(), - element => element, - }; - walk_one_type_expr(element, self); - } - return Recurse::Stop; - } _ => false, }; if !foldable { @@ -737,6 +728,38 @@ mod tests { ); } + /// a callable arrow is lowered whole, by one wide edit, and the fold inside it is written + /// there + #[test] + fn a_fold_inside_a_callable_arrow() { + check( + "x: ((int, str) * 2) -> 1 + 1\n", + indoc! {" + from typing import Callable, Literal + x: Callable[[tuple[int, str, int, str]], Literal[2]] + "}, + ); + } + + /// the types a synthesized class declares are written in its body, which no edit of the + /// annotation reaches, so a fold among them is written there by the class + #[test] + fn a_fold_inside_a_synthesized_class() { + for (source, field) in [ + ("x: (n: int) -> 1 + 1\n", "-> \"Literal[2]\""), + ("x: protocol(n: 1 + 1)\n", " n: \"Literal[2]\"\n"), + ("x: protocol(def m(self) -> 1 + 1)\n", "-> \"Literal[2]\""), + ("x: {\"n\": 1 + 1}\n", " n: \"Literal[2]\"\n"), + ("x: (n: 1 + 1, m: int)\n", " n: Literal[2]\n"), + ] { + let out = transpile(source, &Config::test_default()).unwrap(); + assert!( + out.contains(field) && !out.contains("1 + 1"), + "{source}got:\n{out}" + ); + } + } + #[test] fn typeof_operand_nested_in_subscript() { // the fold consumes the `typeof` operand even inside a subscript slice, diff --git a/crates/by_transforms/src/transforms/synthesized_types.rs b/crates/by_transforms/src/transforms/synthesized_types.rs new file mode 100644 index 0000000000..5d0018876a --- /dev/null +++ b/crates/by_transforms/src/transforms/synthesized_types.rs @@ -0,0 +1,467 @@ +//! Names the class each type expression that lowers to one stands for — a typed dict +//! literal, an anonymous named tuple type, an inline protocol — in one walk over the module. +//! +//! The three nest in one another every way round, `(p: {"a": int})`, `{"a": (p: int)}`, +//! `protocol(x: (p: int))`, and the class an outer one lowers to declares the classes of the +//! ones inside it as the types of its fields. So the walk names a type only once it has named +//! every type inside it, prints each class body through one type-expression lowerer that writes +//! every name given so far in place of its type, and declares the classes in the order they were +//! named — each after every class its body reads, which a `NamedTuple`, whose field annotations +//! are evaluated as the class is made, depends on. +//! +//! The passes that lower each kind replace a type with the name given here, and every other +//! lowering that prints a type expression whole is handed the names as substitutions +//! ([`PassContext::type_substitutions`](super::ast_driver::PassContext::type_substitutions)), +//! so a synthesized type inside a callable arrow, a tuple type or an `__init__` annotation is +//! written as its class wherever it stands. + +use std::collections::{BTreeSet, HashMap}; + +use indexmap::IndexMap; +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange}; +use ty_python_semantic::AnonNamedTupleId; + +use super::anon_named_tuple; +use super::ast_driver::Lowering; +use super::callable::CallableSyntax; +use super::generics::PolyfilledTypeParams; +use super::protocol_type; +use super::quotes::WrittenQuotes; +use super::repeated_underscore::WrittenNames; +use super::type_expr_walker::{ + Recurse, TypeExprVisitor, TypePos, walk_one_type_expr, walk_type_positions_skipping, +}; +use super::typed_dict_literal; +use crate::config::Config; +use crate::type_info::TypeInfo; + +/// the lowerings a pass writes itself when it replaces a type with a class whose body the namer +/// printed: every lowering the shared type-expression lowerer writes, a `typeof` among them, and +/// each synthesized type nested inside, which is written as its class +pub(crate) const CLASS_BODY: &[Lowering] = &[ + Lowering::Callable, + Lowering::OptionalType, + Lowering::LiteralType, + Lowering::JustFloat, + Lowering::DynamicKeyword, + Lowering::FloatConst, + Lowering::SymbolicTypeOp, + Lowering::LiteralString, + Lowering::Typeof, + Lowering::AnonNamedTuple, + Lowering::ProtocolType, + Lowering::TypedDictLiteral, +]; + +/// which kind of class a synthesized type lowers to +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SynthesizedKind { + TypedDict, + NamedTuple, + Protocol, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum ClassShape { + TypedDict(typed_dict_literal::Shape), + NamedTuple(anon_named_tuple::Shape), + Protocol(protocol_type::Shape), +} + +/// Every synthesized type of a module, the class each names, and what declaring the classes takes. +#[derive(Default)] +pub(crate) struct SynthesizedTypes { + /// each synthesized type's source range, its kind and the class it names, in naming order + nodes: Vec<(TextRange, SynthesizedKind, String)>, + /// every anonymous named tuple class, by shape. the named tuple lowering reuses a class + /// for a value form, or a plain tuple it constructs, of the same shape + named_tuples: IndexMap, + /// the class of each anonymous named tuple the checker knows, by its identity there, which + /// a plain tuple literal the checker reads as one is constructed as + named_tuple_ids: Vec<(AnonNamedTupleId, String)>, + /// the class declarations, each after every class it reads + pub(crate) declarations: String, + /// the import lines the declarations read + pub(crate) imports: Vec, + /// the runtime helpers they call + pub(crate) helpers: Vec, + /// why a type could not be declared + pub(crate) errors: Vec, +} + +impl SynthesizedTypes { + /// name every synthesized type in `stmts`, where `folds` are the operations `symbolic_type_op` + /// folded, written in a class body as the type they fold to, and `claimed` their ranges + #[expect(clippy::too_many_arguments)] + pub(crate) fn name_all( + source: &str, + written: WrittenNames, + quotes: &WrittenQuotes, + types: &dyn TypeInfo, + config: &Config, + stmts: &[Stmt], + folds: &[(TextRange, String)], + claimed: &[TextRange], + ) -> Self { + let mut namer = Namer::new(source, written, quotes, types, claimed, config); + for (range, rendered) in folds { + namer.callable.add_substitution(*range, rendered.clone()); + } + let polyfilled = PolyfilledTypeParams::decide(source, written, stmts, config); + let mut scopes = TypeParameterScopes { + polyfilled: &polyfilled, + scopes: &mut namer.type_parameter_scopes, + }; + for stmt in stmts { + scopes.visit_stmt(stmt); + } + walk_type_positions_skipping(stmts, Some(types), claimed, &mut namer); + // `protocol(...)` and `(name: T)` are no python wherever they appear, so the ones in a + // value position, an alias `P = (name: str)` say, are named too + let mut values = ValuePositions { namer: &mut namer }; + for stmt in stmts { + values.visit_stmt(stmt); + } + namer.finish() + } + + /// the source range of each synthesized type of `kind`, with the class it names + pub(crate) fn of_kind(&self, kind: SynthesizedKind) -> Vec<(TextRange, String)> { + self.nodes + .iter() + .filter(|(_, node_kind, _)| *node_kind == kind) + .map(|(range, _, name)| (*range, name.clone())) + .collect() + } + + /// the source range of every synthesized type, with the class it names + pub(crate) fn substitutions(&self) -> impl Iterator + '_ { + self.nodes + .iter() + .map(|(range, _, name)| (*range, name.clone())) + } + + /// the class the synthesized type of `kind` written at exactly `range` names + pub(crate) fn class_at(&self, range: TextRange, kind: SynthesizedKind) -> Option<&str> { + self.nodes + .iter() + .find(|(candidate, node_kind, _)| *candidate == range && *node_kind == kind) + .map(|(_, _, name)| name.as_str()) + } + + /// every anonymous named tuple class declared here, by shape + pub(crate) fn named_tuples(&self) -> &IndexMap { + &self.named_tuples + } + + /// the class declared here for each anonymous named tuple the checker identifies + /// ([`TypeInfo::anon_named_tuple_id`]) + pub(crate) fn named_tuple_identities( + &self, + ) -> impl Iterator + '_ { + self.named_tuple_ids.iter().cloned() + } + + /// whether the module has no synthesized type at all + pub(crate) fn is_empty(&self) -> bool { + self.nodes.is_empty() && self.errors.is_empty() + } +} + +/// The walk in progress: what the kinds of synthesized type print their class bodies with, and +/// what has been named so far. +pub(crate) struct Namer<'src> { + source: &'src str, + types: &'src dyn TypeInfo, + pub(crate) written: WrittenNames<'src>, + /// where a class body's types are written as strings + pub(crate) quotes: &'src WrittenQuotes, + /// the one lowerer every class body is printed through, holding a substitution for each type + /// named so far and for each fold + pub(crate) callable: CallableSyntax<'src>, + /// the names each type parameter of a generic is read by from module scope, where a class + /// lands, keyed by the range of the statement declaring the generic: the polyfill's `_T`, or + /// `A.__type_params__[0]` for a list kept native + type_parameter_scopes: Vec<(TextRange, HashMap)>, + classes: IndexMap, + nodes: Vec<(TextRange, SynthesizedKind, String)>, + named_tuple_ids: Vec<(AnonNamedTupleId, String)>, + pub(crate) errors: Vec, + /// the `(module, name)` of each name a class body reads that the shared lowerer does not + /// import itself: the `Literal` of a typed dict field's literal type, an `Intersection` + pub(crate) imports: BTreeSet<(&'static str, &'static str)>, + /// set when a class body spells a nested optional `T??` with the runtime `Optional` + pub(crate) needs_optional_runtime: bool, +} + +impl<'src> Namer<'src> { + fn new( + source: &'src str, + written: WrittenNames<'src>, + quotes: &'src WrittenQuotes, + types: &'src dyn TypeInfo, + claimed: &'src [TextRange], + config: &Config, + ) -> Self { + Self { + source, + types, + written, + quotes, + callable: CallableSyntax::new(source, written, quotes, config) + .with_types(types) + .with_claimed_ranges(claimed), + type_parameter_scopes: Vec::new(), + classes: IndexMap::new(), + nodes: Vec::new(), + named_tuple_ids: Vec::new(), + errors: Vec::new(), + imports: BTreeSet::new(), + needs_optional_runtime: false, + } + } + + /// what the checker knows of the module + pub(crate) fn types(&self) -> &'src dyn TypeInfo { + self.types + } + + pub(crate) fn src(&self, range: TextRange) -> &'src str { + &self.source[usize::from(range.start())..usize::from(range.end())] + } + + /// what the type written at exactly `range` is printed as, when it is already lowered: a + /// class named before it, or a fold + pub(crate) fn substitution_for(&self, range: TextRange) -> Option { + self.callable.substitution_for(range).map(str::to_owned) + } + + /// `expr` printed as python, every type inside it named so far written as its class + pub(crate) fn lower(&mut self, expr: &Expr) -> String { + self.callable.lower_type_expr(expr) + } + + /// `rendered`, the text of a type written at `range`, reading each type parameter of the + /// generics around it by the name it has at module scope, where the class is declared + pub(crate) fn read_type_parameters(&self, range: TextRange, rendered: &str) -> String { + let mut renames: HashMap<&str, &str> = HashMap::new(); + for (scope, frame) in &self.type_parameter_scopes { + if scope.contains_range(range) { + renames.extend(frame.iter().map(|(k, v)| (k.as_str(), v.as_str()))); + } + } + if renames.is_empty() { + return rendered.to_owned(); + } + super::source_util::rename_identifiers(rendered, &renames) + } + + fn is_named(&self, range: TextRange) -> bool { + self.nodes.iter().any(|(candidate, ..)| *candidate == range) + } + + /// name the type at `range` as the class declaring `shape`, or as `known`, a class named + /// already for a type the checker reads as the same one + fn name( + &mut self, + range: TextRange, + kind: SynthesizedKind, + shape: ClassShape, + known: Option, + ) -> String { + let name = if let Some(known) = known { + known + } else if let Some(existing) = self.classes.get(&shape) { + existing.clone() + } else { + let name = match &shape { + ClassShape::TypedDict(shape) => shape.class_name(self.written), + ClassShape::NamedTuple(shape) => shape.class_name(self.written), + ClassShape::Protocol(shape) => shape.class_name(self.written), + }; + self.classes.insert(shape, name.clone()); + name + }; + // an enclosing type's class body names this one wherever it sits in it — inside a + // subscript, a union, a callable arrow — not only when it is a whole field type + self.callable.add_substitution(range, name.clone()); + self.nodes.push((range, kind, name.clone())); + name + } + + fn finish(mut self) -> SynthesizedTypes { + let written = self.written; + let mut declarations = self.callable.class_defs().to_owned(); + let mut kinds = BTreeSet::new(); + let mut named_tuples = IndexMap::new(); + for (shape, name) in &self.classes { + let declaration = match shape { + ClassShape::TypedDict(shape) => { + kinds.insert(("typing_extensions", "TypedDict")); + shape.class_def( + name, + &written.imported("typing_extensions", "TypedDict"), + self.quotes, + ) + } + ClassShape::NamedTuple(shape) => { + kinds.insert(("typing", "NamedTuple")); + named_tuples.insert(shape.clone(), name.clone()); + shape.class_def(name, &written.imported("typing", "NamedTuple")) + } + ClassShape::Protocol(shape) => { + kinds.insert(("typing", "Protocol")); + shape.class_def(name, &written.imported("typing", "Protocol")) + } + }; + declarations.push_str(&declaration); + declarations.push('\n'); + } + let (mut imports, mut helpers) = self.callable.take_requirements(); + imports.extend( + kinds + .into_iter() + .chain(self.imports.iter().copied()) + .map(|(module, name)| written.import_from(module, &[name])), + ); + if self.needs_optional_runtime { + helpers.push(crate::runtime::OPTIONAL); + } + SynthesizedTypes { + nodes: self.nodes, + named_tuples, + named_tuple_ids: self.named_tuple_ids, + declarations, + imports, + helpers, + errors: self.errors, + } + } +} + +impl TypeExprVisitor for Namer<'_> { + fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { + // a type named already, reached a second time from a value position, keeps its name + if self.is_named(expr.range()) { + return Recurse::Stop; + } + Recurse::Descend + } + + /// every type inside a synthesized one has to be named before it, so the walk goes on into + /// them, where the passes that print a type whole stop + fn descends_into_wholly_lowered(&self) -> bool { + true + } + + fn leave(&mut self, expr: &Expr) { + if self.is_named(expr.range()) { + return; + } + match expr { + Expr::Dict(dict) => { + if let Some(shape) = typed_dict_literal::shape(self, dict) { + self.name( + dict.range(), + SynthesizedKind::TypedDict, + ClassShape::TypedDict(shape), + None, + ); + } + } + Expr::Tuple(tuple) if tuple.is_anon_named_tuple => { + match anon_named_tuple::type_shape(self, tuple) { + Ok(Some(shape)) => { + // the checker reads two spellings of one type, `(a: List[int])` and + // `(a: list[int])`, as one class, which both are written as + let id = self.types.anon_named_tuple_id(tuple); + let known = id.and_then(|id| { + self.named_tuple_ids + .iter() + .find(|(candidate, _)| *candidate == id) + .map(|(_, name)| name.clone()) + }); + let is_known = known.is_some(); + let name = self.name( + tuple.range(), + SynthesizedKind::NamedTuple, + ClassShape::NamedTuple(shape), + known, + ); + if let Some(id) = id + && !is_known + { + self.named_tuple_ids.push((id, name)); + } + } + Ok(None) => {} + Err(message) => self.errors.push(message), + } + } + Expr::ProtocolType(protocol) => { + let shape = protocol_type::shape(self, protocol); + self.name( + protocol.range(), + SynthesizedKind::Protocol, + ClassShape::Protocol(shape), + None, + ); + } + _ => {} + } + } +} + +/// Finds the inline protocols and anonymous named tuple types standing where a value is written, +/// each named as the type it is. +struct ValuePositions<'a, 'src> { + namer: &'a mut Namer<'src>, +} + +impl<'ast> Visitor<'ast> for ValuePositions<'_, '_> { + fn visit_expr(&mut self, expr: &'ast Expr) { + let is_type = match expr { + Expr::ProtocolType(_) => true, + Expr::Tuple(tuple) => tuple.is_anon_named_tuple, + _ => false, + }; + if is_type { + walk_one_type_expr(expr, self.namer); + return; + } + walk_expr(self, expr); + } +} + +/// Collects the names each generic's type parameters are read by from module scope, keyed by the +/// range of the statement declaring the generic. +struct TypeParameterScopes<'a> { + polyfilled: &'a PolyfilledTypeParams, + scopes: &'a mut Vec<(TextRange, HashMap)>, +} + +impl TypeParameterScopes<'_> { + fn record(&mut self, range: TextRange, type_params: Option<&ruff_python_ast::TypeParams>) { + if let Some(frame) = type_params.and_then(|tp| self.polyfilled.hoisted_renames(tp)) + && !frame.is_empty() + { + self.scopes.push((range, frame)); + } + } +} + +impl<'ast> Visitor<'ast> for TypeParameterScopes<'_> { + fn visit_stmt(&mut self, stmt: &'ast Stmt) { + match stmt { + Stmt::ClassDef(class) => self.record(class.range(), class.type_params.as_deref()), + Stmt::FunctionDef(function) => { + self.record(function.range(), function.type_params.as_deref()); + } + Stmt::TypeAlias(alias) => self.record(alias.range(), alias.type_params.as_deref()), + _ => {} + } + walk_stmt(self, stmt); + } +} diff --git a/crates/by_transforms/src/transforms/template_expression.rs b/crates/by_transforms/src/transforms/template_expression.rs index 0bcb378d72..afdd240d1d 100644 --- a/crates/by_transforms/src/transforms/template_expression.rs +++ b/crates/by_transforms/src/transforms/template_expression.rs @@ -3,16 +3,18 @@ //! python 3.14 gives every replacement field of a `t"..."` to the program as a //! `string.templatelib.Interpolation`, whose `expression` attribute is the *source text* //! of the field: everything between the `{` and the `=`, conversion, format spec or `}` -//! that ends it, whitespace and grouping parentheses included. +//! that ends the expression, its grouping parentheses kept and its comments taken out. //! //! ```python //! a = 3 //! t"{ (a) !r}".interpolations[0].expression # ' (a) ' //! ``` //! -//! python 3.14.0 to 3.14.7 dropped the whitespace before that end (python/cpython#154719). -//! the lowering writes the text the way later releases report it, and [`_by_template`] -//! drops the trailing whitespace again when it runs on one of those releases +//! that is the rule from 3.14.8 (python/cpython#154719). 3.14.0 to 3.14.7 also drop the +//! whitespace at the end of it, and report `' (a)'` there. a lowered field is given the +//! newer text, and [`_by_field_text`] trims it at runtime on those releases, so the field +//! reads the same as the ones python built beside it. the polyfill a string tag lowers to +//! below 3.14 runs on interpreters with no rule of their own, and takes the newer one //! //! that text is what the attribute is for — a template library names a cache entry by it, //! shows it in an error, or renders a `=`-style debug view from it. python reads it off @@ -37,6 +39,7 @@ //! gives each field the author's text as it does //! //! [`_by_template`]: crate::runtime +//! [`_by_field_text`]: crate::runtime use ruff_python_ast::comparable::ComparableExpr; use ruff_python_ast::visitor::source_order::{self, SourceOrderVisitor}; @@ -46,7 +49,8 @@ use ruff_python_ast::{ InterpolatedElement, ModModule, Stmt, }; use ruff_python_ast::{ExprContext, name::Name}; -use ruff_text_size::{Ranged, TextRange, TextSize}; +use ruff_python_trivia::CommentRanges; +use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use super::ast_driver::Fragment; use super::debug_field::rewrites_expression; @@ -158,26 +162,52 @@ impl Wrap { /// the source text python hands a reader as a field's `expression` /// /// python takes it from just past the `{` to the `=`, conversion, format spec or `}` that -/// ends the field, keeping whatever grouping parentheses and whitespace stand around the -/// expression. the expression node's own range holds neither, so both ends are read off -/// the field: the start from its brace, and the end by stepping over the parentheses and -/// whitespace between the expression's last token and the end of the field — nothing -/// else can stand there +/// ends the expression, keeping whatever grouping parentheses the expression was written +/// inside and the whitespace after them, and leaving out every comment in it — though not +/// the line break that ends one. the expression node's own range holds neither the +/// parentheses nor the whitespace, so the end is found by stepping over what can stand +/// between the expression's last token and that delimiter: closing parentheses, +/// whitespace, line continuations and comments /// -/// this is the text python reports from 3.14.8 on; [`_by_template`](crate::runtime) drops -/// the trailing whitespace on the earlier 3.14 releases, which did -pub(crate) fn author_text<'src>(source: &'src str, field: &InterpolatedElement) -> &'src str { - let start = usize::from(field.range().start()) + '{'.len_utf8(); - let expression_end = usize::from(field.expression.range().end()); - let Some(tail) = source.get(expression_end..usize::from(field.range().end())) else { - return ""; - }; - let closing = tail - .find(|ch: char| ch != ')' && !ch.is_whitespace()) - .unwrap_or(tail.len()); - source - .get(start..expression_end + closing) - .unwrap_or_default() +/// this is the text 3.14.8 and later report. what 3.14.0 to 3.14.7 report is this with its +/// trailing whitespace dropped, which `_by_field_text` does at runtime +pub(crate) fn field_text( + source: &str, + comments: &CommentRanges, + field: &InterpolatedElement, +) -> String { + let start = field.range().start() + '{'.text_len(); + let end = expression_end(source, comments, field); + let mut text = String::new(); + let mut from = start; + for comment in comments.comments_in_range(TextRange::new(start, end)) { + text.push_str(&source[TextRange::new(from, comment.start())]); + from = comment.end(); + } + text.push_str(&source[TextRange::new(from, end)]); + text +} + +/// where python ends a field's expression: at the first character after its last token that +/// is not a closing parenthesis, whitespace, a line continuation or part of a comment +fn expression_end(source: &str, comments: &CommentRanges, field: &InterpolatedElement) -> TextSize { + let field_end = field.range().end(); + let mut at = field.expression.range().end(); + while at < field_end { + if let Some(comment) = comments + .comments_in_range(TextRange::new(at, field_end)) + .first() + .filter(|comment| comment.start() == at) + { + at = comment.end(); + continue; + } + match source[usize::from(at)..].chars().next() { + Some(ch) if ch == ')' || ch == '\\' || ch.is_whitespace() => at += ch.text_len(), + _ => break, + } + } + at } /// a t-string of the source, the interpolations it hands a reader in the order it hands @@ -318,6 +348,7 @@ pub(crate) fn replaced_in_tree( /// field needs rebuilding pub(crate) fn claim( source: &str, + comments: &CommentRanges, suite: &[Stmt], edited: &[TextRange], replaced: &[TextRange], @@ -332,7 +363,7 @@ pub(crate) fn claim( || edited.iter().any(|edit| { rewrites_expression(*edit, field.expression.range(), field.range()) }); - lowered.then(|| author_text(source, field).to_owned()) + lowered.then(|| field_text(source, comments, field)) }) .collect(); if tagged || texts.iter().any(Option::is_some) { @@ -507,8 +538,9 @@ mod tests { ); } - /// python reads the text from just past the `{` to the end of the field, keeping the - /// grouping parentheses and the whitespace around them. `tstring_expression_runtime` + /// python reads the text from just past the `{` to the delimiter that ends the + /// expression, keeping the grouping parentheses and the whitespace after them, in the + /// form 3.14.8 reports. `a_field_reports_the_source_the_author_wrote` in the runtime test /// holds these to a real 3.14 #[test] fn the_author_text_is_the_span_python_would_have_reported() { @@ -523,6 +555,19 @@ mod tests { ); } + /// a comment is left out of the text, and the line break that ends it is not, as python + /// does — including one written after the expression, before the delimiter + #[test] + fn a_comment_is_left_out_of_the_author_text() { + assert_eq!( + lowered( + "def go(a: int?, b: int):\n print(t\"\"\"{(a! # why\n ?? b) # and\n}\"\"\")\n" + ), + // the last line of the rebuild, which closes the literal and gives the text + "}\"\"\", (\"(a! \\n ?? b) \\n\",)))" + ); + } + /// a t-string nothing lowered is left exactly as written, in a `.by` file as in a /// `.py` one — there is nothing to correct, and python's own text is already right #[test] diff --git a/crates/by_transforms/src/transforms/type_expr_walker.rs b/crates/by_transforms/src/transforms/type_expr_walker.rs index 9cded327ba..3f06078011 100644 --- a/crates/by_transforms/src/transforms/type_expr_walker.rs +++ b/crates/by_transforms/src/transforms/type_expr_walker.rs @@ -107,6 +107,16 @@ pub(crate) trait TypeExprVisitor { /// told ahead of each root the walk visits what kind of root it is, which holds /// for everything nested inside it fn enter_root(&mut self, _kind: RootKind) {} + + /// whether the walk descends into a type one lowering writes whole, rather than stopping + /// at it — see [`wholly_lowered_interior`] + fn descends_into_wholly_lowered(&self) -> bool { + false + } + + /// told of each type expression once everything the walk descended into below it has + /// been visited, for a visitor that needs the inside of a type before the type itself + fn leave(&mut self, _expr: &Expr) {} } /// walk every type-position expression in `stmts`. `types` enables detection @@ -162,6 +172,53 @@ pub(crate) fn walk_one_type_expr_with( walker.visit_type_expr(expr, TypePos::Root); } +/// the type positions directly inside `expr` when it is a type expression one lowering writes +/// whole — a tuple type or an anonymous named tuple, a callable arrow, an inline protocol, a dict +/// literal type — and `None` for any other expression +/// +/// the walk stops at each of these, so that no narrow edit lands inside a span one wide edit +/// replaces. a visitor whose findings that lowering writes itself, wherever they are, reads the +/// positions inside one instead: a symbolic fold, which every such lowering writes in place of +/// the operation +fn wholly_lowered_interior(expr: &Expr) -> Option> { + // a field, member or parameter label is a name, not a type + fn labelled(expr: &Expr) -> &Expr { + match expr { + Expr::Named(named) => named.value.as_ref(), + expr => expr, + } + } + Some(match expr { + Expr::Tuple(tuple) => tuple + .elts + .iter() + .map(|element| match element { + Expr::Starred(starred) => starred.value.as_ref(), + element => labelled(element), + }) + .collect(), + Expr::CallableType(callable) => callable + .receiver + .as_deref() + .into_iter() + .chain(callable.args.iter().map(labelled)) + .chain(std::iter::once(callable.returns.as_ref())) + .collect(), + Expr::ProtocolType(protocol) => protocol + .members + .iter() + .filter_map(|member| match member { + Expr::Named(named) => Some(named.value.as_ref()), + Expr::ProtocolMethod(method) => Some(method.signature.as_ref()), + // a `**Kwargs` pack splices fields in, and is no type of its own + _ => None, + }) + .collect(), + Expr::Dict(dict) => dict.items.iter().map(|item| &item.value).collect(), + _ => return None, + }) +} + struct TypePosWalker<'a> { types: Option<&'a dyn TypeInfo>, claimed: &'a [TextRange], @@ -189,12 +246,21 @@ impl TypePosWalker<'_> { return; } self.descend_into_type_expr(expr); + self.visitor.leave(expr); } /// recurse into sub-positions of a type expression. the walker knows the /// shape of `BinOp` (`|` / `&`), `UnaryOp` (`not`), and `Subscript` /// (generic + special forms `Annotated` / `Callable` / `Literal`) fn descend_into_type_expr(&mut self, expr: &Expr) { + if self.visitor.descends_into_wholly_lowered() + && let Some(interior) = wholly_lowered_interior(expr) + { + for position in interior { + self.visit_type_expr(position, TypePos::Nested); + } + return; + } match expr { Expr::BinOp(b) if matches!(b.op, Operator::BitOr | Operator::BitAnd) => { self.visit_type_expr(&b.left, TypePos::Nested); diff --git a/crates/by_transforms/src/transforms/typed_dict_literal.rs b/crates/by_transforms/src/transforms/typed_dict_literal.rs index 6a009bad82..9f7eaab5ce 100644 --- a/crates/by_transforms/src/transforms/typed_dict_literal.rs +++ b/crates/by_transforms/src/transforms/typed_dict_literal.rs @@ -29,24 +29,17 @@ //! //! field types are emitted as forward references — see `Shape::class_def` -use std::collections::BTreeSet; use std::fmt::Write as _; use std::hash::Hash; -use indexmap::IndexMap; -use ruff_diagnostics::{Edit, Fix}; -use ruff_python_ast::{Expr, ModModule, Operator, Stmt}; +use ruff_python_ast::{Expr, ModModule, Operator}; use ruff_python_stdlib::identifiers::is_identifier; use ruff_text_size::Ranged; -use super::type_expr_walker::{Recurse, TypeExprVisitor, TypePos, walk_type_positions}; - use super::ast_driver::{AstPass, PassContext}; -use super::callable::CallableSyntax; use super::quotes::WrittenQuotes; use super::repeated_underscore::WrittenNames; -use crate::config::Config; -use crate::type_info::TypeInfo; +use super::synthesized_types::{Namer, SynthesizedKind}; /// Sorted list of `(field_name, type_source)` pairs identifying a unique /// shape. Sorted so that `{"a": int, "b": str}` and `{"b": str, "a": int}` @@ -54,14 +47,14 @@ use crate::type_info::TypeInfo; /// basedpython `**: T` marker; when `None` the dict is closed (no extra keys /// allowed). #[derive(Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -struct Shape { +pub(crate) struct Shape { fields: Vec<(String, String)>, extra_items: Option, } impl Shape { /// the name of the class declaring this shape, one the module has no other use of - fn class_name(&self, written: WrittenNames) -> String { + pub(crate) fn class_name(&self, written: WrittenNames) -> String { written.synthesized_class("_TypedDict_", self) } @@ -70,10 +63,9 @@ impl Shape { /// /// every field type is written as a string. the hoisted classes are emitted in the /// preamble, ahead of everything the module defines, and a shape can name a class declared - /// later or a type parameter that only exists inside the generic class the literal was - /// written in. neither resolves where the class lands, and `TypedDict` evaluates its + /// later. that does not resolve where the class lands, and `TypedDict` evaluates its /// class-body annotations eagerly, so every field is a forward reference - fn class_def(&self, name: &str, typed_dict: &str, quotes: &WrittenQuotes) -> String { + pub(crate) fn class_def(&self, name: &str, typed_dict: &str, quotes: &WrittenQuotes) -> String { let bases = match &self.extra_items { Some(ty) => format!("{typed_dict}, extra_items={}", quotes.quote(ty)), None => format!("{typed_dict}, closed=True"), @@ -95,322 +87,160 @@ impl Shape { } } -pub(crate) struct TypedDictLiteral<'src> { - source: &'src str, - written: WrittenNames<'src>, - /// where the field types are written as strings - quotes: &'src WrittenQuotes, - edits: Vec, - errors: Vec, - /// Insertion-ordered map of shape → synthesized class name so emitted - /// classes follow dependency order (inner-before-outer) - shapes: IndexMap, - /// Source range of every typed-dict literal we've assigned a class to. - /// Used when rendering an outer dict's body to substitute nested dict - /// spans with their class names. - range_to_class: Vec<(ruff_text_size::TextRange, String)>, - needs_import: bool, - /// the `(module, name)` of each name a field type this pass writes itself reads: the - /// `Literal` of a promoted literal type, the `Intersection` of an intersection - field_imports: BTreeSet<(&'static str, &'static str)>, - /// set when a nested `T??` field needs the runtime `Optional[...]` wrapper - needs_optional_runtime: bool, - /// the shared type-expression lowerer, which prints every field type this pass does - /// not take apart itself. the class is hoisted out of the statement the field type - /// was written in, so no other pass's edit reaches the field - callable: CallableSyntax<'src>, -} - -impl<'src> TypedDictLiteral<'src> { - fn new( - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - config: &Config, - types: &'src dyn TypeInfo, - ) -> Self { - Self { - source, - written, - quotes, - callable: CallableSyntax::new(source, written, quotes, config).with_types(types), - edits: Vec::new(), - errors: Vec::new(), - shapes: IndexMap::new(), - range_to_class: Vec::new(), - needs_import: false, - field_imports: BTreeSet::new(), - needs_optional_runtime: false, - } - } - - fn class_defs(&self) -> String { - let mut out = String::new(); - let typed_dict = self.written.imported("typing_extensions", "TypedDict"); - for (shape, name) in &self.shapes { - out.push_str(&shape.class_def(name, &typed_dict, self.quotes)); - out.push('\n'); - } - out - } - - /// Render `range`'s source, substituting any nested typed-dict literal - /// span with its synthesized class name. - fn render_subbed(&self, range: ruff_text_size::TextRange) -> String { - use ruff_text_size::TextSize; - let mut subs: Vec<(ruff_text_size::TextRange, &str)> = self - .range_to_class - .iter() - .filter(|(r, _)| r.start() >= range.start() && r.end() <= range.end()) - .map(|(r, n)| (*r, n.as_str())) - .collect(); - subs.sort_by_key(|(r, _)| r.start()); - - let mut out = String::new(); - let mut cursor: TextSize = range.start(); - for (sub_range, replacement) in subs { - if sub_range.start() < cursor { - continue; - } - out.push_str(&self.source[usize::from(cursor)..usize::from(sub_range.start())]); - out.push_str(replacement); - cursor = sub_range.end(); - } - out.push_str(&self.source[usize::from(cursor)..usize::from(range.end())]); - out - } - - /// Try to extract a shape from a dict literal. Returns `None` when the - /// dict has any non-string-literal key, any unpacked item that is neither - /// the basedpython `**: T` extra-items marker nor a `**Kwargs` pack - /// splice, or any field name that isn't a valid Python identifier — those - /// cases are left for the user to spell with the functional `TypedDict()` - /// form instead. - fn extract_shape(&mut self, dict: &ruff_python_ast::ExprDict) -> Option { - // an empty dict literal `{}` is a valid annotation: a TypedDict with no - // fields. it still needs to be hoisted so the surface annotation lands - // on a real class name instead of leaving `{}` raw in python output - let mut fields = Vec::with_capacity(dict.items.len()); - let mut extra_items: Option = None; - for item in &dict.items { - let Some(key_expr) = item.key.as_ref() else { - // `**: T` is encoded as `key = None, value = Starred(Starred(T))` - let Expr::Starred(outer) = &item.value else { - // `{**Kwargs}` splices a keyword-variadic pack. its fields are - // only known once the enclosing class is specialized, and python - // erases type arguments anyway, so the shape lowers to a - // `TypedDict` that admits the pack's keys as extra items - if matches!(&item.value, Expr::Name(_)) && extra_items.is_none() { - extra_items = Some(self.written.builtin("object")); - continue; - } - return None; - }; - let Expr::Starred(inner) = outer.value.as_ref() else { - return None; - }; - if extra_items.is_some() { - return None; +/// The shape of a dict literal type, or `None` when the dict has any +/// non-string-literal key, any unpacked item that is neither the basedpython +/// `**: T` extra-items marker nor a `**Kwargs` pack splice, or any field name +/// that isn't a valid Python identifier — those cases are left for the user to +/// spell with the functional `TypedDict()` form instead. +pub(crate) fn shape(namer: &mut Namer, dict: &ruff_python_ast::ExprDict) -> Option { + // an empty dict literal `{}` is a valid annotation: a TypedDict with no + // fields. it still needs to be hoisted so the surface annotation lands + // on a real class name instead of leaving `{}` raw in python output + let mut fields = Vec::with_capacity(dict.items.len()); + let mut extra_items: Option = None; + for item in &dict.items { + let Some(key_expr) = item.key.as_ref() else { + // `**: T` is encoded as `key = None, value = Starred(Starred(T))` + let Expr::Starred(outer) = &item.value else { + // `{**Kwargs}` splices a keyword-variadic pack. its fields are + // only known once the enclosing class is specialized, and python + // erases type arguments anyway, so the shape lowers to a + // `TypedDict` that admits the pack's keys as extra items + if matches!(&item.value, Expr::Name(_)) && extra_items.is_none() { + extra_items = Some(namer.written.builtin("object")); + continue; } - extra_items = Some(self.render_field_type(&inner.value)); - continue; - }; - let Expr::StringLiteral(s) = key_expr else { - self.errors.push(format!( - "typed-dict literal field key must be a string literal, got `{}`", - &self.source[usize::from(key_expr.range().start()) - ..usize::from(key_expr.range().end())] - )); return None; }; - let name = s.value.to_str().to_owned(); - if !is_identifier(&name) { - self.errors.push(format!( - "typed-dict literal field name `{name}` is not a valid Python identifier" - )); + let Expr::Starred(inner) = outer.value.as_ref() else { return None; - } - if fields.iter().any(|(n, _): &(String, String)| n == &name) { - self.errors - .push(format!("duplicate typed-dict literal field `{name}`")); + }; + if extra_items.is_some() { return None; } - let type_source = self.render_field_type(&item.value); - fields.push((name, type_source)); - } - fields.sort_by(|a, b| a.0.cmp(&b.0)); - Some(Shape { - fields, - extra_items, - }) - } - - /// Render a field's type expression as Python source, applying - /// basedpython type-form lowerings (literal-type promotion, parenthesized - /// tuple types, intersection) recursively. Falls back to source text for - /// untouched expressions. - fn render_field_type(&mut self, expr: &Expr) -> String { - // nested typed-dict literals get substituted with their synthesized - // class names via `range_to_class` - if let Some(name) = self - .range_to_class - .iter() - .find(|(r, _)| *r == expr.range()) - .map(|(_, n)| n.clone()) - { - return name; - } - match expr { - Expr::BinOp(b) if matches!(b.op, Operator::BitOr) => self.render_union(expr), - Expr::BinOp(b) if matches!(b.op, Operator::BitAnd) => { - let parts = flatten_bitand(expr); - let rendered: Vec = - parts.iter().map(|p| self.render_field_type(p)).collect(); - self.field_imports.insert(("ty_extensions", "Intersection")); - format!( - "{}[{}]", - self.written.imported("ty_extensions", "Intersection"), - rendered.join(", ") - ) - } - Expr::Tuple(t) if t.parenthesized && !t.elts.is_empty() => { - let parts: Vec = t.elts.iter().map(|e| self.render_field_type(e)).collect(); - format!("{}[{}]", self.written.builtin("tuple"), parts.join(", ")) - } - // `T?` → `T | None` (nested `T??` → `Optional[T | None]`) so an - // optional field type composes instead of leaking `?` - Expr::UnaryOp(u) if matches!(u.op, ruff_python_ast::UnaryOp::Optional) => { - let mut depth: usize = 1; - let mut inner: &Expr = u.operand.as_ref(); - while let Expr::UnaryOp(u2) = inner { - if u2.op != ruff_python_ast::UnaryOp::Optional { - break; - } - depth += 1; - inner = u2.operand.as_ref(); - } - let inner_str = self.render_field_type(inner); - if depth >= 2 { - self.needs_optional_runtime = true; - } - format!( - "{}{inner_str} | None{}", - "Optional[".repeat(depth - 1), - "]".repeat(depth - 1) - ) - } - _ if is_literal_value(expr) => { - self.field_imports.insert(("typing", "Literal")); - format!( - "{}[{}]", - self.written.imported("typing", "Literal"), - self.render_subbed(expr.range()) - ) - } - _ => self.callable.lower_type_expr(expr), - } - } - - /// Render a `|` union, grouping consecutive bare literal arms into a - /// single `Literal[…]` subscript. - fn render_union(&mut self, expr: &Expr) -> String { - let parts = flatten_bitor(expr); - let mut out_groups: Vec = Vec::new(); - let mut pending_list: Vec = Vec::new(); - let literal = self.written.imported("typing", "Literal"); - let flush = |pending: &mut Vec, out: &mut Vec| { - if !pending.is_empty() { - out.push(format!("{literal}[{}]", pending.join(", "))); - pending.clear(); - } + extra_items = Some(field_type(namer, &inner.value)); + continue; }; - for p in parts { - if is_literal_value(p) { - self.field_imports.insert(("typing", "Literal")); - pending_list.push(self.render_subbed(p.range())); - } else { - flush(&mut pending_list, &mut out_groups); - out_groups.push(self.render_field_type(p)); - } + let Expr::StringLiteral(s) = key_expr else { + namer.errors.push(format!( + "typed-dict literal field key must be a string literal, got `{}`", + namer.src(key_expr.range()) + )); + return None; + }; + let name = s.value.to_str().to_owned(); + if !is_identifier(&name) { + namer.errors.push(format!( + "typed-dict literal field name `{name}` is not a valid Python identifier" + )); + return None; } - flush(&mut pending_list, &mut out_groups); - out_groups.join(" | ") - } + if fields.iter().any(|(n, _): &(String, String)| n == &name) { + namer + .errors + .push(format!("duplicate typed-dict literal field `{name}`")); + return None; + } + let type_source = field_type(namer, &item.value); + fields.push((name, type_source)); + } + fields.sort_by(|a, b| a.0.cmp(&b.0)); + Some(Shape { + fields, + extra_items, + }) +} - fn class_name_for(&mut self, shape: Shape, source_range: ruff_text_size::TextRange) -> String { - let name = if let Some(existing) = self.shapes.get(&shape) { - existing.clone() - } else { - let n = shape.class_name(self.written); - self.shapes.insert(shape, n.clone()); - n - }; - self.range_to_class.push((source_range, name.clone())); - self.callable.add_substitution(source_range, name.clone()); - name - } +/// A field's type as python source, reading the type parameters of the generics around it by +/// the names they have where the class is declared. +fn field_type(namer: &mut Namer, expr: &Expr) -> String { + let rendered = render_field_type(namer, expr); + namer.read_type_parameters(expr.range(), &rendered) +} - fn rewrite_dict_in_annotation(&mut self, dict: &ruff_python_ast::ExprDict) { - let Some(shape) = self.extract_shape(dict) else { - return; - }; - let class_name = self.class_name_for(shape, dict.range()); - self.needs_import = true; - self.edits.push(Fix::safe_edit(Edit::range_replacement( - class_name, - dict.range(), - ))); - } - - /// Recursively descend into an annotation, rewriting any contained dict - /// literals. Walks structurally relevant containers (`BinOp`, Subscript, - /// parenthesized Tuple, `BoolOp`) so that `dict[str, {"k": int}]`, - /// `{"k": int} | None`, etc. all lower correctly. - fn visit_annotation(&mut self, expr: &Expr) { - match expr { - Expr::Dict(d) => { - // Walk children first so nested dicts register before the - // outer renders its field types. - for item in &d.items { - self.visit_annotation(&item.value); - } - self.rewrite_dict_in_annotation(d); - } - Expr::BinOp(b) => { - self.visit_annotation(&b.left); - self.visit_annotation(&b.right); - } - Expr::Subscript(s) => { - self.visit_annotation(&s.slice); - } - Expr::Tuple(t) => { - for elt in &t.elts { - self.visit_annotation(elt); - } - } - Expr::List(l) => { - for elt in &l.elts { - self.visit_annotation(elt); +/// Render a field's type expression as Python source, applying +/// basedpython type-form lowerings (literal-type promotion, parenthesized +/// tuple types, intersection) recursively, and writing every type the walk +/// already named — a nested typed dict, an anonymous named tuple, an inline +/// protocol — as its class. +fn render_field_type(namer: &mut Namer, expr: &Expr) -> String { + if let Some(name) = namer.substitution_for(expr.range()) { + return name; + } + match expr { + Expr::BinOp(b) if matches!(b.op, Operator::BitOr) => render_union(namer, expr), + Expr::BinOp(b) if matches!(b.op, Operator::BitAnd) => { + let parts = flatten_bitand(expr); + let rendered: Vec = parts.iter().map(|p| render_field_type(namer, p)).collect(); + namer.imports.insert(("ty_extensions", "Intersection")); + format!( + "{}[{}]", + namer.written.imported("ty_extensions", "Intersection"), + rendered.join(", ") + ) + } + Expr::Tuple(t) if t.parenthesized && !t.elts.is_empty() => { + let parts: Vec = t.elts.iter().map(|e| render_field_type(namer, e)).collect(); + format!("{}[{}]", namer.written.builtin("tuple"), parts.join(", ")) + } + // `T?` → `T | None` (nested `T??` → `Optional[T | None]`) so an + // optional field type composes instead of leaking `?` + Expr::UnaryOp(u) if matches!(u.op, ruff_python_ast::UnaryOp::Optional) => { + let mut depth: usize = 1; + let mut inner: &Expr = u.operand.as_ref(); + while let Expr::UnaryOp(u2) = inner { + if u2.op != ruff_python_ast::UnaryOp::Optional { + break; } + depth += 1; + inner = u2.operand.as_ref(); } - // `**: T` marker is `Starred(Starred(T))`; recurse so nested dict - // literals inside an extra-items type are registered too - Expr::Starred(s) => { - self.visit_annotation(&s.value); + let inner_str = render_field_type(namer, inner); + if depth >= 2 { + namer.needs_optional_runtime = true; } - _ => {} + format!( + "{}{inner_str} | None{}", + "Optional[".repeat(depth - 1), + "]".repeat(depth - 1) + ) + } + _ if is_literal_value(expr) => { + namer.imports.insert(("typing", "Literal")); + format!( + "{}[{}]", + namer.written.imported("typing", "Literal"), + namer.src(expr.range()) + ) } + _ => namer.lower(expr), } } -impl TypeExprVisitor for TypedDictLiteral<'_> { - fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { - // `visit_annotation` is a deep recursive rewriter (it knows about - // BinOp, Subscript, Tuple, List, Starred, Dict). emit edits, stop - // the walker from descending further to avoid double-processing - self.visit_annotation(expr); - Recurse::Stop +/// Render a `|` union, grouping consecutive bare literal arms into a +/// single `Literal[…]` subscript. +fn render_union(namer: &mut Namer, expr: &Expr) -> String { + let parts = flatten_bitor(expr); + let mut out_groups: Vec = Vec::new(); + let mut pending_list: Vec = Vec::new(); + let literal = namer.written.imported("typing", "Literal"); + let flush = |pending: &mut Vec, out: &mut Vec| { + if !pending.is_empty() { + out.push(format!("{literal}[{}]", pending.join(", "))); + pending.clear(); + } + }; + for p in parts { + if is_literal_value(p) { + namer.imports.insert(("typing", "Literal")); + pending_list.push(namer.src(p.range()).to_owned()); + } else { + flush(&mut pending_list, &mut out_groups); + out_groups.push(render_field_type(namer, p)); + } } + flush(&mut pending_list, &mut out_groups); + out_groups.join(" | ") } /// True when `expr` is a basedpython "bare literal" eligible for promotion to @@ -471,110 +301,28 @@ fn flatten_bitand(expr: &Expr) -> Vec<&Expr> { out } -impl AstPass for TypedDictLiteralPass<'_> { +impl AstPass for TypedDictLiteralPass { fn lowering(&self) -> Option { Some(super::ast_driver::Lowering::TypedDictLiteral) } - /// a typed dict type it replaces moves into a class, whose field types the shared - /// type-expression lowerer prints, a `typeof` among them + /// a typed dict type it replaces moves into a class, whose field types the synthesized + /// type namer prints fn subsumes(&self) -> &'static [super::ast_driver::Lowering] { - &[ - super::ast_driver::Lowering::Callable, - super::ast_driver::Lowering::OptionalType, - super::ast_driver::Lowering::LiteralType, - super::ast_driver::Lowering::JustFloat, - super::ast_driver::Lowering::DynamicKeyword, - super::ast_driver::Lowering::FloatConst, - super::ast_driver::Lowering::Typeof, - ] + super::synthesized_types::CLASS_BODY } fn run(&self, _module: &mut ModModule, ctx: &mut PassContext) { - let mut inner = TypedDictLiteral::new( - self.source, - self.written, - self.quotes, - &self.config, - self.types, - ); - // the module as parsed, not the one the passes before this rewrote: the type - // information the field lowerer asks is about those nodes, and the edits are keyed - // on the source ranges both share - walk_type_positions(self.suite, None, &mut inner); - for (module, name) in &inner.field_imports { - ctx.required_imports - .push(self.written.import_from(module, &[name])); - } - if inner.needs_optional_runtime { - ctx.runtime.insert(crate::runtime::OPTIONAL); + // the class each one names, and the declarations, come from the namer's one walk over + // the module as parsed, which named every kind of synthesized type together + for (range, name) in ctx.synthesized.of_kind(SynthesizedKind::TypedDict) { + ctx.text_edits.push((range, name)); } - if inner.needs_import { - // synthesized classes use `closed=True` / `extra_items=T` (PEP 728), - // which the stdlib `typing.TypedDict` rejects on every released - // python. source from `typing_extensions` so the generated class - // body actually executes at runtime - ctx.required_imports.push( - self.written - .import_from("typing_extensions", &["TypedDict"]), - ); - // synthesized class defs are raw multi-line python source. push - // them into required_imports as a single non-`from` line so - // merge_from_imports leaves them untouched and they end up in - // the preamble verbatim - // the field lowerer's own classes come first: a field type may name one - let defs = format!("{}{}", inner.callable.class_defs(), inner.class_defs()); - if !defs.is_empty() { - ctx.required_imports - .push(defs.trim_end_matches('\n').to_owned()); - } - } - let (imports, helpers) = inner.callable.take_requirements(); - ctx.required_imports.extend(imports); - ctx.runtime.extend(helpers); - for fix in inner.edits { - for edit in fix.edits() { - let range = edit.range(); - let repl = edit.content().unwrap_or_default().to_owned(); - ctx.text_edits.push((range, repl)); - } - } - ctx.errors.extend(inner.errors); } } -/// AST-pass wrapper around [`TypedDictLiteral`] that surfaces the imports its -/// field types read back to the driver, so they are emitted alongside the -/// synthesized class definitions -pub(crate) struct TypedDictLiteralPass<'src> { - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - config: Config, - /// the module as parsed, which is what `types` answers for - suite: &'src [Stmt], - types: &'src dyn TypeInfo, -} - -impl<'src> TypedDictLiteralPass<'src> { - pub(crate) fn new( - source: &'src str, - written: WrittenNames<'src>, - quotes: &'src WrittenQuotes, - config: Config, - suite: &'src [Stmt], - types: &'src dyn TypeInfo, - ) -> Self { - Self { - source, - written, - quotes, - config, - suite, - types, - } - } -} +/// replaces each typed dict literal type with the `TypedDict` class it names +pub(crate) struct TypedDictLiteralPass; #[cfg(test)] mod tests { @@ -589,6 +337,32 @@ mod tests { ); } + /// the class is declared at module scope, so a field naming a type parameter reads it by + /// the name the generic declares it under there: the polyfill's `_T`, or the parameter + /// read off the generic it belongs to where the list stays native + #[test] + fn a_type_parameter_field_reads_the_declared_name() { + let source = indoc! {" + class K[T]: + def get(self) -> {\"a\": T}: ... + type F[U] = {\"b\": U} + "}; + let out = transpile(source, &Config::test_default()).unwrap(); + assert!( + out.contains(" a: \"_T\"\n") && out.contains(" b: \"_U\"\n"), + "got:\n{out}" + ); + let config = Config { + min_version: crate::PythonVersion::PY313, + ..Config::test_default() + }; + let out = transpile(source, &config).unwrap(); + assert!( + out.contains(" a: \"K.__type_params__[0]\"\n"), + "got:\n{out}" + ); + } + #[test] fn simple_annotation() { let out = transpile( @@ -716,6 +490,28 @@ mod tests { assert!(out.contains(" foo: \"Foo\"\n"), "got: {out}"); } + /// a field's type is a type position like any other, so it carries a type modifier: `literal + /// str` is written `LiteralString`, and `final`, which python has no spelling for, is written + /// as the type it marks. the field is quoted, and a quoted type cannot start with the blanks + /// the keyword was replaced with in the source + #[test] + fn a_type_modifier_on_a_field() { + let out = transpile( + indoc! {" + a: {\"s\": literal str, \"n\": final int, \"l\": list[final int], **: final str} + "}, + &Config::test_default(), + ) + .unwrap(); + assert!( + out.contains("(TypedDict, extra_items=\"str\"):"), + "got: {out}" + ); + assert!(out.contains(" s: \"LiteralString\"\n"), "got: {out}"); + assert!(out.contains(" n: \"int\"\n"), "got: {out}"); + assert!(out.contains(" l: \"list[int]\"\n"), "got: {out}"); + } + #[test] fn quoted_field_type_escapes_inner_quotes() { let out = transpile("a: {\"k\": \"Foo\"}\n", &Config::test_default()).unwrap(); diff --git a/crates/by_transforms/src/transforms/unpack.rs b/crates/by_transforms/src/transforms/unpack.rs index e44f91282f..fe8d37e185 100644 --- a/crates/by_transforms/src/transforms/unpack.rs +++ b/crates/by_transforms/src/transforms/unpack.rs @@ -9,6 +9,12 @@ //! //! `d[*a]` → `d[(*a,)]` //! +//! From 3.11 python unpacks a type with the star itself, by iterating it — which the +//! `typing_extensions.TypeAliasType` a `type` alias is polyfilled as below 3.12 cannot do. a +//! starred type alias keeps the `Unpack[...]` spelling until 3.12: +//! +//! `tuple[int, *Pair]` → `tuple[int, Unpack[Pair]]` +//! //! Also lowers basedpython's pack forwarding, which no python version accepts: //! //! `def f(**kwargs: **Kwargs)` → `def f(**kwargs: Kwargs.kwargs)` @@ -49,22 +55,48 @@ impl<'src> UnpackSyntax<'src> { } } -/// the ranges of the subscripts that are type expressions — the ones whose starred elements -/// are spelled `Unpack[...]` rather than as the tuple python reads a starred subscript as +/// what the type checker says about the stars of a module, read before any pass rewrites it #[derive(Default)] -pub(crate) struct TypeSubscripts(HashSet); +pub(crate) struct TypeSubscripts { + /// the ranges of the subscripts that are type expressions — the ones whose starred elements + /// are spelled `Unpack[...]` rather than as the tuple python reads a starred subscript as + types: HashSet, + /// the ranges of the stars whose operand is a type alias + unpacked_aliases: HashSet, +} -/// which subscripts in `stmts` are type expressions, as the type checker reads them +/// which subscripts in `stmts` are type expressions, and which stars unpack a type alias, as the +/// type checker reads them pub(crate) fn collect_type_subscripts(stmts: &[Stmt], types: &dyn TypeInfo) -> TypeSubscripts { + struct Aliases<'a> { + types: &'a dyn TypeInfo, + found: HashSet, + } + impl Visitor<'_> for Aliases<'_> { + fn visit_expr(&mut self, expr: &Expr) { + if let Expr::Starred(starred) = expr + && self.types.is_type_alias(&starred.value) + { + self.found.insert(starred.range()); + } + walk_expr(self, expr); + } + } let mut subscripts = TypeSubscripts::default(); walk_type_positions(stmts, Some(types), &mut subscripts); + let mut aliases = Aliases { + types, + found: HashSet::new(), + }; + aliases.visit_body(stmts); + subscripts.unpacked_aliases = aliases.found; subscripts } impl TypeExprVisitor for TypeSubscripts { fn visit(&mut self, expr: &Expr, _pos: TypePos) -> Recurse { if let Expr::Subscript(subscript) = expr { - self.0.insert(subscript.range()); + self.types.insert(subscript.range()); } Recurse::Descend } @@ -88,7 +120,7 @@ impl AstPass for UnpackSyntax<'_> { } ctx.text_edits.extend(pack.edits.into_inner()); - if self.config.min_version >= PythonVersion::PY311 { + if self.config.min_version >= PythonVersion::PY312 { return; } let mut state = State { @@ -97,6 +129,7 @@ impl AstPass for UnpackSyntax<'_> { needs_import: false, lowered_varargs: pack.lowered_varargs, type_subscripts: &self.type_subscripts, + star_syntax: self.config.min_version >= PythonVersion::PY311, }; for stmt in &module.body { state.visit_stmt(stmt); @@ -192,9 +225,21 @@ struct State<'a> { needs_import: bool, lowered_varargs: Vec, type_subscripts: &'a TypeSubscripts, + /// whether the target has pep 646's star in a subscript, so only a star python cannot + /// carry out at runtime, one unpacking a type alias, is spelled `Unpack[...]` + star_syntax: bool, } impl State<'_> { + /// whether `starred` is spelled `Unpack[...]` on this target + fn lowers(&self, starred: &ruff_python_ast::ExprStarred) -> bool { + !self.star_syntax + || self + .type_subscripts + .unpacked_aliases + .contains(&starred.range()) + } + /// `d[*a]` → `d[(*a,)]` and `d[*a, b]` → `d[(*a, b)]`: the tuple python 3.11 builds /// for a starred subscript, which earlier versions only accept parenthesized fn parenthesize_value_slice(&self, slice: &ruff_python_ast::ExprTuple) { @@ -207,6 +252,9 @@ impl State<'_> { } fn rewrite_subscript_starred(&mut self, starred: &ruff_python_ast::ExprStarred) { + if !self.lowers(starred) { + return; + } self.needs_import = true; let star_range = star_token_range(starred.range()); self.edits @@ -223,7 +271,7 @@ impl State<'_> { return; }; // a forwarded `ParamSpec` already became `P.args`, which is not an unpack - if self.lowered_varargs.contains(&starred.range()) { + if self.lowered_varargs.contains(&starred.range()) || !self.lowers(starred) { return; } self.needs_import = true; @@ -255,10 +303,11 @@ impl<'ast> Visitor<'ast> for State<'_> { walk_expr(self, expr); return; } - let is_type = self.type_subscripts.0.contains(&s.range()); + let is_type = self.type_subscripts.types.contains(&s.range()); match s.slice.as_ref() { Expr::Tuple(t) if !is_type + && !self.star_syntax && !t.parenthesized && !t.has_parameter_shape() && t.elts.iter().any(Expr::is_starred_expr) => @@ -371,6 +420,29 @@ mod tests { ); } + /// a star is carried out by iterating what it unpacks, which the + /// `typing_extensions.TypeAliasType` a type alias is below 3.12 cannot do, so a starred + /// alias keeps the `Unpack[...]` spelling on 3.11, in a subscript and on `*args` alike + #[test] + fn a_starred_type_alias_is_spelled_unpack_on_311() { + check_py311( + indoc! {" + type Pair = (int, str) + x: tuple[int, *Pair] = (1, 2, \"a\") + def f(*args: *Pair): ... + def g(*args: *tuple[int, ...]): ... + "}, + indoc! {" + from typing import Unpack + from typing_extensions import TypeAliasType + Pair = TypeAliasType(\"Pair\", tuple[int, str]) + x: tuple[int, Unpack[Pair]] = (1, 2, \"a\") + def f(*args: Unpack[Pair]): ... + def g(*args: *tuple[int, ...]): ... + "}, + ); + } + #[test] fn no_rewrite_on_311() { let config = Config { diff --git a/crates/by_transforms/src/type_info.rs b/crates/by_transforms/src/type_info.rs index 658c9989f0..ad73032d85 100644 --- a/crates/by_transforms/src/type_info.rs +++ b/crates/by_transforms/src/type_info.rs @@ -16,9 +16,12 @@ use ty_python_semantic::RuntimeSpelling; use ty_python_semantic::types::call_type_forms::CallTypeForm; use ty_python_semantic::types::exceptions::RaisesRuntimeTarget; use ty_python_semantic::types::ide_support::BaseTypeArgument; -use ty_python_semantic::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; +use ty_python_semantic::types::repeated_underscore::{ + RepeatedUnderscoreRead, UnderscoreLowering, UnderscoreRefusal, +}; use ty_python_semantic::types::{ - DisplaySettings, DynamicType, KnownClass, KnownInstanceType, Type, UnpackedKwargs, character, + DisplaySettings, DynamicType, KnownClass, KnownInstanceType, NameResolution, Type, + UnpackedKwargs, character, }; use ty_python_semantic::{ AnonNamedTupleClass, AnonNamedTupleId, Db, HasType, ImplicitReceiverReference, LoweringModel, @@ -478,6 +481,11 @@ pub(crate) trait TypeInfo { /// nested in the class body would not. `None` when ty did not index the reference fn reads_class_scope_binding(&self, reference: &ExprName) -> Option; + /// where python finds the binding of the name `reference` reads or writes: a type-parameter + /// list, a `global` declaration, the module, a scope of its own, or nowhere. `None` when ty + /// did not index the reference + fn name_resolution(&self, reference: &ExprName) -> Option; + /// whether `name`, read in an annotation, is a forward reference: a name the /// program binds, but not by the point python evaluates the annotation where /// it is written, as it does before 3.14 without `from __future__ import @@ -556,6 +564,10 @@ pub(crate) trait TypeInfo { /// `*args`, so a starred one can only be an unpack fn is_typevartuple(&self, expr: &Expr) -> bool; + /// whether `expr` is a type alias — a `type` statement's, or a `TypeAliasType` — which below + /// 3.12 is the `typing_extensions` backport, and cannot be unpacked by a star + fn is_type_alias(&self, expr: &Expr) -> bool; + /// whether `name` is unbound at the scope enclosing `anchor` /// (used to pick a fresh temp-variable name) fn is_unbound_at(&self, name: &str, anchor: &Expr) -> bool; @@ -817,6 +829,16 @@ pub(crate) trait TypeInfo { function: &StmtFunctionDef, ) -> Option>; + /// whether `name` reads `_` where one of a parameter list's repeated `_`s may be the + /// binding it reads — ty's answer, which it reports and the lowering refuses + fn repeated_underscore_read(&self, name: &ExprName) -> Option; + + /// basedpython: whether the body of `function` refers to the `_` its parameters bind — + /// reads or deletes it, or holds a scope that reads it from the function or declares it + /// `nonlocal`. the bindings ty sees answer it, so a binding the python needs is never left + /// out + fn refers_to_own_underscore(&self, function: &StmtFunctionDef) -> bool; + /// whether adding a type annotation to a bare `name = value` assignment in /// `class_def`'s body would change the class's runtime semantics — true for /// dataclass-like classes, framework models, `NamedTuple`s, `TypedDict`s @@ -1137,6 +1159,17 @@ impl TypeInfo for LoweringModel<'_> { ) } + fn is_type_alias(&self, expr: &Expr) -> bool { + // read as a type expression the alias is the `TypeAlias` it names, and read as a value + // the `TypeAliasType` object python binds + expr.inferred_type(self).is_some_and(|ty| { + matches!( + ty, + Type::TypeAlias(_) | Type::KnownInstance(KnownInstanceType::TypeAliasType(_)) + ) + }) + } + fn is_unbound_at(&self, name: &str, anchor: &Expr) -> bool { let db = self.db(); let file = self.program_file(); @@ -1260,6 +1293,10 @@ impl TypeInfo for LoweringModel<'_> { SemanticModel::reads_class_scope_binding(self, reference) } + fn name_resolution(&self, reference: &ExprName) -> Option { + SemanticModel::name_resolution(self, reference) + } + fn is_forward_reference(&self, name: &ExprName) -> Option { SemanticModel::is_forward_reference(self, name) } @@ -1501,6 +1538,14 @@ impl TypeInfo for LoweringModel<'_> { ty_python_semantic::types::ide_support::repeated_underscore_lowering(self, function) } + fn repeated_underscore_read(&self, name: &ExprName) -> Option { + ty_python_semantic::types::ide_support::repeated_underscore_read_of(self, name) + } + + fn refers_to_own_underscore(&self, function: &StmtFunctionDef) -> bool { + ty_python_semantic::types::ide_support::refers_to_own_underscore(self, function) + } + fn reads_names_bound_where_written(&self, expr: &Expr) -> bool { LoweringModel::reads_names_bound_where_written(self, expr) } @@ -1932,7 +1977,7 @@ fn resolvable_in_output( } /// the root names `text` reads, or `None` when it is not an expression at all -fn names_read(text: &str) -> Option> { +pub(crate) fn names_read(text: &str) -> Option> { let parsed = parse_expression(text).ok()?; let mut names = NameCollector(Vec::new()); ruff_python_ast::visitor::Visitor::visit_expr(&mut names, parsed.expr()); diff --git a/crates/by_transforms/tests/anon_named_tuple_runtime.rs b/crates/by_transforms/tests/anon_named_tuple_runtime.rs index aafef442e9..cf0d8842a5 100644 --- a/crates/by_transforms/tests/anon_named_tuple_runtime.rs +++ b/crates/by_transforms/tests/anon_named_tuple_runtime.rs @@ -8,9 +8,9 @@ //! exiting 0. That is exactly what an mdtest cannot see, so it is checked here. mod common; +mod interpreters; use std::fs; -use std::path::{Path, PathBuf}; use std::process::Command; use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; @@ -58,6 +58,38 @@ assert [x.name for x in xs] == ["dee"], "list" print("ok") "#; +/// the checker takes a plain tuple for the named tuple wherever one is expected — not only +/// in a return or an annotated assignment, but as a call's argument, a parameter's default, a +/// later assignment to a declared name, a dict's value, under a union, and inside another such +/// tuple — so each is one at runtime too, or the field access the checker passed raises +/// `AttributeError` +const EXPECTED_PROGRAM: &str = r#" +def f(v: (p: int, q: int)) -> int: + return v.p + +def g(v: (p: int, q: int) = (1, 2)) -> int: + return v.q + +def h() -> (p: int, q: int) | None: + return (3, 4) + +def nested(v: (inner: (a: int, b: int), q: int)) -> int: + return v.inner.b + +x: (p: int, q: int) +x = (5, 6) +d: dict[str, (p: int, q: int)] = {"k": (7, 8)} + +assert f((1, 2)) == 1, "argument" +assert f(v=(9, 0)) == 9, "keyword argument" +assert g() == 2, "default" +assert h()?.q == 4, "union return" +assert x.q == 6, "declared" +assert d["k"].p == 7, "dict value" +assert nested(((1, 2), 3)) == 2, "nested" +print("ok") +"#; + /// both spellings of a construction against the same annotation build the one /// class the transpiler emits, and that class's fields carry the *declared* /// types — which is why the checker gives the named spelling the declared shape @@ -223,28 +255,24 @@ print("ok") "#; fn run(program: &str) { - run_with( - program, - &Config { - min_version: PythonVersion::PY313, - ..Config::default() - }, - ); + run_with(program, SoundnessPositions::defaults()); } -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn run_with(program: &str, config: &Config) { - let Some(python) = common::python() else { - eprintln!("skipping anon named tuple runtime test: no interpreter found"); +/// `program` transpiled for the interpreter that runs it, with the soundness checks at +/// `soundness` +fn run_with(program: &str, soundness: SoundnessPositions) { + let Some(python) = common::python(PythonVersion::PY39) else { return; }; - let transpiled = transpile(program, config).expect("transpile should succeed"); + let config = Config { + min_version: python.version, + soundness, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -269,6 +297,11 @@ fn a_plain_tuple_under_an_alias_is_coerced() { run(ALIAS_PROGRAM); } +#[test] +fn a_plain_tuple_is_the_named_tuple_wherever_one_is_expected() { + run(EXPECTED_PROGRAM); +} + #[test] fn both_construction_spellings_build_one_class() { run(SPELLING_PROGRAM); @@ -319,24 +352,23 @@ fn built_values_pass_every_soundness_check() { NESTED_AND_UNION_PROGRAM, UNCHANGED_PROGRAM, ] { - run_with( - program, - &Config { - min_version: PythonVersion::PY313, - soundness: SoundnessPositions::all(), - ..Config::default() - }, - ); + run_with(program, SoundnessPositions::all()); } } -/// transpile every one of `files` as one project, into a fresh directory under the cargo temp -/// dir, or the first transpile error -fn transpile_project(name: &str, files: &[(&str, &str)]) -> Result { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); +/// transpile every one of `files` as one project for `target`, into a fresh directory under the +/// cargo temp dir, or the first transpile error +fn transpile_project( + name: &str, + target: PythonVersion, + files: &[(&str, &str)], +) -> Result { + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix(name) + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/"), @@ -346,7 +378,7 @@ fn transpile_project(name: &str, files: &[(&str, &str)]) -> Result Result Result Config { +fn config(target: PythonVersion, stamps: &[(&str, &str)]) -> Config { Config { - min_version: PythonVersion::PY313, + min_version: target, stamps: stamps .iter() .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) @@ -24,18 +25,14 @@ fn config(stamps: &[(&str, &str)]) -> Config { } } -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn run(program: &str, stamps: &[(&str, &str)]) { - let Some(python) = common::python() else { - eprintln!("skipping build stamp runtime test: no interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; - let transpiled = transpile(program, &config(stamps)).expect("transpile should succeed"); - let output = Command::new(&python) + let transpiled = + transpile(program, &config(python.version, stamps)).expect("transpile should succeed"); + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/builtin_names_runtime.rs b/crates/by_transforms/tests/builtin_names_runtime.rs index fee87a59dd..3772ef0d3e 100644 --- a/crates/by_transforms/tests/builtin_names_runtime.rs +++ b/crates/by_transforms/tests/builtin_names_runtime.rs @@ -88,10 +88,71 @@ assert f(2) == 2 print("ok") "#; +/// the `super` keyword is the builtin, in a module that binds the name for itself and in a +/// method with a parameter of that name +const SUPER_BINDING: &str = r#" +from typing import override + +class Base: + def m(self, n: int) -> int: + return n + +super = 5 + +class Sub(Base): + @override + def m(self, n: int) -> int: + return super.m(n) + super[Base].m(n) + 1 + +class Param(Base): + def p(self, super: int) -> int: + return super.m(super) + +def read() -> object: + return super + +# a lambda passes its own first parameter as the receiver, which here has a class whose method +# resolution order reaches `Mixin` after `Lam`, where the method's own receiver's does not +class Mixin(Base): + @override + def m(self, n: int) -> int: + return n * 100 + +class Lam(Base): + def via(self, other: Base) -> int: + reader = lambda receiver: super.m(1) + return reader(other) + +class Mixed(Lam, Mixin): ... + +assert Sub().m(1) == 3, Sub().m(1) +assert Param().p(4) == 4 +assert read() === 5 +assert Lam().via(Mixed()) == 100, Lam().via(Mixed()) +print("ok") +"#; + +/// `JustFloat` is the builtin `float` at runtime, so `get_type_hints` reads the builtin, even +/// where the module binds `float` before importing it +const JUST_FLOAT: &str = r#" +import builtins +import typing + +float = int +from ty_extensions import JustFloat + +def f(x: JustFloat): ... + +assert typing.get_type_hints(f)["x"] === builtins.float, typing.get_type_hints(f) +print("ok") +"#; + fn run_at(interpreter: &Interpreter, target: PythonVersion) { run_program_at(PROGRAM, interpreter, target); run_program_at(MODULE_BINDING, interpreter, target); run_program_at(GLOBAL_REBINDING, interpreter, target); + run_program_at(SUPER_BINDING, interpreter, target); + run_program_at(JUST_FLOAT, interpreter, target); } fn run_program_at(program: &str, interpreter: &Interpreter, target: PythonVersion) { @@ -126,9 +187,23 @@ fn a_builtin_a_lowering_writes_is_the_builtin_below_python_310() { } } +/// the newest interpreter, on python transpiled for its own version. the `override` the program +/// writes is `typing`'s from 3.12, so an older interpreter needs `typing_extensions` #[test] +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] fn a_builtin_a_lowering_writes_is_the_builtin() { - if let Some(interpreter) = interpreters::interpreters().pop() { - run_at(&interpreter, interpreter.version); - } + let found = interpreters::interpreters() + .into_iter() + .rev() + .find(|interpreter| { + interpreter.typing_extensions || interpreter.version >= PythonVersion::PY312 + }); + let Some(interpreter) = found else { + eprintln!("skipping: no interpreter found, with `typing_extensions` below 3.12"); + return; + }; + run_at(&interpreter, interpreter.version); } diff --git a/crates/by_transforms/tests/class_pattern_star_runtime.rs b/crates/by_transforms/tests/class_pattern_star_runtime.rs index dc3f04cb41..b1a3eba759 100644 --- a/crates/by_transforms/tests/class_pattern_star_runtime.rs +++ b/crates/by_transforms/tests/class_pattern_star_runtime.rs @@ -11,15 +11,18 @@ //! the star reads the last of `__match_args__` whatever its length, and that a //! trailing star matches exactly what leaving it out matches. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// basedpython whose module-level `assert`s exercise the runtime contract of a /// class pattern's starred wildcard. @@ -27,8 +30,13 @@ const PROGRAM: &str = r#" from dataclasses import dataclass +# a data class gets `__match_args__` only from python 3.10, and the polyfilled `match` runs +# below that too, where a positional class pattern has no positions to read without one. so +# each class spells its own @dataclass class Line: + __match_args__ = ("start", "mid", "stop", "end") + start: int mid: int stop: int @@ -72,6 +80,8 @@ assert first(line) == 0, "a trailing star claims no position of its own" # wildcards, which is the whole point of counting from the end @dataclass class Short: + __match_args__ = ("start", "end") + start: int end: int @@ -112,14 +122,14 @@ assert start_and_stop(line) == (0, 2), "a keyword after the star is unaffected" print("ok") "#; -fn run(python: &str, min_version: PythonVersion) { +fn run(python: &Interpreter, min_version: PythonVersion) { let config = Config { min_version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -135,29 +145,19 @@ fn run(python: &str, min_version: PythonVersion) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn starred_class_patterns_read_the_positions_they_name() { - let Some(python) = python() else { - eprintln!("skipping starred class pattern runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; - run(&python, PythonVersion::PY313); + run(&python, python.version); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn starred_class_patterns_run_under_the_match_polyfill() { // below 3.10 the emitted `match` is itself rewritten into an `if` chain, so // the filled-in wildcards are read by the polyfill's `__match_args__` helper // rather than by python's own `MATCH_CLASS` - let Some(python) = python() else { - eprintln!("skipping starred class pattern polyfill test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, PythonVersion::PY39); diff --git a/crates/by_transforms/tests/common/mod.rs b/crates/by_transforms/tests/common/mod.rs index 67064d0074..61dd80a575 100644 --- a/crates/by_transforms/tests/common/mod.rs +++ b/crates/by_transforms/tests/common/mod.rs @@ -1,26 +1,48 @@ //! Shared helpers for the runtime divergence tests. //! //! These tests transpile basedpython and run the output on a real interpreter, -//! so they all need to locate one. Variants that need more than a bare -//! interpreter — a third-party package, a specific feature probe — keep their -//! own locator next to the test that needs it. +//! so they all need to locate one. Variants that need more than a version of +//! their interpreter — a third-party package, a specific feature probe — find it +//! through [`crate::interpreters`] next to the test that needs it. -use std::process::Command; +use by_transforms::PythonVersion; -/// Locate a usable interpreter: `$PYTHON` first, then common names, newest -/// first. Returns `None` (the caller skips) when none is found. -pub(crate) fn python() -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3", "python"].map(String::from)); +use crate::interpreters::{self, Interpreter, oldest, probe}; - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", ""]) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) +/// the interpreter a test runs on, which is also the version it transpiles for. see +/// [`python_with`] +#[allow( + dead_code, + reason = "each test binary compiles this module, and one whose tests all need more of their \ + interpreter than a version calls only `python_with`" +)] +pub(crate) fn python(at_least: PythonVersion) -> Option { + python_with(at_least, "", "") +} + +/// the interpreter a test runs on, which is also the version it transpiles for, said on stderr: +/// `$PYTHON` when it is `at_least` or later and `probe`, python source, runs on it, which is the +/// interpreter the run was pinned to, then `python3` from the path, as the native backend's tests +/// take it, and otherwise the oldest that is. none is a skip, said so, with `needs` saying what +/// else the test needed of it when it needed more than a version +/// +/// a test transpiles for the interpreter it runs on rather than for a fixed version, so a run +/// pinned to an older python executes the lowerings that python gets. `at_least` is the oldest +/// version the test's program can be written for at all +pub(crate) fn python_with( + at_least: PythonVersion, + probe_source: &str, + needs: &str, +) -> Option { + let ambient = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let interpreter = match probe(ambient) { + Some((interpreter, _)) + if interpreter.version >= at_least && interpreter.runs(probe_source) => + { + interpreter + } + _ => oldest(at_least, probe_source, needs)?, + }; + interpreters::ran(&interpreter, interpreter.version); + Some(interpreter) } diff --git a/crates/by_transforms/tests/conformance_runtime.rs b/crates/by_transforms/tests/conformance_runtime.rs index 376b354134..3152a210e9 100644 --- a/crates/by_transforms/tests/conformance_runtime.rs +++ b/crates/by_transforms/tests/conformance_runtime.rs @@ -8,14 +8,25 @@ //! fallback, or a registration emitted before the interface's own class //! statement would all type-check and fail here. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! A conformance written in another module than its interface is run too, transpiled +//! through a project database as `by build` transpiles it, since the lowering reads the +//! other module's types. +//! +//! Each program is transpiled for whichever interpreter runs it; if none is found the +//! test skips, and says so, rather than fails. The cross-module programs write +//! `override`, which python has from 3.12, so an older interpreter needs +//! `typing_extensions`. +use std::fmt::Write as _; use std::process::Command; -use by_transforms::{Config, PythonVersion, transpile}; +use by_transforms::{Config, PythonVersion, transpile, transpile_typed}; +use ruff_db::files::system_path_to_file; +use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; +use ty_project::{ProjectMetadata, TestDb}; mod common; +mod interpreters; use common::python; /// basedpython whose module-level `assert`s exercise the conformance runtime @@ -152,23 +163,18 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn conformances_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping conformance runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -182,3 +188,104 @@ fn conformances_run_correctly() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +/// a project database over `files`, as the language server or `by build` would hold +fn project_db(files: &[(&str, &str)]) -> TestDb { + let mut db = TestDb::new(ProjectMetadata::new( + ruff_python_ast::name::Name::new_static(""), + SystemPathBuf::from("/"), + )); + db.init_program().expect("program init failed"); + for (path, source) in files { + db.write_file(path, source).expect("write file failed"); + } + db +} + +/// transpile every module of `files` for the interpreter that runs it, run `main`, and assert +/// it prints `expected` +/// +/// a conformance registers in one module and is read in another, so only executing the modules +/// together proves the two halves meet. two tests asserting each module's text once passed while +/// the program they described raised `AttributeError` +fn run_project(files: &[(&str, &str)], expected: &str) { + // an `override` is `typing_extensions`' before 3.12 + let Some(python) = common::python_with( + PythonVersion::PY39, + "import sys\nif sys.version_info < (3, 12): import typing_extensions", + "with `typing_extensions` before 3.12", + ) else { + return; + }; + let config = Config { + min_version: python.version, + ..Config::default() + }; + let db = project_db(files); + // a second database over the same files, for a pre-pass that rewrites a module's source + let rebuild = || Some(Box::new(project_db(files)) as Box); + let dir = tempfile::Builder::new() + .prefix("conformance") + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); + let mut transpiled = String::new(); + for (path, _) in files { + let file = system_path_to_file(&db, path).expect("file not in db"); + let out = transpile_typed(&db, file, &config, Some(&rebuild)) + .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); + let relative = path.trim_start_matches('/').replace(".by", ".py"); + writeln!(transpiled, "--- {relative} ---\n{out}").expect("write to a string"); + std::fs::write(dir.path().join(relative), out).expect("write module"); + } + let output = Command::new(&python.command) + .arg("main.py") + .current_dir(dir.path()) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled project failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), expected); +} + +/// the interface comes from one module, the conformance from another, and the consumer sees +/// both only through its imports +#[test] +fn an_imported_conformance_dispatches() { + run_project( + &[ + ("/iface.by", "protocol Show:\n def show(self) -> str\n"), + ( + "/adapters.by", + "from iface import Show\n\nextension str(Show):\n override def show(self) -> str:\n return \"OWN:\" + self\n", + ), + ( + "/main.by", + "import adapters\nfrom iface import Show\n\ndef render(value: Show) -> str:\n return value.show()\n\nprint(render(\"hi\"))\n", + ), + ], + "OWN:hi", + ); +} + +/// the module that declares the protocol-typed function is upstream of the conformance by +/// construction, so it can never see one. it still has to emit a dispatch +#[test] +fn a_conformance_declared_downstream_of_its_use_dispatches() { + run_project( + &[ + ( + "/lib.by", + "protocol Show:\n def show(self) -> str\n\ndef render(value: Show) -> str:\n return value.show()\n", + ), + ( + "/main.by", + "from lib import Show, render\n\nextension str(Show):\n override def show(self) -> str:\n return \"OWN:\" + self\n\nprint(render(\"hi\"))\n", + ), + ], + "OWN:hi", + ); +} diff --git a/crates/by_transforms/tests/conversion_runtime.rs b/crates/by_transforms/tests/conversion_runtime.rs index a714e52bc8..fe6294738c 100644 --- a/crates/by_transforms/tests/conversion_runtime.rs +++ b/crates/by_transforms/tests/conversion_runtime.rs @@ -20,6 +20,7 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile_analysed_at}; +mod common; mod interpreters; /// basedpython whose module-level `assert`s exercise the three dunders end to end @@ -247,13 +248,14 @@ assert c["x"] == 1 print("ok") "#; -/// transpile `program` for `min_version`, analysed at that version as a project requiring it -/// is, and run it on the oldest interpreter found of that version or later that `probe` runs -/// on, asserting it exits cleanly -fn run_transpiled(program: &str, min_version: PythonVersion, probe: &str, needs: &str) { - let Some(interpreter) = interpreters::oldest(min_version, probe, needs) else { +/// transpile `program` for the interpreter it runs on, analysed at that version as a project +/// requiring it is, and run it there, asserting it exits cleanly. the interpreter is the pinned +/// one, or the oldest found, of `at_least` or later that `probe` runs on +fn run_transpiled(program: &str, at_least: PythonVersion, probe: &str, needs: &str) { + let Some(interpreter) = common::python_with(at_least, probe, needs) else { return; }; + let min_version = interpreter.version; let config = Config { min_version, ..Config::default() @@ -273,17 +275,16 @@ fn run_transpiled(program: &str, min_version: PythonVersion, probe: &str, needs: String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); - interpreters::ran(&interpreter, min_version); } #[test] fn conversions_run_correctly() { - run_transpiled(PROGRAM, PythonVersion::PY313, "", ""); + run_transpiled(PROGRAM, PythonVersion::PY39, "", ""); } #[test] fn frozen_displays_run_correctly() { - run_transpiled(FROZEN_PROGRAM, PythonVersion::PY313, "", ""); + run_transpiled(FROZEN_PROGRAM, PythonVersion::PY39, "", ""); } #[test] diff --git a/crates/by_transforms/tests/decorated_binding_runtime.rs b/crates/by_transforms/tests/decorated_binding_runtime.rs index 0130b1ec30..c3100b9f19 100644 --- a/crates/by_transforms/tests/decorated_binding_runtime.rs +++ b/crates/by_transforms/tests/decorated_binding_runtime.rs @@ -20,6 +20,7 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; /// basedpython whose module-level `assert`s exercise the lowering end to end const PROGRAM: &str = r#" @@ -95,22 +96,17 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn decorated_bindings_run_correctly() { - let Some(python) = common::python() else { - eprintln!("skipping decorated-binding runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/destructure_runtime.rs b/crates/by_transforms/tests/destructure_runtime.rs index a1e670cfdc..be424be6ef 100644 --- a/crates/by_transforms/tests/destructure_runtime.rs +++ b/crates/by_transforms/tests/destructure_runtime.rs @@ -9,15 +9,19 @@ //! of its conjuncts matches would fail here even though the type-level tests //! pass. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. A destructuring is written as a `match` statement, so the program needs +//! python 3.10, and an older target is refused use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// basedpython whose module-level `assert`s exercise the destructuring runtime /// contract end to end. @@ -202,14 +206,14 @@ assert classify(Point(0, 2), 1) == "other" print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -225,13 +229,8 @@ fn run(python: &str, program: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn destructuring_runs_correctly() { - let Some(python) = python() else { - eprintln!("skipping destructuring runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY310) else { return; }; @@ -239,16 +238,37 @@ fn destructuring_runs_correctly() { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_subject_written_over_several_lines_keeps_its_grouping() { - let Some(python) = python() else { - eprintln!( - "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY310) else { return; }; run(&python, MULTI_LINE); } + +/// every binding position, each on its own, refused on a target without `match` +#[test] +fn destructuring_is_refused_below_3_10() { + let forms = [ + "let (a, b) := (1, 2)\n", + "for int(n) in [1]:\n pass\n", + "with open('f') as int(n):\n pass\n", + "def f(int(n): int):\n pass\n", + "class C:\n let (a, b) := (1, 2)\n", + ]; + for min_version in [PythonVersion::PY38, PythonVersion::PY39] { + for form in forms { + let config = Config { + min_version, + ..Config::default() + }; + let error = transpile(form, &config).expect_err(&format!("{form:?} on {min_version}")); + assert!( + error.starts_with(&format!( + "Cannot use a destructuring pattern, which is written as a `match` statement, \ + on Python {min_version} (syntax was added in Python 3.10)" + )), + "{form:?} on {min_version}: {error}" + ); + } + } +} diff --git a/crates/by_transforms/tests/discard_return_runtime.rs b/crates/by_transforms/tests/discard_return_runtime.rs index 43090bc68e..9818a6917e 100644 --- a/crates/by_transforms/tests/discard_return_runtime.rs +++ b/crates/by_transforms/tests/discard_return_runtime.rs @@ -16,10 +16,13 @@ //! - attributes still answer, so a framework reading `cb.__name__` off a wrapped //! callback gets the wrapped function's name rather than an `AttributeError` -use std::process::{Command, Stdio}; +use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; +mod interpreters; + /// basedpython whose module-level `assert`s exercise the adapter end to end const PROGRAM: &str = r#" calls: list[str] = [] @@ -134,42 +137,18 @@ assert plain === silent print("ok") "#; -/// the first interpreter that accepts `probe` -fn python_supporting(probe: &str) -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3"].map(String::from)); - - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", probe]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) -} - #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn discarded_returns_run_correctly() { - let Some(python) = python_supporting("import sys; sys.exit(0)") else { - eprintln!("skipping discard-return runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/enum_variant_value_runtime.rs b/crates/by_transforms/tests/enum_variant_value_runtime.rs index de0be61cae..26d85ebb74 100644 --- a/crates/by_transforms/tests/enum_variant_value_runtime.rs +++ b/crates/by_transforms/tests/enum_variant_value_runtime.rs @@ -9,6 +9,7 @@ //! `enum.Enum`, which gets all three right, so the two lowerings have to agree. mod common; +mod interpreters; use std::process::Command; @@ -65,23 +66,18 @@ def main() -> None: "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn enum_unit_variant_round_trips_to_itself() { - let Some(python) = common::python() else { - eprintln!("skipping enum variant value runtime test: no interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/except_polyfill_runtime.rs b/crates/by_transforms/tests/except_polyfill_runtime.rs new file mode 100644 index 0000000000..9292667310 --- /dev/null +++ b/crates/by_transforms/tests/except_polyfill_runtime.rs @@ -0,0 +1,83 @@ +//! Runtime divergence test for the bracketless `except` of PEP 758. +//! +//! python 3.14 reads `except A, B:` as a handler for either type, and an older python +//! cannot parse it. The transpiler parenthesizes the types for a target before 3.14, and +//! this runs the result to show the handler still catches each type it names, and only +//! those. +//! +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +mod common; +mod interpreters; +use common::python; + +const PROGRAM: &str = r#" +def caught(error: Exception) -> str: + try: + raise error + except KeyError, IndexError: + return "lookup" + except ValueError, TypeError: + return "value" + except Exception: + return "other" + +assert caught(KeyError()) == "lookup" +assert caught(IndexError()) == "lookup" +assert caught(TypeError()) == "value" +assert caught(RuntimeError()) == "other" +print("ok") +"#; + +/// an exception group's members are matched by an `except*` written the same way +const EXCEPT_STAR: &str = r#" +seen: list[str] = [] +try: + raise ExceptionGroup("both", [KeyError(), ValueError()]) +except* KeyError, IndexError: + seen.append("lookup") +except* ValueError, TypeError: + seen.append("value") + +assert seen == ["lookup", "value"] +print("ok") +"#; + +fn run(program: &str, at_least: PythonVersion) { + let Some(python) = python(at_least) else { + return; + }; + let config = Config { + min_version: python.version, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + let output = Command::new(&python.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "transpiled program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +#[test] +fn a_bracketless_except_catches_each_type_it_names() { + run(PROGRAM, PythonVersion::PY39); +} + +#[test] +fn a_bracketless_except_star_catches_each_type_it_names() { + run(EXCEPT_STAR, PythonVersion::PY311); +} diff --git a/crates/by_transforms/tests/extension_receiver_runtime.rs b/crates/by_transforms/tests/extension_receiver_runtime.rs index 4bdb4c8bfa..16b6db6241 100644 --- a/crates/by_transforms/tests/extension_receiver_runtime.rs +++ b/crates/by_transforms/tests/extension_receiver_runtime.rs @@ -9,14 +9,15 @@ //! left on the backing function, `@classmethod` makes it a `classmethod` object, which python //! cannot call //! -//! no third-party packages are needed, so any `python3` will do; if none is found the test skips -//! rather than fails +//! no third-party packages are needed, so the program is transpiled for whichever interpreter +//! runs it; if none is found the test skips, and says so, rather than fails use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; const PROGRAM: &str = r##" @@ -65,23 +66,18 @@ print("ok") "##; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn extension_members_receive_their_receiver() { - let Some(python) = python() else { - eprintln!("skipping extension receiver runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/flexible_keyword_runtime.rs b/crates/by_transforms/tests/flexible_keyword_runtime.rs index 42aba18f4b..3c64d43bf2 100644 --- a/crates/by_transforms/tests/flexible_keyword_runtime.rs +++ b/crates/by_transforms/tests/flexible_keyword_runtime.rs @@ -7,15 +7,18 @@ //! evaluated them. A lowering that gets any of those wrong still produces //! python that parses and runs. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// the keys a call actually delivers, for every shape of name const KEYS: &str = r#" @@ -91,13 +94,13 @@ assert collect(a.b=(7, 8).0, *[]) == {"a.b": 7}, "and so does a reordered one" print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -114,7 +117,7 @@ fn run(python: &str, program: &str) { #[test] fn a_flexible_name_arrives_as_the_key_it_was_written_as() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, KEYS); @@ -122,7 +125,7 @@ fn a_flexible_name_arrives_as_the_key_it_was_written_as() { #[test] fn a_lowered_call_keeps_its_argument_and_evaluation_order() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, ORDER); @@ -130,7 +133,7 @@ fn a_lowered_call_keeps_its_argument_and_evaluation_order() { #[test] fn a_lowering_inside_a_flexible_arguments_value_still_applies() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, COMPOSITION); diff --git a/crates/by_transforms/tests/forward_reference_runtime.rs b/crates/by_transforms/tests/forward_reference_runtime.rs index d4546f9530..82633f27c7 100644 --- a/crates/by_transforms/tests/forward_reference_runtime.rs +++ b/crates/by_transforms/tests/forward_reference_runtime.rs @@ -23,6 +23,7 @@ mod common; use common::python; mod interpreters; +use interpreters::Interpreter; /// every place an annotation runs with the definition, naming something bound after it const PROGRAM: &str = r#" @@ -61,8 +62,8 @@ print("ok") "#; /// whether `python` still evaluates annotations as the definition runs -fn evaluates_annotations_eagerly(python: &str) -> bool { - Command::new(python) +fn evaluates_annotations_eagerly(python: &Interpreter) -> bool { + Command::new(&python.command) .args([ "-c", "import sys; raise SystemExit(sys.version_info >= (3, 14))", @@ -77,7 +78,7 @@ fn evaluates_annotations_eagerly(python: &str) -> bool { reason = "a skipped test must say why it skipped, or it reads as a pass" )] fn a_forward_reference_does_not_raise_at_import() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY310) else { return; }; if !evaluates_annotations_eagerly(&python) { @@ -85,11 +86,11 @@ fn a_forward_reference_does_not_raise_at_import() { return; } let config = Config { - min_version: PythonVersion::PY310, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -243,15 +244,15 @@ print("ok") #[test] fn a_forward_reference_outside_an_annotation_resolves() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY310) else { return; }; let config = Config { - min_version: PythonVersion::PY310, + min_version: python.version, ..Config::default() }; let transpiled = transpile(OUTSIDE_AN_ANNOTATION, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -304,8 +305,8 @@ print(get_type_hints(uses)) "#; /// what `program` prints on `python` -fn printed(python: &str, program: &str) -> String { - let output = Command::new(python) +fn printed(python: &Interpreter, program: &str) -> String { + let output = Command::new(&python.command) .arg("-c") .arg(program) .output() @@ -320,11 +321,11 @@ fn printed(python: &str, program: &str) -> String { #[test] fn a_forward_reference_outside_an_annotation_survives_the_round_trip() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY310) else { return; }; let config = Config { - min_version: PythonVersion::PY310, + min_version: python.version, ..Config::default() }; let by = reverse_transpile(QUOTED_OUTSIDE_AN_ANNOTATION, &config) diff --git a/crates/by_transforms/tests/fstring_debug_runtime.rs b/crates/by_transforms/tests/fstring_debug_runtime.rs index da0d531034..a73c08303b 100644 --- a/crates/by_transforms/tests/fstring_debug_runtime.rs +++ b/crates/by_transforms/tests/fstring_debug_runtime.rs @@ -16,6 +16,7 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; mod interpreters; use interpreters::Interpreter; @@ -36,10 +37,6 @@ def show(a: int?, b: int, d: dict[str, int]) -> None: assert f"{(a!)=!s}" == "(a!)=3", "conversion" # braces in the author's text, which are not a field of their own assert f"{({1: a}[1]!) = }" == "({1: a}[1]!) = 3", "braces" - # a backslash, which python prints as the two characters it was written as - assert f"{('a\n'!) = }" == "('a\\n'!) = 'a\\n'", "backslash" - # the string's own quote, nested the way 3.12 allows - assert f"{(d["k"]!) = }" == '(d["k"]!) = 5', "nested quote" # and a field nothing lowered, which keeps python's own compact rendering assert f"{b = }" == "b = 7", "unlowered" assert f"{ {1: 2}[1] = }" == " {1: 2}[1] = 2", "unlowered braces" @@ -57,8 +54,6 @@ def show(a, b, d): assert f"{(a)=:>6}" == "(a)= 3", "format spec" assert f"{(a)=!s}" == "(a)=3", "conversion" assert f"{({1: a}[1]) = }" == "({1: a}[1]) = 3", "braces" - assert f"{('a\n') = }" == "('a\\n') = 'a\\n'", "backslash" - assert f"{(d["k"]) = }" == '(d["k"]) = 5', "nested quote" assert f"{b = }" == "b = 7", "unlowered" assert f"{ {1: 2}[1] = }" == " {1: 2}[1] = 2", "unlowered braces" @@ -66,6 +61,28 @@ show(3, 7, {"k": 5}) print("ok") "#; +/// fields only 3.12 lets a replacement field hold: a backslash, and the string's own quote +const PEP_701_PROGRAM: &str = r#" +def show(a: int?, d: dict[str, int]) -> None: + # a backslash, which python prints as the two characters it was written as + assert f"{('a\n'!) = }" == "('a\\n'!) = 'a\\n'", "backslash" + # the string's own quote, nested + assert f"{(d["k"]!) = }" == '(d["k"]!) = 5', "nested quote" + +show(3, {"k": 5}) +print("ok") +"#; + +/// [`PEP_701_PROGRAM`] with the basedpython operators taken out +const PEP_701_REFERENCE: &str = r#" +def show(a, d): + assert f"{('a\n') = }" == "('a\\n') = 'a\\n'", "backslash" + assert f"{(d["k"]) = }" == '(d["k"]) = 5', "nested quote" + +show(3, {"k": 5}) +print("ok") +"#; + fn run(python: &Interpreter, source: &str, what: &str) { let output = Command::new(&python.command) .arg("-c") @@ -80,21 +97,38 @@ fn run(python: &Interpreter, source: &str, what: &str) { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } +/// run `reference` on `python`, then `program` transpiled for it +fn run_both(python: &Interpreter, program: &str, reference: &str) { + run(python, reference, "the plain-python reference"); + + let config = Config { + min_version: python.version, + ..Config::default() + }; + let transpiled = transpile(program, &config).expect("transpile should succeed"); + run(python, &transpiled, "the transpiled program"); +} + #[test] fn a_debug_field_prints_the_source_the_author_wrote() { - // a replacement field may nest the string's own quote from 3.12 on, and hold a backslash - // from the same release. the program is transpiled for 3.13 - let Some(python) = interpreters::oldest(PythonVersion::PY313, "", "") else { + let Some(python) = common::python(PythonVersion::PY39) else { return; }; + run_both(&python, PROGRAM, PYTHON_REFERENCE); +} - run(&python, PYTHON_REFERENCE, "the plain-python reference"); - - let config = Config { - min_version: PythonVersion::PY313, - ..Config::default() +/// a replacement field may nest the string's own quote from 3.12 on, and hold a backslash from +/// the same release, so this program is written for 3.12. but until 3.12.9 and 3.13.2 python +/// printed a debug field's backslash decoded rather than as it was written (gh-124363), so the +/// reference is only python's word on a release that has the fix +#[test] +fn a_debug_field_holding_a_backslash_or_its_own_quote_prints_the_source() { + let Some(python) = common::python_with( + PythonVersion::PY312, + "import sys; v = sys.version_info[:3]; assert v >= (3, 13, 2) or (3, 12, 9) <= v < (3, 13)", + "that prints a debug field's text as it was written (3.12.9 or 3.13.2)", + ) else { + return; }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - run(&python, &transpiled, "the transpiled program"); - interpreters::ran(&python, PythonVersion::PY313); + run_both(&python, PEP_701_PROGRAM, PEP_701_REFERENCE); } diff --git a/crates/by_transforms/tests/generated_names_runtime.rs b/crates/by_transforms/tests/generated_names_runtime.rs index b2e1d56abd..96bf98b5fa 100644 --- a/crates/by_transforms/tests/generated_names_runtime.rs +++ b/crates/by_transforms/tests/generated_names_runtime.rs @@ -14,19 +14,22 @@ //! rename that made a temporary ordinary again would fail here even though the //! type-level tests pass. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; -/// basedpython whose module-level `assert`s exercise every lowering that mints -/// a temporary, in both namespaces that record what is assigned in them. -const PROGRAM: &str = r#" +/// what both programs below share: the values they destructure and match, and the +/// helpers that find a lowering's temporaries in a namespace +const COMMON: &str = r#" from contextlib import contextmanager from typing import Iterator @@ -61,8 +64,59 @@ def machinery(names: object) -> list[str]: # underscores and no trailing pair is name-mangled onto the class def recorded(names: object) -> list[str]: return [n for n in machinery(names) if not (n.startswith("__") and n.endswith("__"))] +"#; +/// basedpython whose module-level `assert`s exercise every lowering that mints a temporary +/// for any target, in both namespaces that record what is assigned in them +const ANY_TARGET: &str = r#" # a plain class body: every temporary would be a class attribute +class Holder: + match origin(): + case Point(mx, my) and object(): + pass + + seen.append(maybe() ?? 0) + + if True: + chosen = if True: + 1 + else: + 2 + +assert (Holder.mx, Holder.my) == (0, 0), "a `match` with a conjunction binds its captures" +assert Holder.chosen == 1, "a class-body statement expression still yields" +assert seen == [7], "a class-body `??` still evaluates" +assert not recorded(vars(Holder)), "no temporary is an ordinary class attribute" + +# an `enum class` body: every temporary would be a *member*, and `enum` records +# the name on assignment, so a `del` afterwards cannot take it back +enum class Colour: + case Red, Green + + match origin(): + case Point(cx, cy) and object(): + pass + + seen.append(maybe() ?? 0) + + if True: + picked = if True: + 1 + else: + 2 + +assert Colour.Red.name == "Red" and Colour.Green.name == "Green", ( + "the declared variants are still the enum's own" +) +assert not recorded(vars(Colour)), "no temporary is an ordinary name in the enum" +assert not machinery(Colour.__members__), "and none of them became a member" +assert not machinery(n.name for n in Colour), "or a variant the enum iterates" + +print("ok") +"#; + +/// the same for every destructuring, which needs python 3.10 +const DESTRUCTURING: &str = r#" class Holder: let Point(hx, hy) := origin() else: raise TypeError @@ -70,10 +124,6 @@ class Holder: if let Point(iax, iay) := origin(): pass - match origin(): - case Point(mx, my) and object(): - pass - if let Point(x=int() and 0, y=ay) := origin(): pass @@ -83,26 +133,13 @@ class Holder: with borrow(origin()) as Point(wx, wy): pass - seen.append(maybe() ?? 0) - - if True: - chosen = if True: - 1 - else: - 2 - assert (Holder.hx, Holder.hy) == (0, 0), "a class-body `let` binds its captures" assert (Holder.iax, Holder.iay) == (0, 0), "so does an `if let`" -assert (Holder.mx, Holder.my) == (0, 0), "so does a `match` with a conjunction" assert Holder.ay == 0, "so does a hoisted conjunction" assert (Holder.fx, Holder.fy) == (1, 2), "so does a `for` pattern" assert (Holder.wx, Holder.wy) == (0, 0), "so does a `with` pattern" -assert Holder.chosen == 1, "a class-body statement expression still yields" -assert seen == [7], "a class-body `??` still evaluates" assert not recorded(vars(Holder)), "no temporary is an ordinary class attribute" -# an `enum class` body: every temporary would be a *member*, and `enum` records -# the name on assignment, so a `del` afterwards cannot take it back enum class Colour: case Red, Green @@ -112,10 +149,6 @@ enum class Colour: if let Point(bx, by) := origin(): pass - match origin(): - case Point(cx, cy) and object(): - pass - if let Point(x=int() and 0, y=ny) := origin(): pass @@ -125,14 +158,6 @@ enum class Colour: with borrow(origin()) as Point(vx, vy): pass - seen.append(maybe() ?? 0) - - if True: - picked = if True: - 1 - else: - 2 - assert Colour.Red.name == "Red" and Colour.Green.name == "Green", ( "the declared variants are still the enum's own" ) @@ -143,24 +168,16 @@ assert not machinery(n.name for n in Colour), "or a variant the enum iterates" print("ok") "#; -#[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] -fn generated_names_stay_out_of_the_namespace() { - let Some(python) = python() else { - eprintln!("skipping generated-name runtime test: no `python3` interpreter found"); - return; - }; - +/// transpile `body` after [`COMMON`] for the interpreter that runs it, and run it +fn run(python: &Interpreter, body: &str) { + let program = format!("{COMMON}{body}"); let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let transpiled = transpile(&program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -174,3 +191,19 @@ fn generated_names_stay_out_of_the_namespace() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } + +#[test] +fn generated_names_stay_out_of_the_namespace() { + let Some(python) = python(PythonVersion::PY39) else { + return; + }; + run(&python, ANY_TARGET); +} + +#[test] +fn destructuring_names_stay_out_of_the_namespace() { + let Some(python) = python(PythonVersion::PY310) else { + return; + }; + run(&python, DESTRUCTURING); +} diff --git a/crates/by_transforms/tests/grapheme_runtime.rs b/crates/by_transforms/tests/grapheme_runtime.rs index b3c3e32aba..9b9190d215 100644 --- a/crates/by_transforms/tests/grapheme_runtime.rs +++ b/crates/by_transforms/tests/grapheme_runtime.rs @@ -18,6 +18,7 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; mod interpreters; /// basedpython source whose module-level `assert`s exercise the grapheme @@ -92,23 +93,23 @@ assert Letters.firsts == ["a"], "in a class body comprehension" print("ok") "#; -fn transpiled(program: &str) -> String { +fn transpiled(program: &str, target: PythonVersion) -> String { let config = Config { - min_version: PythonVersion::PY313, + min_version: target, ..Config::default() }; transpile(program, &config).expect("transpile should succeed") } fn run_with_regex(program: &str) { - let Some(interpreter) = interpreters::oldest( - PythonVersion::PY313, + let Some(interpreter) = common::python_with( + PythonVersion::PY39, "import regex", "with `regex` (`pip install regex` into one, or set `PYTHON` to one that has it)", ) else { return; }; - let transpiled = transpiled(program); + let transpiled = transpiled(program, interpreter.version); let output = Command::new(&interpreter.command) .arg("-c") @@ -123,7 +124,6 @@ fn run_with_regex(program: &str) { String::from_utf8_lossy(&output.stderr), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); - interpreters::ran(&interpreter, PythonVersion::PY313); } #[test] @@ -140,10 +140,10 @@ fn a_bound_receiver_runs_where_python_refuses_an_assignment_expression() { /// refuses one is a `SyntaxError` before the helpers that need `regex` ever run #[test] fn a_bound_receiver_compiles_where_python_refuses_an_assignment_expression() { - let Some(interpreter) = interpreters::oldest(PythonVersion::PY313, "", "") else { + let Some(interpreter) = common::python(PythonVersion::PY39) else { return; }; - let transpiled = transpiled(BOUND_RECEIVER); + let transpiled = transpiled(BOUND_RECEIVER, interpreter.version); let output = Command::new(&interpreter.command) .args([ @@ -159,5 +159,4 @@ fn a_bound_receiver_compiles_where_python_refuses_an_assignment_expression() { "the transpiled grapheme program does not compile on {interpreter}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", String::from_utf8_lossy(&output.stderr), ); - interpreters::ran(&interpreter, PythonVersion::PY313); } diff --git a/crates/by_transforms/tests/if_let_runtime.rs b/crates/by_transforms/tests/if_let_runtime.rs index 9c02f09130..70848d6c3b 100644 --- a/crates/by_transforms/tests/if_let_runtime.rs +++ b/crates/by_transforms/tests/if_let_runtime.rs @@ -8,15 +8,18 @@ //! on a real interpreter. A selector that let two clauses run, or an eagerly //! evaluated subject, would fail here even though the type-level tests pass. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// basedpython whose module-level `assert`s exercise the `if let` runtime /// contract end to end. @@ -105,14 +108,14 @@ assert unboxed(Box(None), False) == 0 print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -128,13 +131,8 @@ fn run(python: &str, program: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn if_let_chains_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping `if let` runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY310) else { return; }; @@ -142,16 +140,35 @@ fn if_let_chains_run_correctly() { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_subject_written_over_several_lines_keeps_its_grouping() { - let Some(python) = python() else { - eprintln!( - "skipping `a_subject_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY310) else { return; }; run(&python, MULTI_LINE); } + +/// an `if let` and an `elif let` are written as a `match` statement, so a target without one +/// refuses them +#[test] +fn if_let_is_refused_below_3_10() { + let forms = [ + "x = [1]\nif let [a] := x:\n pass\n", + "x = [1]\nif x:\n pass\nelif let [a] := x:\n pass\n", + ]; + for min_version in [PythonVersion::PY38, PythonVersion::PY39] { + for form in forms { + let config = Config { + min_version, + ..Config::default() + }; + let error = transpile(form, &config).expect_err(&format!("{form:?} on {min_version}")); + assert!( + error.starts_with(&format!( + "Cannot use a destructuring pattern, which is written as a `match` statement, \ + on Python {min_version} (syntax was added in Python 3.10)" + )), + "{form:?} on {min_version}: {error}" + ); + } + } +} diff --git a/crates/by_transforms/tests/implicit_receiver_runtime.rs b/crates/by_transforms/tests/implicit_receiver_runtime.rs index f1191a5a5e..56f59a6eba 100644 --- a/crates/by_transforms/tests/implicit_receiver_runtime.rs +++ b/crates/by_transforms/tests/implicit_receiver_runtime.rs @@ -10,14 +10,16 @@ //! whose parameters don't line up with the call would fail here even though the //! type-level tests pass. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; /// basedpython whose module-level `assert`s exercise the implicit-receiver @@ -79,23 +81,18 @@ print("ok") "##; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn implicit_receivers_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping implicit-receiver runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/imported_conversion_class_runtime.rs b/crates/by_transforms/tests/imported_conversion_class_runtime.rs index fda4d77211..361d237c29 100644 --- a/crates/by_transforms/tests/imported_conversion_class_runtime.rs +++ b/crates/by_transforms/tests/imported_conversion_class_runtime.rs @@ -11,7 +11,6 @@ //! rather than the single-file `transpile`. use std::fs; -use std::path::{Path, PathBuf}; use std::process::Command; use by_transforms::{Config, PythonVersion, transpile_typed}; @@ -64,11 +63,13 @@ const FILES: &[(&str, &str)] = &[ /// transpile every file of the project for `target` into a fresh directory under the cargo /// temp dir -fn build(target: PythonVersion) -> PathBuf { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("imported_conversion_class"); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); +fn build(target: PythonVersion) -> tempfile::TempDir { + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix("imported_conversion_class") + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), @@ -87,7 +88,7 @@ fn build(target: PythonVersion) -> PathBuf { let transpiled = transpile_typed(&db, file, &config, None) .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } dir } @@ -110,7 +111,7 @@ fn a_conversion_target_is_imported_clear_of_every_other_name() { "transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- main.py ---\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), - fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + fs::read_to_string(dir.path().join("main.py")).unwrap_or_default(), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); diff --git a/crates/by_transforms/tests/imported_extension_runtime.rs b/crates/by_transforms/tests/imported_extension_runtime.rs index 4ef16c1fec..d113962fd2 100644 --- a/crates/by_transforms/tests/imported_extension_runtime.rs +++ b/crates/by_transforms/tests/imported_extension_runtime.rs @@ -16,7 +16,6 @@ //! rather than the single-file `transpile`. use std::fs; -use std::path::{Path, PathBuf}; use std::process::Command; use by_transforms::{Config, PythonVersion, transpile_typed}; @@ -94,11 +93,13 @@ const FILES: &[(&str, &str)] = &[("/extensions.by", EXTENSIONS), ("/main.by", MA /// transpile every file of the project for `target` into a fresh directory under the cargo /// temp dir -fn build(target: PythonVersion) -> PathBuf { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("imported_extension"); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); +fn build(target: PythonVersion) -> tempfile::TempDir { + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix("imported_extension") + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), @@ -117,7 +118,7 @@ fn build(target: PythonVersion) -> PathBuf { let transpiled = transpile_typed(&db, file, &config, None) .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } dir } @@ -140,7 +141,7 @@ fn an_imported_backing_function_is_bound_clear_of_the_modules_own_names() { "transpiled program failed on {interpreter}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- main.py ---\n{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), - fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + fs::read_to_string(dir.path().join("main.py")).unwrap_or_default(), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); interpreters::ran(&interpreter, target); diff --git a/crates/by_transforms/tests/inferred_annotation_runtime.rs b/crates/by_transforms/tests/inferred_annotation_runtime.rs index b0014ebbe5..a74058644b 100644 --- a/crates/by_transforms/tests/inferred_annotation_runtime.rs +++ b/crates/by_transforms/tests/inferred_annotation_runtime.rs @@ -10,7 +10,6 @@ //! asks; where they are not, it is imported for a checker alone. use std::fs; -use std::path::Path; use std::process::Command; use by_transforms::{Config, PythonVersion, transpile, transpile_typed}; @@ -171,11 +170,12 @@ fn an_inferred_annotation_adds_no_import_cycle() { db.write_file(path, source).expect("write file failed"); } for lazy_imports in [true, false] { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")) - .join(format!("inferred_annotation_cycle_{lazy_imports}")); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); + // a directory of its own, so a run beside another, from the same checkout, neither + // reads nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix(&format!("inferred_annotation_cycle_{lazy_imports}")) + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); let config = Config { min_version: PythonVersion::PY310, lazy_imports, @@ -186,9 +186,9 @@ fn an_inferred_annotation_adds_no_import_cycle() { let transpiled = transpile_typed(&db, file, &config, None) .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } - let m = fs::read_to_string(dir.join("m.py")).expect("read m"); + let m = fs::read_to_string(dir.path().join("m.py")).expect("read m"); // a deferred `make` is read through the module's `_LazyFrom` let make = if lazy_imports { "_by_lazy.make" diff --git a/crates/by_transforms/tests/inherited_defaults_runtime.rs b/crates/by_transforms/tests/inherited_defaults_runtime.rs index bc538c6bca..b183aad9c3 100644 --- a/crates/by_transforms/tests/inherited_defaults_runtime.rs +++ b/crates/by_transforms/tests/inherited_defaults_runtime.rs @@ -5,15 +5,18 @@ //! to the same value. Asserting on the lowered *text* cannot see that: a default written in //! the wrong place still parses, and a call binding the wrong value still runs. //! -//! No third-party packages are needed, so any `python3` will do; if none is found the test -//! skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// each shape a default can be declared in, taken by an override that declares none const INHERITED: &str = r#" @@ -96,13 +99,13 @@ assert B(7).a == 7, "and an argument still wins" print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -119,7 +122,7 @@ fn run(python: &str, program: &str) { #[test] fn an_override_is_callable_without_the_argument_the_base_defaulted() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, INHERITED); @@ -127,7 +130,7 @@ fn an_override_is_callable_without_the_argument_the_base_defaulted() { #[test] fn a_default_reaches_every_override_down_the_chain() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, CHAIN); @@ -135,7 +138,7 @@ fn a_default_reaches_every_override_down_the_chain() { #[test] fn a_parameter_after_an_inherited_default_still_binds() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, RELAXED_ORDER); @@ -143,7 +146,7 @@ fn a_parameter_after_an_inherited_default_still_binds() { #[test] fn an_init_shorthand_takes_the_default_its_base_declares() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, INIT_SHORTHAND); diff --git a/crates/by_transforms/tests/interpreters/mod.rs b/crates/by_transforms/tests/interpreters/mod.rs index 0ae3db04e7..a3a11fcdd6 100644 --- a/crates/by_transforms/tests/interpreters/mod.rs +++ b/crates/by_transforms/tests/interpreters/mod.rs @@ -29,7 +29,7 @@ impl std::fmt::Display for Interpreter { impl Interpreter { /// whether `probe`, python source, runs to completion here - fn runs(&self, probe: &str) -> bool { + pub(crate) fn runs(&self, probe: &str) -> bool { Command::new(&self.command) .args(["-c", probe]) .stdout(Stdio::null()) @@ -59,31 +59,40 @@ pub(crate) fn interpreters() -> Vec { let mut found: Vec = candidates .into_iter() .filter_map(|command| { - let output = Command::new(&command) - .args(["-c", PROBE]) - .stderr(Stdio::null()) - .output() - .ok() - .filter(|output| output.status.success())?; - let stdout = String::from_utf8(output.stdout).ok()?; - let mut fields = stdout.trim_end().splitn(4, ' '); - let major = fields.next()?.parse().ok()?; - let minor = fields.next()?.parse().ok()?; - let typing_extensions = fields.next()? == "1"; - let executable = fields.next()?; - let executable = std::fs::canonicalize(executable) - .unwrap_or_else(|_| std::path::PathBuf::from(executable)); - seen.insert(executable).then(|| Interpreter { - command, - version: PythonVersion::from((major, minor)), - typing_extensions, - }) + let (interpreter, executable) = probe(command)?; + seen.insert(executable).then_some(interpreter) }) .collect(); found.sort_by_key(|interpreter| interpreter.version); found } +/// the interpreter `command` names, if it runs, and the file it runs from +pub(crate) fn probe(command: String) -> Option<(Interpreter, std::path::PathBuf)> { + let output = Command::new(&command) + .args(["-c", PROBE]) + .stderr(Stdio::null()) + .output() + .ok() + .filter(|output| output.status.success())?; + let stdout = String::from_utf8(output.stdout).ok()?; + let mut fields = stdout.trim_end().splitn(4, ' '); + let major = fields.next()?.parse().ok()?; + let minor = fields.next()?.parse().ok()?; + let typing_extensions = fields.next()? == "1"; + let executable = fields.next()?; + let executable = + std::fs::canonicalize(executable).unwrap_or_else(|_| std::path::PathBuf::from(executable)); + Some(( + Interpreter { + command, + version: PythonVersion::from((major, minor)), + typing_extensions, + }, + executable, + )) +} + /// the oldest interpreter of `at_least` or later on which `probe`, python source, runs. none is a /// skip, said so, with `needs` saying what else the test needed of it when it needed more than a /// version diff --git a/crates/by_transforms/tests/lazy_import_runtime.rs b/crates/by_transforms/tests/lazy_import_runtime.rs index 38fd171daf..020cbf2113 100644 --- a/crates/by_transforms/tests/lazy_import_runtime.rs +++ b/crates/by_transforms/tests/lazy_import_runtime.rs @@ -13,7 +13,7 @@ //! other assertion while quietly destroying the feature use std::fs; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; use by_transforms::{Config, transpile, transpile_typed}; @@ -764,17 +764,23 @@ fn python() -> Option { interpreters::oldest(Config::default().min_version, "", "") } +/// a fresh directory for `case` under the cargo temp dir, of its own, so a run beside another, +/// from the same checkout, neither reads nor deletes the other's modules +fn case_dir(case: &str) -> tempfile::TempDir { + tempfile::Builder::new() + .prefix(case) + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir") +} + /// transpile `sources` (`module stem` → basedpython source) into a fresh /// directory under the cargo temp dir -fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); +fn build_case(case: &str, sources: &[(&str, &str)]) -> tempfile::TempDir { + let dir = case_dir(case); for (stem, source) in sources { let transpiled = transpile(source, &Config::default()).expect("transpile should succeed"); - let path = dir.join(format!("{stem}.py")); + let path = dir.path().join(format!("{stem}.py")); if let Some(parent) = path.parent() { fs::create_dir_all(parent).expect("create package dir"); } @@ -788,10 +794,9 @@ fn build_case(case: &str, sources: &[(&str, &str)]) -> PathBuf { /// the reference fn agrees_with_python(python: &Interpreter, case: &str, sources: &[(&str, &str)]) { let transpiled = build_case(case, sources); - let reference = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!("{case}_python")); - let _ = fs::remove_dir_all(&reference); + let reference = case_dir(&format!("{case}_python")); for (stem, source) in sources { - let path = reference.join(format!("{stem}.py")); + let path = reference.path().join(format!("{stem}.py")); if let Some(parent) = path.parent() { fs::create_dir_all(parent).expect("create package dir"); } @@ -813,10 +818,10 @@ fn agrees_with_python(python: &Interpreter, case: &str, sources: &[(&str, &str)] // the two trees are in directories of their own, which a message naming a file says String::from_utf8_lossy(&output.stdout).replace(&dir.display().to_string(), "") }; - let expected = run(&reference); + let expected = run(reference.path()); assert!(!expected.trim().is_empty(), "the reference printed nothing"); assert_eq!( - run(&transpiled), + run(transpiled.path()), expected, "the transpiled program disagrees with python" ); @@ -850,7 +855,7 @@ fn an_imported_value_is_the_object_itself() { "lazy_values", &[("by_values_mod", VALUES_MODULE), ("main", VALUES_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -862,7 +867,7 @@ fn character_identity_survives_an_import() { "char_identity", &[("by_char_mod", CHAR_MODULE), ("main", CHAR_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -874,7 +879,7 @@ fn import_stays_lazy_until_first_use() { "lazy_defer", &[("by_lazy_mod", LAZY_MODULE), ("main", LAZY_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -886,7 +891,7 @@ fn a_name_is_claimed_only_once_the_module_is_lazy() { "lazy_window", &[("by_watched_mod", WATCHED_MODULE), ("main", WINDOW_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -895,7 +900,7 @@ fn an_unlazifiable_name_in_a_multi_name_import_stays_bound() { return; }; let dir = build_case("lazy_mixed", &[("main", MIXED_MAIN)]); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -904,7 +909,7 @@ fn a_submodule_resolves_on_first_use() { return; }; let dir = build_case("lazy_submodule", &[("main", SUBMODULE_MAIN)]); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -913,7 +918,7 @@ fn an_annotation_reads_the_imported_object() { return; }; let dir = build_case("lazy_annotations", &[("main", ANNOTATIONS_MAIN)]); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -925,7 +930,7 @@ fn a_string_annotation_resolves_to_the_imported_object() { "lazy_future_annotations", &[("main", FUTURE_ANNOTATIONS_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -934,7 +939,7 @@ fn the_polyfill_reads_builtins_the_module_binds_for_itself() { return; }; let dir = build_case("lazy_builtin_names", &[("main", BUILTIN_NAMES_MAIN)]); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -946,7 +951,7 @@ fn a_read_in_every_scope_is_the_object_itself() { "lazy_scopes", &[("by_token_mod", TOKEN_MODULE), ("main", SCOPES_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -958,7 +963,7 @@ fn a_name_bound_again_reads_as_python_does() { "lazy_rebound", &[("by_token_mod", TOKEN_MODULE), ("main", REBOUND_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -970,7 +975,7 @@ fn a_read_before_the_import_runs_raises_python_s_name_error() { "lazy_early", &[("by_token_mod", TOKEN_MODULE), ("main", EARLY_MAIN)], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -988,7 +993,7 @@ fn a_deferred_name_is_passed_on_to_other_modules() { ("main", REEXPORT_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -1006,7 +1011,7 @@ fn modules_importing_each_other_resolve_on_use() { ("main", CYCLE_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -1022,7 +1027,7 @@ fn a_patch_of_the_importing_module_reaches_its_reads() { ("main", PATCH_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -1038,10 +1043,7 @@ fn a_check_on_a_deferred_class_is_made_once_its_import_has_run() { for (path, source) in ONCE_BOUND { db.write_file(path, source).expect("write file failed"); } - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("lazy_once_bound"); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); + let dir = case_dir("lazy_once_bound"); for (path, _) in ONCE_BOUND { let file = system_path_to_file(&db, path).expect("file not in db"); let transpiled = transpile_typed(&db, file, &Config::default(), None) @@ -1055,13 +1057,13 @@ fn a_check_on_a_deferred_class_is_made_once_its_import_has_run() { ); } let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } - run_main(&python, &dir); + run_main(&python, dir.path()); } /// transpile each of `files` against a project db holding all of them, into `case`'s directory -fn build_project(case: &str, files: &[(&str, &str)]) -> PathBuf { +fn build_project(case: &str, files: &[(&str, &str)]) -> tempfile::TempDir { let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/"), @@ -1070,16 +1072,13 @@ fn build_project(case: &str, files: &[(&str, &str)]) -> PathBuf { for (path, source) in files { db.write_file(path, source).expect("write file failed"); } - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); + let dir = case_dir(case); for (path, _) in files { let file = system_path_to_file(&db, path).expect("file not in db"); let transpiled = transpile_typed(&db, file, &Config::default(), None) .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } dir } @@ -1090,7 +1089,7 @@ fn a_hoisted_class_s_quoted_field_binds_what_it_reads() { return; }; let dir = build_case("lazy_hoisted_fields", &[("main", HOISTED_FIELDS)]); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -1099,9 +1098,9 @@ fn a_quote_in_a_type_argument_of_a_sibling_module_s_generic_binds_what_it_reads( return; }; let dir = build_project("lazy_quoted_through_a_sibling", QUOTED_THROUGH_A_SIBLING); - let main = fs::read_to_string(dir.join("main.py")).expect("read main"); + let main = fs::read_to_string(dir.path().join("main.py")).expect("read main"); assert!(main.contains("Box[\"F | JSONDecoder\"]"), "{main}"); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] @@ -1118,7 +1117,7 @@ fn a_relative_import_of_the_package_reads_its_attribute_or_its_submodule() { ("main", RELATIVE_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] diff --git a/crates/by_transforms/tests/mutable_defaults_runtime.rs b/crates/by_transforms/tests/mutable_defaults_runtime.rs index 1c66cded4a..d94c153435 100644 --- a/crates/by_transforms/tests/mutable_defaults_runtime.rs +++ b/crates/by_transforms/tests/mutable_defaults_runtime.rs @@ -8,15 +8,18 @@ //! program returns the wrong answer (`1 is int` is `False` where //! `isinstance(1, int)` is `True`). //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// every lowering that can be written inside a default, checked against the /// same expression evaluated as ordinary body code @@ -332,13 +335,13 @@ assert parameters["s"].default == "a", repr(parameters["s"].default) print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -355,7 +358,7 @@ fn run(python: &str, program: &str) { #[test] fn a_lowering_inside_a_default_survives_the_move_into_the_body() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, COMPOSITION); @@ -363,7 +366,7 @@ fn a_lowering_inside_a_default_survives_the_move_into_the_body() { #[test] fn a_context_argument_inside_a_default_survives_the_move_into_the_body() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, CONTEXT); @@ -371,22 +374,15 @@ fn a_context_argument_inside_a_default_survives_the_move_into_the_body() { #[test] fn an_extension_members_default_is_hoisted_below_the_preamble() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, EXTENSION); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_default_written_over_several_lines_survives_the_move_into_the_body() { - let Some(python) = python() else { - eprintln!( - "skipping `a_default_written_over_several_lines_survives_the_move_into_the_body`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, MULTI_LINE); @@ -397,11 +393,11 @@ fn a_default_written_over_several_lines_survives_the_move_into_the_body() { /// target below 3.10 gets #[test] fn a_leading_bom_stays_ahead_of_the_preamble() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; - - for version in [PythonVersion::PY313, PythonVersion::PY39] { + // the interpreter's own version takes the sentinel alone, and 3.9 the future import too + for version in [python.version, PythonVersion::PY39] { let config = Config { min_version: version, ..Config::default() @@ -418,7 +414,7 @@ fn a_leading_bom_stays_ahead_of_the_preamble() { let path = dir.path().join("emitted.py"); std::fs::write(&path, &transpiled).expect("failed to write the module"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg(&path) .output() .expect("failed to spawn python"); @@ -435,7 +431,7 @@ fn a_leading_bom_stays_ahead_of_the_preamble() { #[test] fn a_name_default_shows_as_written_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, NAME_DEFAULT); @@ -443,7 +439,7 @@ fn a_name_default_shows_as_written_in_the_signature() { #[test] fn a_call_default_shows_as_written_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, CALL_DEFAULT); @@ -451,7 +447,7 @@ fn a_call_default_shows_as_written_in_the_signature() { #[test] fn a_late_bound_default_shows_as_written_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, LATE_BOUND_DEFAULT); @@ -459,7 +455,7 @@ fn a_late_bound_default_shows_as_written_in_the_signature() { #[test] fn an_init_field_default_shows_as_written_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, INIT_FIELD_DEFAULT); @@ -467,7 +463,7 @@ fn an_init_field_default_shows_as_written_in_the_signature() { #[test] fn a_required_parameter_after_a_default_shows_as_required_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, REQUIRED_AFTER_DEFAULT); @@ -475,7 +471,7 @@ fn a_required_parameter_after_a_default_shows_as_required_in_the_signature() { #[test] fn a_scalar_default_shows_its_value_in_the_signature() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, SCALAR_DEFAULT); @@ -483,7 +479,7 @@ fn a_scalar_default_shows_its_value_in_the_signature() { #[test] fn a_copied_or_pickled_marker_is_still_the_default() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, COPIED_MARKER); @@ -491,7 +487,7 @@ fn a_copied_or_pickled_marker_is_still_the_default() { #[test] fn another_defaults_marker_is_a_value() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, ANOTHER_DEFAULTS_MARKER); @@ -499,7 +495,7 @@ fn another_defaults_marker_is_a_value() { #[test] fn a_marker_shows_its_default_on_one_line() { - let Some(python) = python() else { + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, MULTI_LINE_MARKER); diff --git a/crates/by_transforms/tests/none_coalesce_runtime.rs b/crates/by_transforms/tests/none_coalesce_runtime.rs index 62ac371c01..b08de55d2c 100644 --- a/crates/by_transforms/tests/none_coalesce_runtime.rs +++ b/crates/by_transforms/tests/none_coalesce_runtime.rs @@ -12,15 +12,17 @@ //! - where python refuses `:=`: a comprehension's iterable, and any part of a //! comprehension in a class body //! -//! no third-party packages are needed, so any `python3` will do; if none is found the -//! test skips rather than fails +//! no third-party packages are needed, so the program is transpiled for whichever interpreter +//! runs it; if none is found the test skips, and says so, rather than fails use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// a `??` inside an operand of another one const NESTED: &str = r#" @@ -214,13 +216,13 @@ assert issubclass(Generic, Base) and annotated == 1 print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -236,58 +238,35 @@ fn run(python: &str, program: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_coalesce_inside_an_operand_is_lowered() { - let Some(python) = python() else { - eprintln!( - "skipping `a_coalesce_inside_an_operand_is_lowered`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, NESTED); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_coalesce_keeps_its_grouping() { - let Some(python) = python() else { - eprintln!("skipping `a_coalesce_keeps_its_grouping`: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, GROUPING); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression() { - let Some(python) = python() else { - eprintln!( - "skipping `a_coalesce_binds_its_value_where_python_refuses_an_assignment_expression`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, BINDING); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope() { - let Some(python) = python() else { - eprintln!( - "skipping `a_coalesce_binds_its_value_in_an_annotation_and_a_type_parameter_scope`: no `python3` interpreter found" - ); + // below 3.12 the `type` statement is written by the generics polyfill, which does not yet + // carry a `??` lowered inside the alias's value: the transpile is refused as a transform + // conflict + let Some(python) = python(PythonVersion::PY312) else { return; }; run(&python, ANNOTATION_SCOPES); diff --git a/crates/by_transforms/tests/optional_chain_runtime.rs b/crates/by_transforms/tests/optional_chain_runtime.rs index 50e3dc95ee..3036da38f8 100644 --- a/crates/by_transforms/tests/optional_chain_runtime.rs +++ b/crates/by_transforms/tests/optional_chain_runtime.rs @@ -8,14 +8,15 @@ //! refuses the `:=` that binds the temporary — a comprehension's iterable, and any //! part of a comprehension in a class body //! -//! no third-party packages are needed, so any `python3` will do; if none is found the -//! test skips rather than fails +//! no third-party packages are needed, so the program is transpiled for whichever interpreter +//! runs it; if none is found the test skips, and says so, rather than fails use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; const CHAINS: &str = r#" @@ -100,22 +101,17 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn optional_chains_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping optional chain runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(CHAINS, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/parametric_is_runtime.rs b/crates/by_transforms/tests/parametric_is_runtime.rs index a82a1c7e71..744a10bab8 100644 --- a/crates/by_transforms/tests/parametric_is_runtime.rs +++ b/crates/by_transforms/tests/parametric_is_runtime.rs @@ -12,17 +12,20 @@ //! method-bearing protocol degrades to an unchecked pass-through rather than //! raising a `TypeError` from an `isinstance` against the protocol. //! -//! The programs are split by interpreter requirement: [`PROGRAM`] needs only -//! PEP 695 (3.12+), while [`DEFAULTS_PROGRAM`] needs PEP 696 defaults in native -//! syntax and `TypeVar.has_default()` (3.13+). Each is probed for what it -//! actually uses — a 3.12 interpreter parses PEP 695 quite happily, so a probe -//! that only checked that would run the defaults program and die on a -//! `SyntaxError`. Whichever has no capable interpreter skips rather than fails. +//! Each program is transpiled for the interpreter it runs on, so below 3.12 and +//! 3.13 its generics reach the PEP 695 and PEP 696 polyfills. A reified type +//! parameter is the exception: the transpiler refuses one below 3.12, so the case +//! that needs one is [`REIFIED_CELL_PROGRAM`], and an older interpreter asserts the +//! refusal instead of running it. -use std::process::{Command, Stdio}; +use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; +mod interpreters; +use interpreters::Interpreter; + /// basedpython whose module-level `assert`s exercise the structural protocol /// check end to end: an exact match, a mismatch, an inferred class-body /// annotation, a covariant read-only member, a nested-generic member, a missing @@ -325,14 +328,6 @@ class StrList(list[str]): ... assert (IntList() cast? list[int]) is not None, "recorded arguments agree" assert (StrList() cast? list[int]) is None, "recorded arguments contradict" -# a value typed by a *reified* type parameter carries the answer in a runtime -# cell, so the cast is exact — the same `T == int` lowering `is` uses -def cast_cell[T](data: list[T]) -> list[int] | None: - return data cast? list[int] - -assert cast_cell([1, 2]) is not None, "reified cell says int" -assert cast_cell(["a"]) is None, "reified cell says str" - # a member declared inside `__init__` has no annotation any runtime can read, # and the checker accepts the class as satisfying the protocol regardless — so # answering `False` would be a silent contradiction. the check refuses instead @@ -364,10 +359,20 @@ else: print("ok") "#; -/// The pep 696 half, split out because native defaults (`[T = int]`) and the -/// `has_default()` accessor both need 3.13 — a 3.12 interpreter parses the pep -/// 695 syntax in [`PROGRAM`] quite happily, so the two must be probed -/// separately or a 3.12 box runs this and dies on a `SyntaxError`. +/// a value typed by a *reified* type parameter, which carries the answer in a runtime cell +const REIFIED_CELL_PROGRAM: &str = r#" +# the cast is exact — the same `T == int` lowering `is` uses +def cast_cell[T](data: list[T]) -> list[int] | None: + return data cast? list[int] + +assert cast_cell([1, 2]) is not None, "reified cell says int" +assert cast_cell(["a"]) is None, "reified cell says str" + +print("ok") +"#; + +/// the pep 696 half: a parameter's default, which the probe reads in place of an argument +/// the value was never given const DEFAULTS_PROGRAM: &str = r#" from typing import Never, Protocol # a class records its generic bases *unsubstituted* (`class L[T = Never] @@ -406,10 +411,6 @@ assert (OwnNever() is OwnNever[int]) is False, "and is not some other argument" print("ok") "#; -/// Locate a usable 3.13 interpreter: `$PYTHON` first, then common names. -/// Returns `None` (test skips) when none is found — PEP 695 class syntax is a -/// hard requirement here. -/// pep 695 type parameters — what [`PROGRAM`] needs (3.12+) /// basedpython whose module-level `assert`s exercise the runtime check each /// *non-parametric* target lowers to. The checker's own tests pin the type it /// reads and the transpiler's pin the text it emits; only running the result @@ -708,46 +709,48 @@ assert str_is_named("a") is False print("ok") "#; -const PEP695_PROBE: &str = "type X[T] = T"; - -/// pep 696 defaults in native syntax plus the `has_default()` accessor the probe -/// reads them back through — what [`DEFAULTS_PROGRAM`] needs (3.13+) -const PEP696_PROBE: &str = "\ -class _P[T = int](list[T]): pass -assert _P.__type_params__[0].has_default() -"; - -/// Locate an interpreter satisfying `probe`: `$PYTHON` first, then common names. -/// `None` (test skips) when none qualifies. -fn python_supporting(probe: &str) -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3"].map(String::from)); - - candidates.into_iter().find(|py| { - // a rejected probe writes a `SyntaxError` to stderr; swallow it so a - // passing run's log doesn't look like a failure - Command::new(py) - .args(["-c", probe]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) +/// generic classes whose bases fix or reorder the arguments, which the probe of a value typed +/// `object` resolves down the declared base chain. below 3.12 the pep 695 polyfill declares the parameters through +/// `Generic[...]`, so the chain is read from what that records +const BASE_CHAIN_PROGRAM: &str = r#" +class Odd[T](list[int]): ... + +class Swap[A, B](dict[B, A]): ... + +def is_list_int(x: object) -> bool: + return x is list[int] + +def is_list_str(x: object) -> bool: + return x is list[str] + +def is_dict_str_int(x: object) -> bool: + return x is dict[str, int] + +def is_dict_int_str(x: object) -> bool: + return x is dict[int, str] + +assert is_list_int(Odd[str]()), "the base fixes `int`" +assert not is_list_str(Odd[str]()), "`T` is not list's argument" +assert is_dict_str_int(Swap[int, str]()), "the base reorders" +assert not is_dict_int_str(Swap[int, str]()), "and not the other way round" + +print("ok") +"#; + +/// transpile `program` for `python` and run its module-level `assert`s on it +fn run_program(python: &Interpreter, program: &str) { + run_program_for(python, program, python.version); } -/// transpile `program` and run its module-level `assert`s on `python` -fn run_program(python: &str, program: &str) { +/// transpile `program` for `min_version` and run its module-level `assert`s on `python` +fn run_program_for(python: &Interpreter, program: &str, min_version: PythonVersion) { let config = Config { - min_version: PythonVersion::PY313, + min_version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -755,7 +758,7 @@ fn run_program(python: &str, program: &str) { assert!( output.status.success(), - "transpiled parametric-is program failed on {python}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + "transpiled parametric-is program failed on {python} at {min_version}:\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); @@ -763,68 +766,77 @@ fn run_program(python: &str, program: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn parametric_protocol_checks_run_correctly() { - let Some(python) = python_supporting(PEP695_PROBE) else { - eprintln!("skipping parametric-is runtime test: no PEP 695-capable interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; run_program(&python, PROGRAM); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] +fn a_reified_cell_makes_a_cast_exact() { + let Some(python) = common::python(PythonVersion::PY39) else { + return; + }; + // the reified cell is a pep 695 closure cell, which needs native type-parameter syntax + if python.version < PythonVersion::PY312 { + let config = Config { + min_version: python.version, + ..Config::default() + }; + let refusal = transpile(REIFIED_CELL_PROGRAM, &config) + .expect_err("a reified generic is refused below python 3.12"); + assert!( + refusal.contains("requires python 3.12 or newer"), + "{refusal}" + ); + return; + } + run_program(&python, REIFIED_CELL_PROGRAM); +} + +#[test] fn every_target_kind_checks_correctly_at_runtime() { - let Some(python) = python_supporting(PEP695_PROBE) else { - eprintln!("skipping type-test target runtime test: no PEP 695-capable interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; run_program(&python, TARGETS_PROGRAM); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn type_param_defaults_resolve_at_runtime() { - let Some(python) = python_supporting(PEP696_PROBE) else { - eprintln!( - "skipping parametric-is defaults test: no PEP 696-capable (3.13+) interpreter found" - ); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; run_program(&python, DEFAULTS_PROGRAM); } -/// a reified class needs 3.13, so this is probed like [`DEFAULTS_PROGRAM`] #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] +fn a_base_chain_resolves_on_native_and_polyfilled_parameters() { + let Some(python) = common::python(PythonVersion::PY39) else { + return; + }; + // the interpreter's own version declares the parameters natively from 3.12, and 3.11 is + // the newest version that still polyfills them + for target in [python.version, PythonVersion::PY311] { + if target <= python.version { + run_program_for(&python, BASE_CHAIN_PROGRAM, target); + } + } +} + +/// a reified class needs 3.13 +#[test] fn a_check_written_from_a_float_type_checks_just_float() { - let Some(python) = python_supporting(PEP696_PROBE) else { - eprintln!("skipping just-float runtime test: no 3.13+ interpreter found"); + let Some(python) = common::python(PythonVersion::PY313) else { return; }; run_program(&python, JUST_FLOAT_PROGRAM); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_target_another_lowering_rewrites_checks_correctly_at_runtime() { - let Some(python) = python_supporting(PEP695_PROBE) else { - eprintln!("skipping lowered-target runtime test: no PEP 695-capable interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; run_program(&python, LOWERED_TARGETS_PROGRAM); diff --git a/crates/by_transforms/tests/polyfill_runtime.rs b/crates/by_transforms/tests/polyfill_runtime.rs index c6a15d1248..87a9d50ef7 100644 --- a/crates/by_transforms/tests/polyfill_runtime.rs +++ b/crates/by_transforms/tests/polyfill_runtime.rs @@ -1,10 +1,10 @@ //! Runtime divergence tests for the **pep 695 polyfill** target. //! -//! Every other runtime test transpiles at `PY313`, where type parameters are -//! native and the polyfill never runs. That left the default output target -//! (`--min-version 3.10`) — the one a user gets without passing a flag — -//! untested at runtime, and three separate lowerings reached it emitting python -//! that raises on import while `by check` reported nothing: +//! The other runtime tests transpile for the interpreter they run on, which reaches +//! the polyfill only on an interpreter older than 3.12. These run it on purpose, at +//! targets below 3.12 — the default output target (`--min-version 3.10`), the one a +//! user gets without passing a flag, among them. Three separate lowerings reached it +//! emitting python that raised on import while `by check` reported nothing: //! //! - a callable arrow (`(T) -> R`) rendered `Callable[[T], R]`, keeping the //! pre-polyfill parameter names while the polyfill bound `_T` / `_R` @@ -20,6 +20,7 @@ use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; mod interpreters; use interpreters::Interpreter; @@ -311,6 +312,80 @@ assert Box().put(1) == 1 print("ok") "#; +/// a class body runs its annotations as the class is made, in a loop or a `with` as anywhere +/// else, so each has to read the type variable the polyfill declared +const COMPOUND_STATEMENT_RENAMES: &str = r#" +import contextlib +import typing + +class Box[T]: + for _i in range(1): + looped: list[T] + with contextlib.nullcontext() as _f: + opened: T + try: + tried: T + except Exception: + pass + +hints = typing.get_type_hints(Box) +declared = Box.__parameters__[0] +assert hints["looped"] == list[declared], hints +assert hints["opened"] === declared, hints +assert hints["tried"] === declared, hints +print("ok") +"#; + +/// type modifiers in a type parameter's bound, tuple bound, constraints and default. `literal +/// str` is `LiteralString`, the one modifier python can spell, so the runtime `TypeVar` carries +/// it; the others have no spelling and erase to the type they modify, and a lower bound has none +/// either, so a range is its upper end +const MODIFIED_BOUNDS: &str = r#" +import sys +import typing + +if sys.version_info >= (3, 11): + from typing import LiteralString +else: + from typing_extensions import LiteralString + +def bounded[T: literal str](t: T) -> T: + return t + +def in_a_tuple[T: (literal str, literal int)](t: T) -> T: + return t + +def constrained[T in (literal str, literal int)](t: T) -> T: + return t + +def defaulted[T: literal int = literal int](t: T) -> T: + return t + +def ranged[T: literal int..object](t: T) -> T: + return t + +class Box[T: literal str]: + def get(self, t: T) -> T: + return t + +type Alias[T: literal str] = list[T] + +def parameter(f: object) -> typing.Any: + return typing.get_type_hints(f)["t"] + +assert parameter(bounded).__bound__ === LiteralString, parameter(bounded).__bound__ +assert parameter(in_a_tuple).__bound__ == tuple[LiteralString, int], parameter(in_a_tuple).__bound__ +assert parameter(constrained).__constraints__ == (LiteralString, int), ( + parameter(constrained).__constraints__ +) +assert parameter(defaulted).__bound__ === int, parameter(defaulted).__bound__ +assert parameter(defaulted).__default__ === int, parameter(defaulted).__default__ +assert parameter(ranged).__bound__ === object, parameter(ranged).__bound__ +assert parameter(Box.get).__bound__ === LiteralString, parameter(Box.get).__bound__ +assert Alias.__type_params__[0].__bound__ === LiteralString, Alias.__type_params__ +print("ok") +"#; + /// a method whose bound reads a name of its class. the declaration reads it through the class, /// so it is at module scope with the others: `typing.get_type_hints` finds it, and so does a /// function nested in the method, whose annotations are evaluated when the method runs on a @@ -436,6 +511,14 @@ fn run_on_oldest(program: &str, target: PythonVersion, extensions: bool) { } } +/// run `program`, transpiled for the version of the interpreter it runs on, which is the pinned +/// one when there is one. see [`common::python`] +fn run_at_its_own_version(program: &str) { + if let Some(interpreter) = common::python(PythonVersion::PY39) { + run_at(&interpreter, program, interpreter.version); + } +} + /// the target the polyfill lowers for on `interpreter`: 3.10, the one a user gets without /// asking, or the interpreter's own version when that is older fn polyfill_target(interpreter: &Interpreter) -> PythonVersion { @@ -494,7 +577,7 @@ fn polyfilled_dependent_bounds_run() { #[test] fn some_parameters_run_at_every_version() { run_on_oldest(SOME_PARAMETERS, PythonVersion::PY310, false); - run_on_oldest(SOME_PARAMETERS, PythonVersion::PY313, false); + run_at_its_own_version(SOME_PARAMETERS); } /// an interpreter without `type.__or__` is the only one that can tell a union spelled for @@ -563,19 +646,19 @@ fn a_module_binding_the_type_variables_runs() { #[test] fn defaulted_parameter_specifications_and_variadics_run() { run_polyfilled_with_extensions(DEFAULTED_PACKS); - run_on_oldest(DEFAULTED_PACKS, PythonVersion::PY313, false); + run_at_its_own_version(DEFAULTED_PACKS); } #[test] fn a_hoisted_type_names_its_generics_type_variable() { run_on_oldest(HOISTED_TYPES, PythonVersion::PY39, false); - run_on_oldest(HOISTED_TYPES, PythonVersion::PY313, false); + run_at_its_own_version(HOISTED_TYPES); } #[test] fn a_module_binding_the_typing_names_runs() { run_on_oldest(OWN_TYPING_NAMES, PythonVersion::PY39, true); - run_on_oldest(OWN_TYPING_NAMES, PythonVersion::PY313, false); + run_at_its_own_version(OWN_TYPING_NAMES); } #[test] @@ -583,6 +666,19 @@ fn a_method_type_variable_resolves_in_type_hints() { run_on_oldest(METHOD_TYPE_VARIABLES, PythonVersion::PY39, false); } +#[test] +fn a_type_variable_read_in_a_compound_statement_resolves() { + run_on_oldest(COMPOUND_STATEMENT_RENAMES, PythonVersion::PY39, false); + run_at_its_own_version(COMPOUND_STATEMENT_RENAMES); +} + +/// below 3.12 the polyfill writes each `TypeVar` itself, and from 3.12 python does +#[test] +fn a_modified_bound_is_the_type_it_means_at_runtime() { + run_polyfilled_with_extensions(MODIFIED_BOUNDS); + run_at_its_own_version(MODIFIED_BOUNDS); +} + /// 3.10 and 3.11 evaluate annotations when the function is defined, which a nested function /// is each time the method runs #[test] diff --git a/crates/by_transforms/tests/reexport_conversion_runtime.rs b/crates/by_transforms/tests/reexport_conversion_runtime.rs index 15e558c6ad..fb66ba830a 100644 --- a/crates/by_transforms/tests/reexport_conversion_runtime.rs +++ b/crates/by_transforms/tests/reexport_conversion_runtime.rs @@ -14,13 +14,7 @@ //! packages are needed; if no `python3` is found the test skips rather than //! fails. -#![expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] - use std::fs; -use std::path::{Path, PathBuf}; use std::process::Command; use by_transforms::{Config, PythonVersion, transpile_typed}; @@ -29,6 +23,7 @@ use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ty_project::{ProjectMetadata, TestDb}; mod common; +mod interpreters; use common::python; const PACKAGE_INIT: &str = "from .geometry export Dp\n"; @@ -79,17 +74,20 @@ fn build_db() -> TestDb { db } -/// transpile every file of the project through the typed pipeline into a fresh +/// transpile every file of the project for `target` through the typed pipeline into a fresh /// directory under the cargo temp dir, keeping the package layout -fn build_case(case: &str) -> PathBuf { - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(dir.join("ui")).expect("create case dir"); +fn build_case(case: &str, target: PythonVersion) -> tempfile::TempDir { + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix(case) + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); + fs::create_dir_all(dir.path().join("ui")).expect("create case dir"); let db = build_db(); let config = Config { - min_version: PythonVersion::PY313, + min_version: target, ..Config::default() }; for (path, _) in FILES { @@ -97,20 +95,19 @@ fn build_case(case: &str) -> PathBuf { let transpiled = transpile_typed(&db, file, &config, None) .unwrap_or_else(|error| panic!("transpile of {path} should succeed: {error}")); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } dir } #[test] fn conversion_through_a_re_export_resolves_at_runtime() { - let Some(python) = python() else { - eprintln!("skipping re-export conversion runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; - let dir = build_case("reexport_conversion"); + let dir = build_case("reexport_conversion", python.version); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("main.py") .current_dir(&dir) .output() diff --git a/crates/by_transforms/tests/return_value_use_runtime.rs b/crates/by_transforms/tests/return_value_use_runtime.rs index 77aa45a96b..7707fc7b2c 100644 --- a/crates/by_transforms/tests/return_value_use_runtime.rs +++ b/crates/by_transforms/tests/return_value_use_runtime.rs @@ -17,10 +17,13 @@ //! asserts the transpiled text does not mention them, and a literal would //! satisfy that search without a decorator surviving at all. -use std::process::{Command, Stdio}; +use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; +mod interpreters; + /// basedpython whose module-level `assert`s run the marked declarations const PROGRAM: &str = r#" @ignorable_return_value @@ -53,37 +56,13 @@ assert Query().where("id = 1").rows() == [""] print("ok") "#; -/// the first interpreter that accepts `probe` -fn python_supporting(probe: &str) -> Option { - let mut candidates = Vec::new(); - if let Ok(p) = std::env::var("PYTHON") { - candidates.push(p); - } - candidates.extend(["python3.13", "python3"].map(String::from)); - - candidates.into_iter().find(|py| { - Command::new(py) - .args(["-c", probe]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) - }) -} - #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn marked_declarations_run_with_no_marker_left() { - let Some(python) = python_supporting("import sys; sys.exit(0)") else { - eprintln!("skipping return-value-marker runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); @@ -97,7 +76,7 @@ fn marked_declarations_run_with_no_marker_left() { "the marker survived the transpile:\n{transpiled}" ); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/statement_expression_runtime.rs b/crates/by_transforms/tests/statement_expression_runtime.rs index 71bb3f93fa..b5fd64d560 100644 --- a/crates/by_transforms/tests/statement_expression_runtime.rs +++ b/crates/by_transforms/tests/statement_expression_runtime.rs @@ -12,15 +12,17 @@ //! either fails to parse or parses as a statement of its own, and the program quietly //! computes something else //! -//! no third-party packages are needed, so any `python3` will do; if none is found the -//! test skips rather than fails +//! no third-party packages are needed, so the program is transpiled for whichever interpreter +//! runs it; if none is found the test skips, and says so, rather than fails use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// operands written over several lines, one per early-exit form const MULTI_LINE_OPERANDS: &str = r#" @@ -149,13 +151,13 @@ assert incremented(Node(None)) is None print("ok") "#; -fn run(python: &str, program: &str) { +fn run(python: &Interpreter, program: &str) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -171,28 +173,16 @@ fn run(python: &str, program: &str) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn an_operand_written_over_several_lines_keeps_its_grouping() { - let Some(python) = python() else { - eprintln!("skipping statement expression runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, MULTI_LINE_OPERANDS); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_propagated_operand_written_over_several_lines_keeps_its_grouping() { - let Some(python) = python() else { - eprintln!( - "skipping `a_propagated_operand_written_over_several_lines_keeps_its_grouping`: no `python3` interpreter found" - ); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, PROPAGATE_MULTI_LINE); diff --git a/crates/by_transforms/tests/static_resource_runtime.rs b/crates/by_transforms/tests/static_resource_runtime.rs index fb9c67754c..4df8006ff9 100644 --- a/crates/by_transforms/tests/static_resource_runtime.rs +++ b/crates/by_transforms/tests/static_resource_runtime.rs @@ -7,25 +7,22 @@ //! trailing comma, a helper class named before it was defined, a string whose //! escaping did not survive — fails here. -#![expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] - use std::fs; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::Command; -use by_transforms::{Config, transpile_typed}; +use by_transforms::{Config, PythonVersion, transpile_typed}; use ruff_db::files::system_path_to_file; use ruff_db::system::{DbWithWritableSystem, SystemPathBuf}; use ty_project::{ProjectMetadata, TestDb}; mod common; +mod interpreters; +use interpreters::Interpreter; /// transpile `/proj/main.by` with `files` written around it, into a fresh /// directory under the cargo temp dir. -fn build_case(case: &str, files: &[(&str, &str)]) -> PathBuf { +fn build_case(case: &str, files: &[(&str, &str)]) -> tempfile::TempDir { let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/proj"), @@ -39,17 +36,19 @@ fn build_case(case: &str, files: &[(&str, &str)]) -> PathBuf { let transpiled = transpile_typed(&db, file, &Config::default(), None).expect("transpile should succeed"); - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(case); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); - fs::write(dir.join("main.py"), transpiled).expect("write module"); + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix(case) + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); + fs::write(dir.path().join("main.py"), transpiled).expect("write module"); dir } /// run `main.py` in `dir`, asserting it exits cleanly and prints `ok` -fn run_main(python: &str, dir: &Path) { - let output = Command::new(python) +fn run_main(python: &Interpreter, dir: &Path) { + let output = Command::new(&python.command) .arg("main.py") .current_dir(dir) .output() @@ -157,8 +156,7 @@ print("ok") #[test] fn a_yaml_document_reads_back_as_it_was_written() { - let Some(python) = common::python() else { - eprintln!("skipping static resource runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY310) else { return; }; let dir = build_case( @@ -168,13 +166,12 @@ fn a_yaml_document_reads_back_as_it_was_written() { ("/proj/main.by", YAML_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] fn a_mapping_nested_in_a_sequence_is_defined_before_it_is_named() { - let Some(python) = common::python() else { - eprintln!("skipping static resource runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY310) else { return; }; let dir = build_case( @@ -184,13 +181,12 @@ fn a_mapping_nested_in_a_sequence_is_defined_before_it_is_named() { ("/proj/main.by", JSON_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] fn a_toml_document_reads_back_as_it_was_written() { - let Some(python) = common::python() else { - eprintln!("skipping static resource runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY310) else { return; }; let dir = build_case( @@ -200,13 +196,12 @@ fn a_toml_document_reads_back_as_it_was_written() { ("/proj/main.by", TOML_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } #[test] fn a_resource_imported_inside_a_function_runs_there() { - let Some(python) = common::python() else { - eprintln!("skipping static resource runtime test: no python interpreter found"); + let Some(python) = common::python(PythonVersion::PY310) else { return; }; let dir = build_case( @@ -216,5 +211,5 @@ fn a_resource_imported_inside_a_function_runs_there() { ("/proj/main.by", SCOPED_MAIN), ], ); - run_main(&python, &dir); + run_main(&python, dir.path()); } diff --git a/crates/by_transforms/tests/synthesized_types_runtime.rs b/crates/by_transforms/tests/synthesized_types_runtime.rs new file mode 100644 index 0000000000..7f14f7e22c --- /dev/null +++ b/crates/by_transforms/tests/synthesized_types_runtime.rs @@ -0,0 +1,125 @@ +//! Runtime test for the classes nested synthesized types lower to. +//! +//! A typed dict literal, an anonymous named tuple and an inline protocol each lower to a class, +//! and each can be the type of another's field. The outer class's body has to name the inner +//! one's class: written as the source spells it, a typed dict field reads back from +//! `typing.get_type_hints` as a plain dict, `{'k': int}`, which is no type at all — with `by +//! check` clean and nothing failing until something reads the hints. A field naming a type +//! parameter has to read the one the generic declares, from module scope, where the class is. + +mod common; +mod interpreters; + +use std::process::Command; + +use by_transforms::{Config, PythonVersion, transpile}; + +const PROGRAM: &str = r#" +import typing +import typing_extensions + +def f( + a: (p: {"k": int}, q: int), + b: protocol(p: {"k": int}), + c: {"k": (p: int, q: str)}, + d: {"k": protocol(n: int)}, + e: (p: protocol(n: int), q: int), + g: protocol(p: (x: int, y: str)), +) -> None: ... + +hints = typing.get_type_hints(f) + +def field(name: str, member: str) -> object: + return typing.get_type_hints(hints[name])[member] + +assert typing_extensions.is_typeddict(field("a", "p")), field("a", "p") +assert typing.get_type_hints(field("a", "p")) == {"k": int}, "a" +assert typing_extensions.is_typeddict(field("b", "p")), field("b", "p") +assert field("c", "k")._fields == ("p", "q"), field("c", "k") +assert typing.get_type_hints(field("c", "k")) == {"p": int, "q": str}, "c" +assert typing.get_type_hints(field("d", "k")) == {"n": int}, field("d", "k") +assert typing.get_type_hints(field("e", "p")) == {"n": int}, field("e", "p") +assert field("g", "p")._fields == ("x", "y"), field("g", "p") + + +class C[T]: + def get(self) -> {"a": T}: ... + + +assert typing.get_type_hints(typing.get_type_hints(C.get)["return"])["a"] === C.__parameters__[0] +print("ok") +"#; + +#[test] +fn a_synthesized_type_inside_another_is_its_class() { + let Some(python) = common::python_with( + PythonVersion::PY39, + "import typing_extensions", + "with `typing_extensions`", + ) else { + return; + }; + let config = Config { + min_version: python.version, + ..Config::default() + }; + let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); + let output = Command::new(&python.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} + +/// A named tuple class is declared ahead of everything the module binds, and below 3.14 python +/// evaluates its field annotations as it makes the class. A field naming a class or an import of +/// the module, declared above or below the type, is not bound there yet, and raised `NameError` +/// on import of a module that checks clean. Each reads back, once the module has run, as the +/// class it names. +const MODULE_NAMES: &str = r#" +import typing +from fractions import Fraction + +class Earlier: ... + +def f(p: (a: Earlier, b: Later, c: Fraction, d: int)) -> None: ... + +v = (e=Earlier(), g=Fraction(1), n=1) + +class Later: ... + +hints = typing.get_type_hints(typing.get_type_hints(f)["p"]) +assert hints == {"a": Earlier, "b": Later, "c": Fraction, "d": int}, hints +value_hints = typing.get_type_hints(type(v)) +assert value_hints == {"e": Earlier, "g": Fraction, "n": int}, value_hints +print("ok") +"#; + +#[test] +fn a_field_naming_a_binding_of_the_module_reads_it() { + let Some(python) = common::python(PythonVersion::PY39) else { + return; + }; + let config = Config { + min_version: python.version, + ..Config::default() + }; + let transpiled = transpile(MODULE_NAMES, &config).expect("transpile should succeed"); + let output = Command::new(&python.command) + .arg("-c") + .arg(&transpiled) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "failed on {python}:\n--- stderr ---\n{}\n--- transpiled ---\n{transpiled}", + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); +} diff --git a/crates/by_transforms/tests/trailing_lambda_runtime.rs b/crates/by_transforms/tests/trailing_lambda_runtime.rs index c9fa2ac3fa..2d8aa23e50 100644 --- a/crates/by_transforms/tests/trailing_lambda_runtime.rs +++ b/crates/by_transforms/tests/trailing_lambda_runtime.rs @@ -9,14 +9,16 @@ //! `nonlocal` `SyntaxError`, a wrong-scope write, or a broken return cell would //! fail here even though the type-level tests pass. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; /// basedpython whose module-level `assert`s exercise the trailing-lambda runtime @@ -106,23 +108,18 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn trailing_lambda_blocks_run_correctly() { - let Some(python) = python() else { - eprintln!("skipping trailing-lambda runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/tstring_expression_runtime.rs b/crates/by_transforms/tests/tstring_expression_runtime.rs index 79f2eeeb54..1602e67fba 100644 --- a/crates/by_transforms/tests/tstring_expression_runtime.rs +++ b/crates/by_transforms/tests/tstring_expression_runtime.rs @@ -5,32 +5,45 @@ //! that field, so the text a lowered field reports is a claim about the *source*, //! not about the lowering. Only an interpreter can settle it: the transform's unit //! tests can say what was emitted, but not what python makes of it — and the exact -//! span python reports (its whitespace, its grouping parentheses) is python's own -//! rule, which is easy to state wrongly from the outside — and which changed in -//! 3.14.8, when python stopped dropping the whitespace before the end of a field -//! (python/cpython#154719). Both programs write the text as 3.14.8 reports it, and -//! `reported` drops that whitespace again on an earlier 3.14. +//! span python reports (its leading whitespace, its grouping parentheses, its +//! comments) is python's own rule, which is easy to state wrongly from the outside. //! -//! Every assertion here is written twice over: the basedpython program asserts what -//! the field reports, and [`PYTHON_REFERENCE`] asserts that the same expectation -//! holds for the plain-python spelling of the same field. So a wrong expectation -//! fails rather than being blessed. +//! The rule changed within 3.14: from 3.14.8 a field's text runs up to the `=`, +//! conversion, format spec or `}` that ends it, trailing whitespace and all, where +//! 3.14.0 to 3.14.7 drop that whitespace (python/cpython#154719). A lowered field +//! reports what the interpreter it runs on would have reported for it, so the +//! expectations here are written for both, chosen by the version that runs them. +//! +//! Every assertion is written twice over: the basedpython program asserts what the +//! field reports, and [`PYTHON_REFERENCE`] asserts that the same expectation holds +//! for the plain-python spelling of the same field. So a wrong expectation — the +//! version split included — fails rather than being blessed. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; +mod common; mod interpreters; use interpreters::Interpreter; +/// the field text an interpreter reports, chosen by its version: `ends(kept, dropped)` is +/// `kept` where a field's trailing whitespace is kept and `dropped` where it is not. an +/// interpreter older than 3.14 has no t-strings, and the polyfill a string tag lowers to there +/// keeps it, as python does now +const ENDS: &str = r#" +import sys + +def ends(kept: str, dropped: str) -> str: + if (3, 14) <= sys.version_info < (3, 14, 8): + return dropped + return kept +"#; + /// basedpython whose t-string fields hold expressions that lower const PROGRAM: &str = r#" -import sys from string.templatelib import Template -def reported(text: str) -> str: - return text if sys.version_info >= (3, 14, 8) else text.rstrip() - def show(a: int?, b: int, d: dict[str, int]) -> None: def text(t) -> list: return [i.expression for i in t.interpolations] @@ -44,23 +57,29 @@ def show(a: int?, b: int, d: dict[str, int]) -> None: # `typeof`, which is folded in the syntax tree rather than emitted as an edit assert text(t"{typeof(b)}") == ["typeof(b)"], text(t"{typeof(b)}") # the author's own spacing and grouping parentheses, which python keeps - assert text(t"{ (a!) }") == [reported(" (a!) ")], text(t"{ (a!) }") + assert text(t"{ (a!) }") == [ends(" (a!) ", " (a!)")], text(t"{ (a!) }") # a conversion and a format spec end the reported text where the brace would - assert text(t"{ (a!) !r:>6}") == [reported(" (a!) ")], text(t"{ (a!) !r:>6}") + assert text(t"{ (a!) !r:>6}") == [ends(" (a!) ", " (a!)")], text(t"{ (a!) !r:>6}") # a field nothing lowered keeps the interpolation python built for it - assert text(t"{b} {a!} { b }") == ["b", "a!", reported(" b ")], text(t"{b} {a!} { b }") + assert text(t"{b} {a!} { b }") == ["b", "a!", ends(" b ", " b")], text(t"{b} {a!} { b }") + # a comment is not part of the text, though the line it ends is + assert text(t"""{a! # why +}""") == [ends("a! \n", "a!")], text(t"""{a! # why +}""") + assert text(t"""{(a! # why + ?? b)}""") == ["(a! \n ?? b)"], "a comment inside the parentheses" # the values and the rendering are the ones the lowering produced assert t"{a!}".values == (3,), t"{a!}".values assert t"{ (a!) !r:>6}".values == (3,), "spec values" assert t"{a!}".strings == ("", ""), t"{a!}".strings # a `=` field, which python itself splits into literal text and a field beside it assert list(t"{(a!) = }")[0] == "(a!) = ", list(t"{(a!) = }")[0] - assert text(t"{(a!) = }") == [reported("(a!) ")], text(t"{(a!) = }") + assert text(t"{(a!) = }") == [ends("(a!) ", "(a!)")], text(t"{(a!) = }") # a template nothing lowered is left exactly as written assert text(t"{b}") == ["b"], text(t"{b}") # a string tag is handed the template with the same texts assert text(same"{a!}") == ["a!"], text(same"{a!}") - assert text(same"{b} {a ?? b}") == ["b", "a ?? b"], text(same"{b} {a ?? b}") + assert text(same"{b} {a ?? b }") == ["b", ends("a ?? b ", "a ?? b")], text(same"{b} {a ?? b }") assert text(same"{b}") == ["b"], text(same"{b}") def same(t: Template) -> Template: @@ -73,29 +92,52 @@ print("ok") /// the same fields with the basedpython operators taken out, so the expectations above /// are checked against python's own reporting rather than against themselves const PYTHON_REFERENCE: &str = r#" -import sys - -def reported(text): - return text if sys.version_info >= (3, 14, 8) else text.rstrip() - def show(a, b, d): def text(t): return [i.expression for i in t.interpolations] assert text(t"{a}") == ["a"], text(t"{a}") - assert text(t"{ (a) }") == [reported(" (a) ")], text(t"{ (a) }") - assert text(t"{ (a) !r:>6}") == [reported(" (a) ")], text(t"{ (a) !r:>6}") - assert text(t"{b} {a} { b }") == ["b", "a", reported(" b ")], text(t"{b} {a} { b }") + assert text(t"{ (a) }") == [ends(" (a) ", " (a)")], text(t"{ (a) }") + assert text(t"{ (a) !r:>6}") == [ends(" (a) ", " (a)")], text(t"{ (a) !r:>6}") + assert text(t"{b} {a} { b }") == ["b", "a", ends(" b ", " b")], text(t"{b} {a} { b }") + assert text(t"""{a # why +}""") == [ends("a \n", "a")], text(t"""{a # why +}""") + assert text(t"""{(a # why + or b)}""") == ["(a \n or b)"], "a comment inside the parentheses" assert t"{a}".values == (3,), t"{a}".values assert t"{a}".strings == ("", ""), t"{a}".strings assert list(t"{(a) = }")[0] == "(a) = ", list(t"{(a) = }")[0] - assert text(t"{(a) = }") == [reported("(a) ")], text(t"{(a) = }") + assert text(t"{(a) = }") == [ends("(a) ", "(a)")], text(t"{(a) = }") assert text(t"{b}") == ["b"], text(t"{b}") + assert text(t"{b} {a or b }") == ["b", ends("a or b ", "a or b")], text(t"{b} {a or b }") show(3, 7, {"k": 5}) print("ok") "#; +/// string tags, which are t-strings from 3.14 and a polyfilled template below it, so they run on +/// every version +const TAGS: &str = r#" +def text(t: object) -> list[str]: + return [i.expression for i in t.interpolations] + +def same(t: object) -> object: + return t + +def show(a: int?, b: int) -> None: + assert text(same"{a!}") == ["a!"], text(same"{a!}") + assert text(same"{ (a!) }") == [ends(" (a!) ", " (a!)")], text(same"{ (a!) }") + assert text(same"{ (a!) !r:>6}") == [ends(" (a!) ", " (a!)")], text(same"{ (a!) !r:>6}") + assert text(same"{b} {a ?? b} { b }") == ["b", "a ?? b", ends(" b ", " b")], ( + text(same"{b} {a ?? b} { b }") + ) + assert same"{a!}".values == (3,), same"{a!}".values + +show(3, 7) +print("ok") +"#; + fn run(python: &Interpreter, source: &str, what: &str) { let output = Command::new(&python.command) .arg("-c") @@ -110,20 +152,48 @@ fn run(python: &Interpreter, source: &str, what: &str) { assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } +/// `program` transpiled for `target`, with [`ENDS`] ahead of it +fn transpiled(program: &str, target: PythonVersion) -> String { + let config = Config { + min_version: target, + ..Config::default() + }; + transpile(&format!("{ENDS}{program}"), &config).expect("transpile should succeed") +} + #[test] fn a_field_reports_the_source_the_author_wrote() { // an interpreter that has t-strings - let Some(python) = interpreters::oldest(PythonVersion::PY314, "", "") else { + let Some(python) = common::python(PythonVersion::PY314) else { return; }; - run(&python, PYTHON_REFERENCE, "the plain-python reference"); + run( + &python, + &format!("{ENDS}{PYTHON_REFERENCE}"), + "the plain-python reference", + ); + run( + &python, + &transpiled(PROGRAM, python.version), + "the transpiled program", + ); +} - let config = Config { - min_version: PythonVersion::PY314, - ..Config::default() +/// a string tag below 3.14 builds its template from the polyfill, whose fields report the same +/// text — and on an interpreter that has t-strings, the text that interpreter would report +#[test] +fn a_string_tag_field_reports_the_source_the_author_wrote() { + let Some(python) = common::python(PythonVersion::PY39) else { + return; }; - let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); - run(&python, &transpiled, "the transpiled program"); - interpreters::ran(&python, PythonVersion::PY314); + for target in [python.version, PythonVersion::PY313] { + if target <= python.version { + run( + &python, + &transpiled(TAGS, target), + &format!("the tags transpiled for {target}"), + ); + } + } } diff --git a/crates/by_transforms/tests/type_checking_import_runtime.rs b/crates/by_transforms/tests/type_checking_import_runtime.rs index 3b0cd2d0b3..19a4d94058 100644 --- a/crates/by_transforms/tests/type_checking_import_runtime.rs +++ b/crates/by_transforms/tests/type_checking_import_runtime.rs @@ -8,9 +8,9 @@ //! `NameError` with `by check` clean, unless it looks the name up as it runs. mod common; +mod interpreters; use std::fs; -use std::path::Path; use std::process::Command; use by_transforms::{Config, PythonVersion, SoundnessPositions, transpile, transpile_typed}; @@ -267,13 +267,8 @@ fn a_class_declared_further_down_is_checked_once_it_is_bound() { /// a module importing a name another module binds one way for the checker and another way as /// it runs reads what the program binds, which need not be the class the checker resolved #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_name_imported_from_a_type_checking_block_elsewhere_is_never_evaluated() { - let Some(python) = common::python() else { - eprintln!("skipping type checking import runtime test: no interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let files = [ @@ -309,10 +304,12 @@ print(\"ok\") ", ), ]; - let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("type_checking_import_elsewhere"); - // a stale directory from an earlier run would mask a transpile failure - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).expect("create case dir"); + // a directory of its own, so a run beside another, from the same checkout, neither reads + // nor deletes the other's modules + let dir = tempfile::Builder::new() + .prefix("type_checking_import_elsewhere") + .tempdir_in(env!("CARGO_TARGET_TMPDIR")) + .expect("create case dir"); let mut db = TestDb::new(ProjectMetadata::new( ruff_python_ast::name::Name::new_static(""), SystemPathBuf::from("/"), @@ -322,7 +319,7 @@ print(\"ok\") db.write_file(path, source).expect("write file failed"); } let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, soundness: SoundnessPositions::all(), ..Config::default() }; @@ -331,9 +328,9 @@ print(\"ok\") let transpiled = transpile_typed(&db, file, &config, None).expect("transpile should succeed"); let relative = path.trim_start_matches('/').replace(".by", ".py"); - fs::write(dir.join(relative), transpiled).expect("write module"); + fs::write(dir.path().join(relative), transpiled).expect("write module"); } - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("main.py") .current_dir(&dir) .output() @@ -342,27 +339,22 @@ print(\"ok\") output.status.success(), "transpiled program failed on {python}:\n--- stderr ---\n{}\n--- main.py ---\n{}", String::from_utf8_lossy(&output.stderr), - fs::read_to_string(dir.join("main.py")).unwrap_or_default(), + fs::read_to_string(dir.path().join("main.py")).unwrap_or_default(), ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "ok"); } -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn run(program: &str) { - let Some(python) = common::python() else { - eprintln!("skipping type checking import runtime test: no interpreter found"); + let Some(python) = common::python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, soundness: SoundnessPositions::all(), ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/unique_loop_bindings_runtime.rs b/crates/by_transforms/tests/unique_loop_bindings_runtime.rs index d7bf651889..aeeb482cba 100644 --- a/crates/by_transforms/tests/unique_loop_bindings_runtime.rs +++ b/crates/by_transforms/tests/unique_loop_bindings_runtime.rs @@ -8,15 +8,18 @@ //! loop's own cell, a decorated / annotated / recursive `def` surviving the //! closure rebuild, and a generator or coroutine staying one. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; mod common; +mod interpreters; use common::python; +use interpreters::Interpreter; /// the capture itself, across every form that binds an iteration const CAPTURES: &str = r#" @@ -55,20 +58,6 @@ for left, right in [(1, 2), (3, 4)]: pairs.append(lambda: (left, right)) assert [fn() for fn in pairs] == [(1, 2), (3, 4)], "each unpacked target binds" -# a pattern in the target position binds its captures, not the binder the -# destructuring lowering matches against -class Pair: - __match_args__ = ("left", "right") - - def __init__(self, left: int, right: int): - self.left = left - self.right = right - -patterned = [] -for Pair(left, right) in [Pair(1, 2), Pair(3, 4)]: - patterned.append(lambda: (left, right)) -assert [fn() for fn in patterned] == [(1, 2), (3, 4)], "each pattern capture binds" - # a `def`'s parameter default is re-evaluated per call from inside the body, so # a loop binding it names is captured there like any other body read def collect_defaults() -> list: @@ -130,6 +119,24 @@ assert after() == 3, "the else clause sees the final value" print("ok") "#; +/// a pattern in the target position binds its captures, not the binder the destructuring +/// lowering matches against. a destructuring needs python 3.10 +const PATTERN_CAPTURES: &str = r#" +class Pair: + __match_args__ = ("left", "right") + + def __init__(self, left: int, right: int): + self.left = left + self.right = right + +patterned = [] +for Pair(left, right) in [Pair(1, 2), Pair(3, 4)]: + patterned.append(lambda: (left, right)) +assert [fn() for fn in patterned] == [(1, 2), (3, 4)], "each pattern capture binds" + +print("ok") +"#; + /// what the closure rebuild must not damage const REBUILD: &str = r#" # a `nonlocal` write goes through the loop's own cell, so that name is never @@ -175,9 +182,15 @@ assert set(typing.get_type_hints(decorated[0])) == {"a", "b", "return"}, "annota # the signature is what every framework reads to decide what a function wants — # a query parameter, a fixture, a validated argument — so the rebuild must not # perturb it. `getfullargspec` is checked too: it is the other reader in the wild +# +# the signature is compared with that of a twin no loop rebuilds, which reads its annotations +# back as the target writes them: as strings below 3.10, which the module gets +# `from __future__ import annotations` for import inspect +def unrebuilt(a: int, b: str = "b") -> str: + return b assert list(inspect.signature(decorated[0]).parameters) == ["a", "b"], "the signature is exact" -assert str(inspect.signature(decorated[0])) == "(a: int, b: str = 'b') -> str", ( +assert str(inspect.signature(decorated[0])) == str(inspect.signature(unrebuilt)), ( "annotations and defaults read back unchanged" ) assert inspect.getfullargspec(decorated[0]).args == ["a", "b"], "argspec is exact" @@ -323,26 +336,14 @@ assert collect() == ["1:True", "2:True"], "the loop binding and the type argumen print("ok") "#; -/// reification keeps native pep 695 syntax, so it needs a 3.12+ interpreter -fn python_with_native_type_params(python: &str) -> bool { - Command::new(python) - .args([ - "-c", - "import sys; raise SystemExit(0 if sys.version_info >= (3, 12) else 1)", - ]) - .status() - .map(|status| status.success()) - .unwrap_or(false) -} - -fn run(python: &str, program: &str, unique_loop_bindings: bool) { +fn run(python: &Interpreter, program: &str, unique_loop_bindings: bool) { let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, unique_loop_bindings, ..Config::default() }; let transpiled = transpile(program, &config).expect("transpile should succeed"); - let output = Command::new(python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() @@ -358,56 +359,56 @@ fn run(python: &str, program: &str, unique_loop_bindings: bool) { } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn loop_bindings_are_per_iteration() { - let Some(python) = python() else { - eprintln!("skipping loop-binding runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, CAPTURES, true); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] +fn pattern_captures_are_per_iteration() { + let Some(python) = python(PythonVersion::PY310) else { + return; + }; + run(&python, PATTERN_CAPTURES, true); +} + +#[test] fn the_closure_rebuild_preserves_the_function() { - let Some(python) = python() else { - eprintln!("skipping loop-binding runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, REBUILD, true); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn what_stays_pythons_stays_pythons() { - let Some(python) = python() else { - eprintln!("skipping loop-binding runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; run(&python, UNCHANGED, true); } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_reified_generic_in_a_loop_keeps_both_bindings() { - let Some(python) = python() else { - eprintln!("skipping loop-binding runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; - if !python_with_native_type_params(&python) { - eprintln!("skipping reified-generic runtime test: {python} is older than 3.12"); + // the type argument's cell is a pep 695 closure cell, which needs native type-parameter + // syntax + if python.version < PythonVersion::PY312 { + let config = Config { + min_version: python.version, + unique_loop_bindings: true, + ..Config::default() + }; + let refusal = transpile(REIFIED, &config) + .expect_err("a reified generic is refused below python 3.12"); + assert!( + refusal.contains("requires python 3.12 or newer"), + "{refusal}" + ); return; } run(&python, REIFIED, true); @@ -416,17 +417,12 @@ fn a_reified_generic_in_a_loop_keeps_both_bindings() { /// with the option off the output is python's, sharing one cell per loop — /// the same program that asserts per-iteration values must fail #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn disabling_the_option_restores_pythons_sharing() { - let Some(python) = python() else { - eprintln!("skipping loop-binding runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, unique_loop_bindings: false, ..Config::default() }; @@ -435,7 +431,7 @@ fn disabling_the_option_restores_pythons_sharing() { !transpiled.contains("_by_loop_bind"), "the disabled pass emitted its runtime:\n{transpiled}" ); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(&transpiled) .output() diff --git a/crates/by_transforms/tests/visibility_runtime.rs b/crates/by_transforms/tests/visibility_runtime.rs index cafe837a3f..3e53237647 100644 --- a/crates/by_transforms/tests/visibility_runtime.rs +++ b/crates/by_transforms/tests/visibility_runtime.rs @@ -10,16 +10,17 @@ //! The transform unit tests pin the text of the emitted `__all__`. This test //! closes the loop by importing the module the way `__all__` is for. //! -//! No third-party packages are needed, so any `python3` will do; if none is -//! found the test skips rather than fails. +//! No third-party packages are needed, so the program is transpiled for whichever +//! interpreter runs it; if none is found the test skips, and says so, rather than +//! fails. use std::process::Command; use by_transforms::{Config, PythonVersion, transpile}; -mod common; use common::python; +mod common; mod interpreters; /// Every visibility shape that reaches `__all__`, in both keyword orders. @@ -56,18 +57,13 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn every_exported_name_exists_at_runtime() { - let Some(python) = python() else { - eprintln!("skipping visibility runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(PROGRAM, &config).expect("transpile should succeed"); @@ -76,7 +72,7 @@ fn every_exported_name_exists_at_runtime() { let dir = tempfile::tempdir().expect("failed to create a temp dir"); std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(IMPORTER) .current_dir(dir.path()) @@ -138,18 +134,13 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_renamed_member_is_the_same_attribute_from_every_place_that_may_reach_it() { - let Some(python) = python() else { - eprintln!("skipping member visibility runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(MEMBERS, &config).expect("transpile should succeed"); @@ -157,7 +148,7 @@ fn a_renamed_member_is_the_same_attribute_from_every_place_that_may_reach_it() { let dir = tempfile::tempdir().expect("failed to create a temp dir"); std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(MEMBER_IMPORTER) .current_dir(dir.path()) @@ -220,18 +211,13 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_renamed_variable_is_the_same_binding_wherever_it_is_the_symbol() { - let Some(python) = python() else { - eprintln!("skipping variable visibility runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(VARIABLES, &config).expect("transpile should succeed"); @@ -239,7 +225,7 @@ fn a_renamed_variable_is_the_same_binding_wherever_it_is_the_symbol() { let dir = tempfile::tempdir().expect("failed to create a temp dir"); std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(VARIABLE_IMPORTER) .current_dir(dir.path()) @@ -348,18 +334,13 @@ print("ok") "#; #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test must say why it skipped, or it reads as a pass" -)] fn a_renamed_name_is_read_back_wherever_python_looks_it_up() { - let Some(python) = python() else { - eprintln!("skipping visibility shapes runtime test: no `python3` interpreter found"); + let Some(python) = python(PythonVersion::PY39) else { return; }; let config = Config { - min_version: PythonVersion::PY313, + min_version: python.version, ..Config::default() }; let transpiled = transpile(SHAPES, &config).expect("transpile should succeed"); @@ -367,7 +348,7 @@ fn a_renamed_name_is_read_back_wherever_python_looks_it_up() { let dir = tempfile::tempdir().expect("failed to create a temp dir"); std::fs::write(dir.path().join("emitted.py"), &transpiled).expect("failed to write the module"); - let output = Command::new(&python) + let output = Command::new(&python.command) .arg("-c") .arg(SHAPE_IMPORTER) .current_dir(dir.path()) diff --git a/crates/ruff_python_formatter/resources/test/fixtures/ruff/type_modifiers.by b/crates/ruff_python_formatter/resources/test/fixtures/ruff/type_modifiers.by index e4b86d8ac2..95534c64db 100644 --- a/crates/ruff_python_formatter/resources/test/fixtures/ruff/type_modifiers.by +++ b/crates/ruff_python_formatter/resources/test/fixtures/ruff/type_modifiers.by @@ -15,6 +15,9 @@ class I: j: final int = 1 +n: {"o": literal str, "p": final int, **: final str} + + # `literal` and `final` are only modifiers when a name follows them k: literal m: final[int] diff --git a/crates/ruff_python_formatter/tests/snapshots/format@type_modifiers.by.snap b/crates/ruff_python_formatter/tests/snapshots/format@type_modifiers.by.snap index ecccebfe30..77b5bcfac6 100644 --- a/crates/ruff_python_formatter/tests/snapshots/format@type_modifiers.by.snap +++ b/crates/ruff_python_formatter/tests/snapshots/format@type_modifiers.by.snap @@ -21,6 +21,9 @@ class I: j: final int = 1 +n: {"o": literal str, "p": final int, **: final str} + + # `literal` and `final` are only modifiers when a name follows them k: literal m: final[int] @@ -45,6 +48,9 @@ class I: j: final int = 1 +n: {"o": literal str, "p": final int, **: final str} + + # `literal` and `final` are only modifiers when a name follows them k: literal m: final[int] diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index f8342fb865..4082c4910a 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -1002,7 +1002,7 @@ pub enum UnsupportedSyntaxErrorKind { /// ## Examples /// /// ```by - /// let Point(x, y) := origin # error before 3.8, where no `match` can be written + /// let Point(x, y) := origin # error before 3.10 /// ``` Destructuring, } @@ -1209,28 +1209,36 @@ impl UnsupportedSyntaxErrorKind { | UnsupportedSyntaxErrorKind::StarAnnotation | UnsupportedSyntaxErrorKind::StarExpressionInIndex | UnsupportedSyntaxErrorKind::LazyImportStatement => true, + // the types of a bracketless `except` are written as the parenthesized tuple they are + UnsupportedSyntaxErrorKind::UnparenthesizedExceptionTypes => true, // the `match` lowering writes each pattern as a test built of assignment - // expressions, in which a unary `+` is an ordinary operator. a destructuring is - // written as a `match`, so it goes wherever one goes + // expressions, in which a unary `+` is an ordinary operator UnsupportedSyntaxErrorKind::Match - | UnsupportedSyntaxErrorKind::UnaryPlusMatchPattern - | UnsupportedSyntaxErrorKind::Destructuring => { + | UnsupportedSyntaxErrorKind::UnaryPlusMatchPattern => { UnsupportedSyntaxErrorKind::Match.is_unsupported(target_version) && UnsupportedSyntaxErrorKind::Walrus.is_supported(target_version) } - UnsupportedSyntaxErrorKind::Walrus + // a destructuring is written as a `match`, but is not lowered where a `match` is. + // the `match` lowering leaves the temporaries it binds behind, as attributes of a + // class whose body it is written in, and unbinding them takes a statement that has + // no source line of its own to go on. the lowering of a `match` leaves nothing of + // it in the output to re-parse, so the transpiler refuses it on the source + UnsupportedSyntaxErrorKind::Destructuring + | UnsupportedSyntaxErrorKind::Walrus | UnsupportedSyntaxErrorKind::ExceptStar | UnsupportedSyntaxErrorKind::UnparenthesizedNamedExpr(_) | UnsupportedSyntaxErrorKind::ParenthesizedKeywordArgumentName | UnsupportedSyntaxErrorKind::StarTuple(_) | UnsupportedSyntaxErrorKind::RelaxedDecorator(_) | UnsupportedSyntaxErrorKind::PositionalOnlyParameter - | UnsupportedSyntaxErrorKind::Pep701FString(_) | UnsupportedSyntaxErrorKind::ParenthesizedContextManager | UnsupportedSyntaxErrorKind::UnpackingInComprehension(_) | UnsupportedSyntaxErrorKind::UnparenthesizedUnpackInFor - | UnsupportedSyntaxErrorKind::UnparenthesizedExceptionTypes | UnsupportedSyntaxErrorKind::TemplateStrings => false, + // an f-string is kept, but one that is a template literal type is written as the `str` + // or `Literal[...]` it means, so none of its syntax reaches the python. what is a type + // expression only the checker knows, so it is the checker that lets those through + UnsupportedSyntaxErrorKind::Pep701FString(_) => false, } } diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 363393511d..be323467d1 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1311,7 +1311,7 @@ impl<'src> Parser<'src> { return self.parse_parenthesized_expression(context); } TokenKind::Lsqb => self.parse_list_like_expression(context), - TokenKind::Lbrace => self.parse_set_or_dict_like_expression(), + TokenKind::Lbrace => self.parse_set_or_dict_like_expression(context), kind => { if kind.is_keyword() { @@ -1443,7 +1443,22 @@ impl<'src> Parser<'src> { self.error_if_not_basedpython( "custom string tags are not valid in .py files".to_string(), ); + let held_before = self.unsupported_syntax_errors.len(); let template = self.parse_strings(); + // the tag's lowering writes the template for whatever python the file + // targets, so the template is not held to the python that has t-strings. a + // t-string written inside one of its fields is not the tag's, and still is + let parts: Vec = match &template { + Expr::TString(tstring) => tstring.value.iter().map(Ranged::range).collect(), + _ => Vec::new(), + }; + let mut index = 0; + self.unsupported_syntax_errors.retain(|error| { + index += 1; + index <= held_before + || error.kind != UnsupportedSyntaxErrorKind::TemplateStrings + || !parts.contains(&error.range) + }); let arguments = ast::Arguments { range: template.range(), node_index: AtomicNodeIndex::NONE, @@ -3400,7 +3415,7 @@ impl<'src> Parser<'src> { /// - /// - /// - - fn parse_set_or_dict_like_expression(&mut self) -> Expr { + fn parse_set_or_dict_like_expression(&mut self, outer: ExpressionContext) -> Expr { // test_ok pep_798_unpacking_comprehensions_py315 // # parse_options: {"target-version": "3.15"} // [*x for x in y] @@ -3450,6 +3465,11 @@ impl<'src> Parser<'src> { let after_brace = self.node_start(); + // basedpython: a dict in a type expression is a typed dict literal, whose values (and + // `**: T` extra-items type) are types, so they may carry a type modifier as any other + // type position does + let value_context = ExpressionContext::default().inheriting_type_expression(outer); + if self.at(TokenKind::DoubleStar) { self.bump(TokenKind::DoubleStar); // basedpython `**: T` extra-items marker in a typed-dict literal. @@ -3457,7 +3477,9 @@ impl<'src> Parser<'src> { // can distinguish it from regular `**other_dict` unpacking if self.at(TokenKind::Colon) { self.bump(TokenKind::Colon); - let inner = self.parse_conditional_expression_or_higher().expr; + let inner = self + .parse_conditional_expression_or_higher_impl(value_context) + .expr; let inner_range = inner.range(); let outer_range = self.node_range(after_brace); let inner_starred = Expr::Starred(ast::ExprStarred { @@ -3472,7 +3494,12 @@ impl<'src> Parser<'src> { range: outer_range, node_index: AtomicNodeIndex::NONE, }); - return Expr::Dict(self.parse_dictionary_expression(None, outer_starred, start)); + return Expr::Dict(self.parse_dictionary_expression( + None, + outer_starred, + start, + value_context, + )); } // Handle dictionary unpacking. Here, the grammar is `'**' bitwise_or` // which requires limiting the expression. @@ -3510,10 +3537,16 @@ impl<'src> Parser<'src> { Some(value.expr), dict_value.expr, start, + value_context, )); } - return Expr::Dict(self.parse_dictionary_expression(None, value.expr, start)); + return Expr::Dict(self.parse_dictionary_expression( + None, + value.expr, + start, + value_context, + )); } // For dictionary expressions, the key uses the `expression` rule while for @@ -3584,7 +3617,7 @@ impl<'src> Parser<'src> { ); value } else { - self.parse_conditional_expression_or_higher() + self.parse_conditional_expression_or_higher_impl(value_context) }; if matches!(self.current_token_kind(), TokenKind::Async | TokenKind::For) { @@ -3598,6 +3631,7 @@ impl<'src> Parser<'src> { Some(key_or_element.expr), value.expr, start, + value_context, )) } } @@ -4958,11 +4992,15 @@ impl<'src> Parser<'src> { /// Parses a dictionary expression. /// /// See: + /// + /// `value_context` is the context each value is parsed in, which carries a type expression's + /// position onto a typed dict literal's values fn parse_dictionary_expression( &mut self, key: Option, value: Expr, start: TextSize, + value_context: ExpressionContext, ) -> ast::ExprDict { if !self.at_sequence_end() { self.expect(TokenKind::Comma); @@ -4979,7 +5017,9 @@ impl<'src> Parser<'src> { // dictionary unpacking if parser.at(TokenKind::Colon) { parser.bump(TokenKind::Colon); - let inner = parser.parse_conditional_expression_or_higher().expr; + let inner = parser + .parse_conditional_expression_or_higher_impl(value_context) + .expr; let inner_range = inner.range(); let outer_range = parser.node_range(doublestar_start); let inner_starred = Expr::Starred(ast::ExprStarred { @@ -5012,7 +5052,9 @@ impl<'src> Parser<'src> { items.push(ast::DictItem { key: Some(key), - value: parser.parse_conditional_expression_or_higher().expr, + value: parser + .parse_conditional_expression_or_higher_impl(value_context) + .expr, }); } }); diff --git a/crates/ruff_python_parser/src/parser/tests.rs b/crates/ruff_python_parser/src/parser/tests.rs index bb8d01a41e..89b6d0b0e5 100644 --- a/crates/ruff_python_parser/src/parser/tests.rs +++ b/crates/ruff_python_parser/src/parser/tests.rs @@ -3248,6 +3248,9 @@ fn basedpython_type_modifier_in_nested_type_positions() { "def f(**kwargs: final int): ...\n", "a: list[literal str]\n", "type X = literal str\n", + "a: {\"k\": literal str}\n", + "a: {\"k\": int, **: final int}\n", + "a: {**: literal str}\n", ] { let parsed = parse_basedpython_module(source); let mut found = false; diff --git a/crates/ty/docs/rules.md b/crates/ty/docs/rules.md index f0b9da6c16..bc85b6f7fc 100644 --- a/crates/ty/docs/rules.md +++ b/crates/ty/docs/rules.md @@ -4325,7 +4325,8 @@ Added in 0.0.83 str: + # error: [invalid-repeated-underscore] + return _ +``` + +A read that only a binding the body makes reaches reads that binding, and `del _` reads no value: + +```by +def rebound(_: int, _: int) -> int: + _ = 5 + return _ +``` + ## `invalid-return-type` diff --git a/crates/ty/src/by_commands.rs b/crates/ty/src/by_commands.rs index 6198450da1..c6740e42dc 100644 --- a/crates/ty/src/by_commands.rs +++ b/crates/ty/src/by_commands.rs @@ -1646,7 +1646,8 @@ pub(crate) fn cmd_transpile( } } } else { - by_transforms::transpile(&source, &config).map_err(|e| anyhow::anyhow!("{e}"))? + by_transforms::transpile_analysed_at(&source, &config, config.min_version) + .map_err(|e| anyhow::anyhow!("{e}"))? }; print!("{output}"); diff --git a/crates/ty/tests/by_e2e.rs b/crates/ty/tests/by_e2e.rs index 923e65ba2b..16bc9d3f32 100644 --- a/crates/ty/tests/by_e2e.rs +++ b/crates/ty/tests/by_e2e.rs @@ -1,3 +1,4 @@ +use by_ir::mangle::mangle; use std::fmt::Write as _; use std::fs; use std::io::Write; @@ -552,7 +553,7 @@ fn compile_emits_only_the_files_it_was_given_and_still_resolves_the_others() { // attribute read at all let emitted = std::fs::read_to_string(out.join("wanted.c")).expect("the C is readable"); assert!( - emitted.contains("by_wanted_go"), + emitted.contains(&format!("by_{}_{}", mangle("wanted"), mangle("go"))), "`go` lowered natively, so `Point` resolved out of the uncompiled sibling" ); } @@ -627,8 +628,14 @@ fn compile_writes_each_package_member_at_its_own_place_in_the_output_tree() { // the two `dup` members are distinct modules, not one file written twice let first = fs::read_to_string(out.join("pkg/dup.c")).unwrap(); let second = fs::read_to_string(out.join("pkg/sub/dup.c")).unwrap(); - assert!(first.contains("by_pkg_dup_tag"), "{first}"); - assert!(second.contains("by_pkg_sub_dup_tag"), "{second}"); + assert!( + first.contains(&format!("by_{}_{}", mangle("pkg.dup"), mangle("tag"))), + "{first}" + ); + assert!( + second.contains(&format!("by_{}_{}", mangle("pkg.sub.dup"), mangle("tag"))), + "{second}" + ); } #[test] @@ -857,7 +864,7 @@ fn compile_reads_whether_to_bind_functions_early_from_the_project() { return; }; assert!( - asked.contains("By_ResolveFunction(&by_fs_add"), + asked.contains(&format!("By_ResolveFunction(&by_fs_{}", mangle("add"))), "the call asks" ); let Some(early) = emitted( @@ -4232,25 +4239,55 @@ fn transpile_directory_round_trips_through_build() { ); } -/// An interpreter new enough to run a reified class: the specializer reads the -/// class's own `__type_params__`, which needs the native PEP 695 header. -fn python_at_least_312() -> Option { - // `$PYTHON` is what pins the interpreter for every python-executing test, - // so an explicit one is the answer whether or not it is new enough — a run - // that pinned an old interpreter should say so rather than quietly pick - // another - if let Ok(pinned) = std::env::var("PYTHON") { - return Some(pinned); +/// an interpreter of python `3.minor` or later, for a test whose program needs it: `needs` says +/// why. none is a skip, said so +/// +/// `$PYTHON` is what pins the interpreter for every python-executing test, so an explicit one is +/// the answer whenever it is new enough, and a skip when it is not: a run that pinned an old +/// interpreter is asking about that interpreter, and quietly picking another would answer a +/// question nobody asked +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn python_at_least(minor: u8, needs: &str) -> Option { + let new_enough = |python: &str| { + Command::new(python) + .args([ + "-c", + &format!("import sys; sys.exit(sys.version_info < (3, {minor}))"), + ]) + .output() + .is_ok_and(|out| out.status.success()) + }; + let found = match std::env::var("PYTHON") { + Ok(pinned) => Some(pinned).filter(|pinned| new_enough(pinned)), + Err(_) => ["python3.13", "python3.12", "python3"] + .into_iter() + .find(|python| new_enough(python)) + .map(str::to_owned), + }; + if found.is_none() { + eprintln!("skipping: no python 3.{minor}+ interpreter, which {needs}"); } - ["python3.13", "python3.12", "python3"] - .into_iter() - .find(|python| { - Command::new(python) - .args(["-c", "import sys; sys.exit(sys.version_info < (3, 12))"]) - .output() - .is_ok_and(|out| out.status.success()) - }) - .map(str::to_owned) + found +} + +/// an interpreter new enough to run a reified generic: reification keeps the native pep 695 +/// type-parameter list, which python has from 3.12, and the transpiler refuses one for an older +/// target rather than write a program that cannot run there (see +/// `reified_generic_below_312_is_refused_rather_than_run`) +fn python_at_least_312() -> Option { + python_at_least(12, "a reified generic needs") +} + +/// source read from stdin is analysed at the version it is transpiled for, as a project's +/// modules are. `frozendict` is a builtin from 3.15, past ty's newest python, so the +/// conversion its annotation makes is only written where the analysis knows it +#[test] +fn stdin_is_analysed_at_the_version_it_is_transpiled_for() { + let out = run_transpile("a: frozendict[str, str] = {}\n", &["--min-version", "3.15"]); + assert!(out.contains("= frozendict()"), "{out}"); } /// transpile with `--min-version 3.13` — reified generics require native PEP @@ -4312,15 +4349,10 @@ a.f() } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn reified_class_reads_its_type_argument_at_runtime() { // the whole contract end to end: an instance built from `A[int]` answers // `int` when its method reads the class's type parameter let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -4357,6 +4389,50 @@ a.f() ); } +/// below 3.12 python has no native type-parameter list for a reified generic to keep, so +/// `by run` refuses the program rather than run one that cannot read its type argument. a 3.11 +/// interpreter makes 3.11 the target, as `--min-version 3.11` does here on any interpreter +#[test] +fn reified_generic_below_312_is_refused_rather_than_run() { + let Some(python) = python_at_least(11, "the program is built for") else { + return; + }; + + let dir = tempfile::tempdir().expect("tempdir"); + fs::write( + dir.path().join("main.by"), + "\ +class A[T]: + def f(self): + print(T) + +print(\"ran\") +A[int]().f() +", + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_by")) + .env(EnvVars::BY_NO_PROJECT_SERVER, "1") + .args(["run", "--min-version", "3.11", "main"]) + .env("PYTHON", &python) + .current_dir(dir.path()) + .output() + .expect("failed to spawn by"); + + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success(), "by run should refuse:\n{stderr}"); + assert!( + stderr.contains("reified generic class `A` requires python 3.12 or newer"), + "the refusal should name the class and the version it needs:\n{stderr}" + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "", + "nothing should run" + ); +} + #[test] fn declared_reified_generic_wraps_a_body_that_never_reads_it() { // the `reified` modifier drives the whole pipeline on its own: the keyword @@ -4875,12 +4951,41 @@ x(a) ); } +/// the interpreter `by run` picks outside an environment — `$PYTHON`, and `python3` when nothing +/// pins one — when a program writing `override` can run on it: the transpiler takes it from +/// `typing` from 3.12, and from `typing_extensions` below, which a bare interpreter does not +/// have. none is a skip, said so +#[expect( + clippy::print_stderr, + reason = "a skipped test must say why it skipped, or it reads as a pass" +)] +fn python_running_override() -> Option { + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let runs = Command::new(&python) + .args([ + "-c", + "import sys\nif sys.version_info < (3, 12): import typing_extensions", + ]) + .output() + .is_ok_and(|out| out.status.success()); + if !runs { + eprintln!( + "skipping: `{python}` is older than 3.12 and has no `typing_extensions`, which the \ + program's `override` comes from there" + ); + } + runs.then_some(python) +} + /// the headline case: a concrete subclass fixes its type arguments in /// `__orig_bases__`, so the runtime probe confirms it — even across the /// `list` -> `Sequence` boundary, which pure `__mro__` introspection can't see /// but `issubclass` + the recorded arguments can #[test] fn parametric_is_concrete_subclass_confirms_across_origins() { + let Some(python) = python_running_override() else { + return; + }; let dir = tempfile::tempdir().expect("tempdir"); fs::write( dir.path().join("main.by"), @@ -4906,6 +5011,7 @@ print(object() is Sequence[int]) let output = Command::new(env!("CARGO_BIN_EXE_by")) .env(EnvVars::BY_NO_PROJECT_SERVER, "1") .args(["run", "main"]) + .env("PYTHON", &python) .current_dir(dir.path()) .output() .expect("failed to spawn by"); @@ -4971,10 +5077,6 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_a_reified_generic_tests_the_type_argument() { // a reified type parameter carries the type the caller chose, so the guard // tests exactly that rather than the ceiling: `PermissionError` is an @@ -4982,7 +5084,6 @@ fn raises_guard_of_a_reified_generic_tests_the_type_argument() { // the specialization is applied after the guard decorator, so this also says // the guard kept `rethrow[...]` answering let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5030,15 +5131,10 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_a_reified_class_reads_the_receiver() { // a class's type argument belongs to the instance, so a method's guard asks // the receiver it was called on for it let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5088,10 +5184,6 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_an_unreified_parameter_tests_its_ceiling() { // nothing carries `T` at runtime here, and asking for it would mean reifying // the parameter — an option that adds a check must not change the shape of @@ -5099,8 +5191,7 @@ fn raises_guard_of_an_unreified_parameter_tests_its_ceiling() { // an `OSError` passes, anything else does not. built for 3.9, the generic // `def` goes through the pep 695 polyfill, whose `TypeVar` definitions have // to land above the guard - let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); + let Some(python) = python_at_least(9, "the program is built for") else { return; }; @@ -5154,16 +5245,11 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_a_reified_generic_method_reads_the_bound_instance() { // a method that is itself reified is wrapped in `generic`, which binds the // receiver and passes the rest on: the class's argument comes from the // instance the method was looked up on, not from whatever is passed first let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5214,15 +5300,10 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_a_nested_function_tests_a_class_parameter_at_its_ceiling() { // `inner` is not a method, so its first argument is not a receiver and the // class's argument cannot be read off it: the guard tests the bound instead let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5271,15 +5352,10 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_of_a_nested_function_reads_an_enclosing_reified_argument() { // the guard on `inner` is evaluated inside a call of `outer`, where `T` is // already the argument `outer` was specialized with let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5327,16 +5403,11 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_tests_a_subscripted_type_argument_by_its_origin() { // `isinstance` refuses `MyErr[int]`, and a guard that raised `TypeError` // there would replace the exception the function legitimately raised. the // shallow test is the origin, as `list[str]` is tested as `list` let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -5387,15 +5458,10 @@ def main(): } #[test] -#[expect( - clippy::print_stderr, - reason = "a skipped test prints why it was skipped" -)] fn raises_guard_keeps_a_reified_generic_documented() { // the guard wraps a reified generic in an object of its own, which must not // answer for the function's docstring with its own let Some(python) = python_at_least_312() else { - eprintln!("skipping: no python 3.12+ interpreter available"); return; }; @@ -6703,15 +6769,17 @@ fn interpreter_version(python: &Path) -> (u8, u8) { } /// the version of the interpreter `by run` would pick, so a test can name one -/// that is definitely newer +/// that is definitely newer. outside an environment that is `$PYTHON`, and +/// `python3` when nothing pins one fn running_python_version() -> (u8, u8) { - let output = Command::new("python3") + let python = std::env::var("PYTHON").unwrap_or_else(|_| "python3".to_owned()); + let output = Command::new(python) .args([ "-c", "import sys; print(f'{sys.version_info[0]} {sys.version_info[1]}')", ]) .output() - .expect("python3 is needed to run this test"); + .expect("an interpreter is needed to run this test"); let rendered = String::from_utf8_lossy(&output.stdout); let mut parts = rendered.split_whitespace(); let major = parts.next().unwrap().parse().unwrap(); diff --git a/crates/ty/tests/django_lookup_runtime.rs b/crates/ty/tests/django_lookup_runtime.rs index 0ebec68f2d..4486641519 100644 --- a/crates/ty/tests/django_lookup_runtime.rs +++ b/crates/ty/tests/django_lookup_runtime.rs @@ -13,21 +13,21 @@ use std::fs; use std::path::Path; use std::process::Command; -/// a django small enough to read and complete enough for both halves: ty -/// recognizes the model, the fields and the manager from it (the `KnownClass` -/// names have to be *defined* in django's own module paths, on a third-party -/// search path, hence the site-packages layout), and it runs, recording the -/// keywords a lookup call is handed +/// the stubs of a django small enough to read: ty recognizes the model, the +/// fields and the manager from them (the `KnownClass` names have to be *defined* +/// in django's own module paths, on a third-party search path, hence the +/// site-packages layout). the modules beside them, [`MOCK_DJANGO_RUNTIME`], are +/// what runs const MOCK_DJANGO: &[(&str, &str)] = &[ - ("django/__init__.py", ""), + ("django/__init__.pyi", ""), // `from django.db import models` reads `models` as an attribute of the // package, which only exists once the submodule is imported ( - "django/db/__init__.py", + "django/db/__init__.pyi", "from django.db import models as models\n", ), ( - "django/db/models/__init__.py", + "django/db/models/__init__.pyi", "from django.db.models.base import Model as Model\n\ from django.db.models.fields import CharField as CharField, DateField as DateField\n\ from django.db.models.fields.json import JSONField as JSONField\n\ @@ -36,7 +36,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ from django.db.models.query import Q as Q, QuerySet as QuerySet\n", ), ( - "django/db/models/base.py", + "django/db/models/base.pyi", "from typing import Any, ClassVar, Self\n\ \n\ from django.db.models.manager import Manager\n\ @@ -51,7 +51,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ \x20 cls.objects = Manager()\n", ), ( - "django/db/models/manager.py", + "django/db/models/manager.pyi", "from typing import Any\n\ \n\ from django.db.models.query import QuerySet\n\ @@ -69,7 +69,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ \x20 pass\n", ), ( - "django/db/models/query.py", + "django/db/models/query.pyi", "from typing import Any\n\ \n\ \n\ @@ -82,7 +82,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ \x20 self.recorded = recorded or {}\n", ), ( - "django/db/models/fields/__init__.py", + "django/db/models/fields/__init__.pyi", "from typing import Any\n\ \n\ \n\ @@ -107,7 +107,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ // the key/index transforms live on `JSONField`, which is recognized by the // module it is declared in, so the mock has to declare it in that module too ( - "django/db/models/fields/json.py", + "django/db/models/fields/json.pyi", "from typing import Any\n\ \n\ from django.db.models.fields import Field\n\ @@ -118,7 +118,7 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ \x20 _pyi_private_get_type: Any\n", ), ( - "django/db/models/fields/related.py", + "django/db/models/fields/related.pyi", "from typing import Any\n\ \n\ from django.db.models.fields import Field\n\ @@ -134,6 +134,113 @@ const MOCK_DJANGO: &[(&str, &str)] = &[ ), ]; +/// the modules [`MOCK_DJANGO`] describes, generic through `typing.Generic` so +/// that they run on every version the transpiler targets: the lowering writes a +/// field's construction as the specialization it checked, `CharField[str, str]`. +/// a manager records the keywords a lookup call is handed +const MOCK_DJANGO_RUNTIME: &[(&str, &str)] = &[ + ("django/__init__.py", ""), + ( + "django/db/__init__.py", + "from django.db import models as models\n", + ), + ( + "django/db/models/__init__.py", + "from django.db.models.base import Model as Model\n\ + from django.db.models.fields import CharField as CharField, DateField as DateField\n\ + from django.db.models.fields.json import JSONField as JSONField\n\ + from django.db.models.fields.related import CASCADE as CASCADE, ForeignKey as ForeignKey\n\ + from django.db.models.manager import Manager as Manager\n\ + from django.db.models.query import Q as Q, QuerySet as QuerySet\n", + ), + ( + "django/db/models/base.py", + "from django.db.models.manager import Manager\n\ + \n\ + \n\ + class Model:\n\ + \x20 def __init_subclass__(cls, **kwargs):\n\ + \x20 super().__init_subclass__(**kwargs)\n\ + \x20 cls.objects = Manager()\n", + ), + ( + "django/db/models/manager.py", + "from typing import Generic, TypeVar\n\ + \n\ + from django.db.models.query import QuerySet\n\ + \n\ + _M = TypeVar(\"_M\")\n\ + \n\ + \n\ + class BaseManager(Generic[_M]):\n\ + \x20 def filter(self, *args, **kwargs):\n\ + \x20 return QuerySet(kwargs)\n\ + \n\ + \x20 def exclude(self, *args, **kwargs):\n\ + \x20 return QuerySet(kwargs)\n\ + \n\ + \n\ + class Manager(BaseManager[_M]):\n\ + \x20 pass\n", + ), + ( + "django/db/models/query.py", + "from typing import Generic, TypeVar\n\ + \n\ + _M = TypeVar(\"_M\")\n\ + _Row = TypeVar(\"_Row\")\n\ + \n\ + \n\ + class Q:\n\ + \x20 def __init__(self, **kwargs):\n\ + \x20 pass\n\ + \n\ + \n\ + class QuerySet(Generic[_M, _Row]):\n\ + \x20 def __init__(self, recorded=None):\n\ + \x20 self.recorded = recorded or {}\n", + ), + ( + "django/db/models/fields/__init__.py", + "from typing import Generic, TypeVar\n\ + \n\ + _ST = TypeVar(\"_ST\")\n\ + _GT = TypeVar(\"_GT\")\n\ + \n\ + \n\ + class Field(Generic[_ST, _GT]):\n\ + \x20 def __init__(self, **kwargs):\n\ + \x20 pass\n\ + \n\ + \n\ + class CharField(Field[_ST, _GT]):\n\ + \x20 pass\n\ + \n\ + \n\ + class DateField(Field[_ST, _GT]):\n\ + \x20 pass\n", + ), + ( + "django/db/models/fields/json.py", + "from django.db.models.fields import Field, _GT, _ST\n\ + \n\ + \n\ + class JSONField(Field[_ST, _GT]):\n\ + \x20 pass\n", + ), + ( + "django/db/models/fields/related.py", + "from django.db.models.fields import Field, _GT, _ST\n\ + \n\ + CASCADE = \"cascade\"\n\ + \n\ + \n\ + class ForeignKey(Field[_ST, _GT]):\n\ + \x20 def __init__(self, to, on_delete=None, **kwargs):\n\ + \x20 pass\n", + ), +]; + const MODELS: &str = "\ from django.db import models @@ -188,9 +295,10 @@ label("json key and field", Doc.objects.filter(name == "n", data["key"] == 1)) print("calls", calls) "#; -/// a CPython 3.13, provisioned through uv the way the divergence harness does: -/// the mock django uses pep 695 generics and pep 696 defaults, whose runtime -/// floor is 3.13 +/// the interpreter the lowered program runs on: `$PYTHON` when it is set, which +/// is the interpreter the run was pinned to, and otherwise a CPython 3.13 +/// provisioned through uv the way the divergence harness does. the project +/// targets whichever version it is #[cfg(not(windows))] fn python() -> Option { if let Ok(path) = std::env::var("PYTHON") { @@ -221,16 +329,32 @@ fn python() -> Option { std::env::var("PYTHON").ok() } -/// where a venv keeps its installed packages, in the layout the host platform -/// actually uses: `/Lib/site-packages` on windows, and -/// `/lib/pythonX.Y/site-packages` everywhere else. ty resolves the -/// directory the same way, and looks for the windows one directly rather than -/// scanning `lib/`, so a unix layout hands it nothing to find -fn site_packages() -> &'static str { +/// the `major.minor` version of `python` +fn python_version(python: &str) -> String { + let output = Command::new(python) + .args([ + "-c", + "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}')", + ]) + .output() + .expect("failed to spawn python"); + assert!( + output.status.success(), + "{python} could not say its version" + ); + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +/// where a venv of python `version` keeps its installed packages, in the layout +/// the host platform actually uses: `/Lib/site-packages` on windows, +/// and `/lib/pythonX.Y/site-packages` everywhere else. ty resolves +/// the directory the same way, and looks for the windows one directly rather +/// than scanning `lib/`, so a unix layout hands it nothing to find +fn site_packages(version: &str) -> String { if cfg!(windows) { - ".venv/Lib/site-packages" + ".venv/Lib/site-packages".to_owned() } else { - ".venv/lib/python3.13/site-packages" + format!(".venv/lib/python{version}/site-packages") } } @@ -262,7 +386,7 @@ fn transpile(project: &Path, module: &str) -> String { fn lookup_expressions_hand_django_the_keywords_they_spell() { let Some(python) = python() else { eprintln!( - "skipping django lookup runtime test: no python 3.13 found \ + "skipping django lookup runtime test: no interpreter found \ (set PYTHON to one, or make `uv` available)" ); return; @@ -270,22 +394,27 @@ fn lookup_expressions_hand_django_the_keywords_they_spell() { let project = tempfile::tempdir().expect("tempdir"); let root = project.path(); + // the project targets the interpreter that runs it, so the lowered program + // is written for that version + let version = python_version(&python); // ty only reads django's `KnownClass` names off a third-party search path, // so the mock lives in a venv's site-packages rather than beside the source - let site_packages = site_packages(); - for (relative, contents) in MOCK_DJANGO { + let site_packages = site_packages(&version); + for (relative, contents) in MOCK_DJANGO.iter().chain(MOCK_DJANGO_RUNTIME) { write(root, &format!("{site_packages}/{relative}"), contents); } write( root, ".venv/pyvenv.cfg", - "home = /usr/bin\nversion = 3.13.0\n", + &format!("home = /usr/bin\nversion = {version}.0\n"), ); write( root, "pyproject.toml", - "[project]\nname = \"probe\"\nversion = \"0\"\nrequires-python = \">=3.13\"\n\n\ - [tool.ty.environment]\npython = \".venv\"\npython-version = \"3.13\"\n", + &format!( + "[project]\nname = \"probe\"\nversion = \"0\"\nrequires-python = \">={version}\"\n\n\ + [tool.ty.environment]\npython = \".venv\"\npython-version = \"{version}\"\n" + ), ); write(root, "models.by", MODELS); write(root, "main.by", MAIN); @@ -303,7 +432,7 @@ fn lookup_expressions_hand_django_the_keywords_they_spell() { let output = Command::new(&python) .arg("main.py") .current_dir(&lowered) - .env("PYTHONPATH", root.join(site_packages).as_os_str()) + .env("PYTHONPATH", root.join(&site_packages).as_os_str()) .output() .expect("failed to spawn python"); assert!( diff --git a/crates/ty/tests/mdtest_divergence.rs b/crates/ty/tests/mdtest_divergence.rs index 3135088749..7117c7ca00 100644 --- a/crates/ty/tests/mdtest_divergence.rs +++ b/crates/ty/tests/mdtest_divergence.rs @@ -1,6 +1,6 @@ //! checker/runtime divergence harness. //! -//! every `.by` code block in the basedpython mdtests that the checker accepts +//! every `.by` code block in the mdtests that the checker accepts //! (no `# error:` or `# snapshot` assertions) must also transpile and *execute* //! cleanly: the mdtest framework verifies ty's diagnostics, this test verifies //! the runtime half of the contract. divergences of the form "checks clean but crashes at @@ -18,6 +18,10 @@ //! differently, or no longer checks. it sees only what this corpus exercises: an import read //! only by a lowering, in a checker-clean block of a single file. a multi-file section is not //! run, so a read across modules is out of its reach, as is any lowering no such block makes +//! +//! each block runs as a program of its own, where mdtest checks the blocks of a +//! section as one file, so a block that builds on an earlier one in its section +//! repeats what it needs of it use std::fs; use std::io::Write; @@ -30,13 +34,14 @@ fn mdtest_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../ty_python_semantic/resources/mdtest") } -/// A CPython 3.13 interpreter, provisioned through uv so the harness runs the -/// same interpreter everywhere instead of riding on whatever `python3` the host -/// happens to ship. The transpiler emits modern syntax — PEP 695 generics, PEP -/// 696 type-parameter defaults (`class C[T = int]`), PEP 646 unpacking — whose -/// runtime floor is 3.13; CI runners range from 3.10 upward, so a checker-clean -/// block can fail to even parse on an older interpreter. Returns `None` (the -/// test then skips) when uv or the interpreter can't be obtained. +/// The interpreter the blocks run on: `$PYTHON` when it is set, which is the +/// interpreter the run was pinned to, and otherwise a CPython 3.13 provisioned +/// through uv, so the harness runs the same interpreter everywhere instead of +/// riding on whatever `python3` the host happens to ship. Every block is +/// transpiled for that interpreter's version, so what runs is the lowering for +/// it, and a section whose blocks need a newer one is listed in [`REQUIRES`]. +/// Returns `None` (the test then skips) when uv or the interpreter can't be +/// obtained. #[cfg(not(windows))] fn python() -> Option { if let Ok(p) = std::env::var("PYTHON") { @@ -85,13 +90,41 @@ fn python_version(python: &str) -> Option { .then(|| String::from_utf8_lossy(&output.stdout).trim().to_owned()) } -/// Extract the `by` fenced code blocks of a markdown file, in order, with a flag -/// for blocks living in a multi-file section (one that declares companion -/// modules via a `` `name.py`: `` marker) — those import section-local modules -/// and cannot run standalone. -fn by_blocks(markdown: &str) -> Vec<(String, bool)> { +/// a `by` fenced code block of an mdtest file +#[derive(Debug, PartialEq)] +struct Block { + source: String, + /// whether the block lives in a multi-file section (one that declares + /// companion modules via a `` `name.py`: `` marker). those import + /// section-local modules and cannot run standalone + multi_file: bool, + /// the text of the heading the block is under + heading: String, +} + +/// a checker-clean block the harness runs +struct Item { + name: String, + index: usize, + source: String, + /// the entry of [`REQUIRES`] that speaks for the block on this interpreter + requires: Option, + /// the entry of [`DIVERGES_BELOW`] that speaks for the block on this interpreter + diverges_below: Option, +} + +/// whether `error` is the transpiler refusing a block for needing a python newer than +/// `3.{minor}`, and no newer than `3.{floor}` +fn refuses_for_version(error: &str, minor: u8, floor: u8) -> bool { + (minor + 1..=floor) + .any(|needed| error.contains(&format!("requires python 3.{needed} or newer"))) +} + +/// extract the `by` fenced code blocks of a markdown file, in order +fn by_blocks(markdown: &str) -> Vec { let mut blocks: Vec<(String, usize)> = Vec::new(); let mut multi_file_sections: Vec = Vec::new(); + let mut headings: Vec = vec![String::new()]; let mut section = 0usize; // the backtick run of the fence currently open, if any. every fence is // tracked, not just the `by` ones: a `# error:` comment at column 0 inside a @@ -126,6 +159,7 @@ fn by_blocks(markdown: &str) -> Vec<(String, bool)> { } if line.starts_with('#') { section += 1; + headings.push(line.trim_start_matches('#').trim().to_owned()); } // a companion-module marker: `` `pylib.py`: `` ahead of its fence if trimmed.starts_with('`') @@ -138,7 +172,11 @@ fn by_blocks(markdown: &str) -> Vec<(String, bool)> { } blocks .into_iter() - .map(|(b, s)| (b, multi_file_sections.contains(&s))) + .map(|(source, section)| Block { + source, + multi_file: multi_file_sections.contains(§ion), + heading: headings[section].clone(), + }) .collect() } @@ -293,6 +331,120 @@ fn unused_import_leg( }) } +/// the sections whose checker-clean blocks do not run, each with the reason, keyed +/// by mdtest file (relative to the mdtest directory) and heading text +/// +/// each entry is a known divergence that is not fixed yet. a block that is not +/// meant to run as a program has no place here: it is written to run, or asserts +/// the diagnostic that leaves its runtime behaviour unspecified. an entry that no +/// longer matches a checker-clean block fails the test, so the list cannot +/// outlive its reasons +const EXCLUDED: &[(&str, &str, &str)] = &[( + "abstract.md", + "abstract methods are tracked", + "ty does not report instantiating a class that leaves an abstract method \ + unimplemented (the block's `TODO`), so the block checks clean and python raises \ + `TypeError` at the instantiation", +)]; + +/// the sections a transpile for an older python refuses, each with the version its blocks +/// need and the reason, keyed by mdtest file (relative to the mdtest directory) and heading +/// text. a heading of `None` stands for every section of the file +/// +/// below its version, a block of such a section is either refused with the transpiler's +/// version requirement — `requires python 3.N or newer`, for an `N` the interpreter is older +/// than — or transpiled and run like any other block, and a refusal is not a divergence. an +/// entry that no longer matches a checker-clean block fails the test, and so does one that +/// none of its blocks is refused for below its version +const REQUIRES: &[(&str, Option<&str>, u8, &str)] = &[ + ( + "basedpython_reified_generics.md", + None, + 13, + "a reified generic keeps its native pep 695 type-parameter list, which python has from \ + 3.12, and a pep 696 default in it, which python has from 3.13", + ), + ( + "basedpython_reified_class_generics.md", + None, + 13, + "a reified class keeps its native pep 695 type-parameter list, which python has from \ + 3.12, and a pep 696 default in it, which python has from 3.13", + ), + ( + "basedpython_checked_cast.md", + Some("a reified type parameter is checked, not assumed"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_exceptions.md", + Some("an explicit specialization is read like a solved call"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_optional_type.md", + Some("a wrapped optional read as a value is the wrapper class"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_parametric_is.md", + Some("reified type parameters carry the answer"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_parametric_is.md", + Some("a tuple target compares each element cell"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_parametric_is.md", + Some("a nested generic value unifies structurally"), + 12, + "a reified generic keeps its native pep 695 type-parameter list", + ), + ( + "basedpython_parametric_is.md", + Some("a reified class answers the probe from its specialization"), + 12, + "a reified class keeps its native pep 695 type-parameter list", + ), +]; + +/// the sections whose checker-clean blocks diverge on a target older than a python version, +/// each with the version and the reason, keyed like [`EXCLUDED`] +/// +/// each entry is a known divergence that is not fixed yet. below its version a block of the +/// section still runs, and has to diverge: an entry none of whose blocks diverges there fails +/// the test, so the list cannot outlive its reasons. from its version on, the section runs like +/// any other +const DIVERGES_BELOW: &[(&str, &str, u8, &str)] = &[]; + +/// every mdtest under `dir`, at any depth, in a stable order +fn markdown_files(dir: &Path) -> Vec { + let mut files = Vec::new(); + let mut pending = vec![dir.to_path_buf()]; + while let Some(dir) = pending.pop() { + for entry in fs::read_dir(&dir).expect("mdtest dir") { + let path = entry.expect("mdtest dir entry").path(); + if path.is_dir() { + pending.push(path); + } else if path + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("md")) + { + files.push(path); + } + } + } + files.sort(); + files +} + /// Stub `reveal_type` (an mdtest debugging device with no runtime binding) /// after the `__future__` import, which must stay first. fn with_reveal_stub(transpiled: &str) -> String { @@ -318,14 +470,12 @@ fn clean_mdtest_blocks_run() { return; }; - // reified generics emit native pep 695 syntax (the closure mechanism), so - // they only transpile on a 3.12+ target — on an older interpreter those - // blocks legitimately can't run, and their transpile error is documented, - // not a divergence - let supports_reification = version + let Some(minor) = version .split_once('.') - .and_then(|(_, minor)| minor.parse::().ok()) - .is_some_and(|minor| minor >= 12); + .and_then(|(_, minor)| minor.parse::().ok()) + else { + panic!("`{python}` reports a version that is not `3.N`: {version}"); + }; // third-party runtime deps are environment-dependent; skip blocks that // need one the interpreter doesn't have @@ -351,6 +501,14 @@ fn clean_mdtest_blocks_run() { .output() .is_ok_and(|o| o.status.success()); + // the django mdtests run their models against the framework; skip their + // blocks when it isn't installed, and run them locally against an + // interpreter that has django to enforce the contract + let has_django = Command::new(&python) + .args(["-c", "import django"]) + .output() + .is_ok_and(|o| o.status.success()); + // the pytest divergence suite drives real pytest over the transpiled output // (each block re-invokes `pytest.main` on itself under `__main__`); skip its // blocks when pytest isn't installed, and run them locally against an @@ -387,19 +545,8 @@ fn clean_mdtest_blocks_run() { .is_ok_and(|o| o.status.success()); let dir = mdtest_dir(); - let mut files: Vec = fs::read_dir(&dir) - .expect("mdtest dir") - .filter_map(|e| e.ok().map(|e| e.path())) - .filter(|p| { - p.extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("md")) - && p.file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.starts_with("basedpython_")) - }) - .collect(); - files.sort(); - assert!(!files.is_empty(), "no basedpython mdtests found in {dir:?}"); + let files = markdown_files(&dir); + assert!(!files.is_empty(), "no mdtests found in {dir:?}"); // gather every runnable block up front — skipping ones with expected // diagnostics (only checker-clean blocks carry the contract) or that import @@ -421,6 +568,7 @@ fn clean_mdtest_blocks_run() { (!has_templatelib).then_some("string.templatelib"), (!has_pydantic).then_some("pydantic"), (!has_sqlalchemy).then_some("sqlalchemy"), + (!has_django).then_some("django"), (!has_pytest).then_some("pytest"), (!has_basedpython_ui).then_some("basedpython_ui"), Some("ty_extensions"), @@ -429,25 +577,99 @@ fn clean_mdtest_blocks_run() { .flatten() .collect(); - let mut items: Vec<(String, usize, String)> = Vec::new(); + let mut items: Vec = Vec::new(); + let mut excluded_seen = vec![false; EXCLUDED.len()]; + let mut requires_seen = vec![false; REQUIRES.len()]; + let mut diverges_below_seen = vec![false; DIVERGES_BELOW.len()]; for file in &files { - let name = file.file_name().unwrap().to_string_lossy().into_owned(); + let name = file + .strip_prefix(&dir) + .expect("an mdtest is under the mdtest directory") + .to_string_lossy() + .replace('\\', "/"); let markdown = fs::read_to_string(file).expect("read mdtest"); - for (i, (block, multi_file)) in by_blocks(&markdown).into_iter().enumerate() { - if has_expected_diagnostics(&block) || multi_file { + for (i, block) in by_blocks(&markdown).into_iter().enumerate() { + if has_expected_diagnostics(&block.source) || block.multi_file { continue; } - if unavailable.iter().any(|module| block.contains(module)) { + if let Some(excluded) = EXCLUDED + .iter() + .position(|(file, heading, _)| *file == name && *heading == block.heading) + { + excluded_seen[excluded] = true; continue; } - items.push((name.clone(), i, block)); + let requires = REQUIRES.iter().position(|(file, heading, _, _)| { + *file == name && heading.is_none_or(|heading| heading == block.heading) + }); + if let Some(entry) = requires { + requires_seen[entry] = true; + } + let diverges_below = DIVERGES_BELOW + .iter() + .position(|(file, heading, _, _)| *file == name && *heading == block.heading); + if let Some(entry) = diverges_below { + diverges_below_seen[entry] = true; + } + if unavailable + .iter() + .any(|module| block.source.contains(module)) + { + continue; + } + items.push(Item { + name: name.clone(), + index: i, + source: block.source, + // an entry only speaks for an interpreter older than its version + requires: requires.filter(|&entry| minor < REQUIRES[entry].2), + diverges_below: diverges_below.filter(|&entry| minor < DIVERGES_BELOW[entry].2), + }); } } + let stale: Vec = EXCLUDED + .iter() + .zip(&excluded_seen) + .filter(|(_, seen)| !**seen) + .map(|((file, heading, _), _)| format!("{file}: {heading}")) + .chain( + REQUIRES + .iter() + .zip(&requires_seen) + .filter(|(_, seen)| !**seen) + .map(|((file, heading, _, _), _)| { + format!("{file}: {}", heading.unwrap_or("every section")) + }), + ) + .chain( + DIVERGES_BELOW + .iter() + .zip(&diverges_below_seen) + .filter(|(_, seen)| !**seen) + .map(|((file, heading, _, _), _)| format!("{file}: {heading}")), + ) + .collect(); + assert!( + stale.is_empty(), + "a listed section no longer holds a checker-clean block; remove it from `EXCLUDED`, \ + `REQUIRES` or `DIVERGES_BELOW`:\n{}", + stale.join("\n") + ); let total = items.len(); assert!(total > 0, "no checker-clean by blocks found"); let tmp = tempfile::tempdir().expect("tempdir"); let failures: Mutex> = Mutex::new(Vec::new()); + // the entries of `REQUIRES` a block was refused for, and of `DIVERGES_BELOW` a block + // diverged for, on this interpreter + let refused: Mutex> = Mutex::new(vec![false; REQUIRES.len()]); + let diverged: Mutex> = Mutex::new(vec![false; DIVERGES_BELOW.len()]); + // a divergence of a block `DIVERGES_BELOW` lists for this interpreter is the one expected + // of it, and any other is reported + let diverge = |item: &Item, report: String| match item.diverges_below { + Some(entry) => diverged.lock().unwrap()[entry] = true, + None => failures.lock().unwrap().push(report), + }; let next = AtomicUsize::new(0); // each block alternates a cpu-bound `by transpile` with a wait on the python // subprocess, and `by` startup also reads typeshed from disk — so a little @@ -466,20 +688,28 @@ fn clean_mdtest_blocks_run() { std::thread::scope(|scope| { for _ in 0..workers { scope.spawn(|| { - while let Some((name, i, block)) = - items.get(next.fetch_add(1, Ordering::Relaxed)) - { + while let Some(item) = items.get(next.fetch_add(1, Ordering::Relaxed)) { + let Item { + name, + index: i, + source: block, + .. + } = item; let transpiled = match transpile(block, &version) { Ok(t) => t, Err(e) => { - // a reified generic on a <3.12 interpreter is a - // documented transpile error, not a divergence - if !supports_reification && e.contains("python 3.12 or newer") { + if let Some(entry) = item.requires + && refuses_for_version(&e, minor, REQUIRES[entry].2) + { + refused.lock().unwrap()[entry] = true; continue; } - failures.lock().unwrap().push(format!( - "{name} block {i}: transpile failed:\n{e}\n--- block ---\n{block}" - )); + diverge( + item, + format!( + "{name} block {i}: transpile failed:\n{e}\n--- block ---\n{block}" + ), + ); continue; } }; @@ -505,6 +735,9 @@ fn clean_mdtest_blocks_run() { if !has_sqlalchemy && transpiled.contains("sqlalchemy") { continue; } + if !has_django && transpiled.contains("django") { + continue; + } if !has_pytest && transpiled.contains("pytest") { continue; } @@ -513,7 +746,7 @@ fn clean_mdtest_blocks_run() { } let py = tmp.path().join(format!( "{}_{i}.py", - name.trim_end_matches(".md").replace('-', "_") + name.trim_end_matches(".md").replace(['-', '/'], "_") )); fs::write(&py, with_reveal_stub(&transpiled)).unwrap(); let run = Command::new(&python) @@ -521,13 +754,16 @@ fn clean_mdtest_blocks_run() { .output() .expect("failed to run python"); if !run.status.success() { - failures.lock().unwrap().push(format!( - "{name} block {i}: checker-clean block crashed at runtime:\n{}\n--- block ---\n{block}", - String::from_utf8_lossy(&run.stderr) - )); + diverge( + item, + format!( + "{name} block {i}: checker-clean block crashed at runtime:\n{}\n--- block ---\n{block}", + String::from_utf8_lossy(&run.stderr) + ), + ); continue; } - let key = format!("{}_{i}", name.trim_end_matches(".md").replace('-', "_")); + let key = format!("{}_{i}", name.trim_end_matches(".md").replace(['-', '/'], "_")); if let Some(divergence) = unused_import_leg(block, &version, &python, tmp.path(), &key) { @@ -540,7 +776,25 @@ fn clean_mdtest_blocks_run() { } }); - let failures = failures.into_inner().expect("failures mutex poisoned"); + let mut failures = failures.into_inner().expect("failures mutex poisoned"); + let refused = refused.into_inner().expect("refusals mutex poisoned"); + let diverged = diverged.into_inner().expect("divergences mutex poisoned"); + for (entry, (file, heading, floor, _)) in REQUIRES.iter().enumerate() { + if minor < *floor && !refused[entry] { + failures.push(format!( + "{file}: {}: no block is refused below 3.{floor} any more; remove it from `REQUIRES`", + heading.unwrap_or("every section") + )); + } + } + for (entry, (file, heading, floor, _)) in DIVERGES_BELOW.iter().enumerate() { + if minor < *floor && !diverged[entry] { + failures.push(format!( + "{file}: {heading}: no block diverges below 3.{floor} any more; remove it from \ + `DIVERGES_BELOW`" + )); + } + } assert!( failures.is_empty(), "{} of {} checker-clean blocks diverge at runtime:\n\n{}", @@ -573,7 +827,11 @@ f() let blocks = by_blocks(markdown); assert_eq!( blocks, - vec![("from m import f\nf()\n".to_string(), true)], + vec![Block { + source: "from m import f\nf()\n".to_string(), + multi_file: true, + heading: "a section".to_string(), + }], "the block belongs to the marker's section" ); } @@ -599,12 +857,17 @@ from m import f print(1) ``` "; + let block = |source: &str, multi_file: bool, heading: &str| Block { + source: source.to_string(), + multi_file, + heading: heading.to_string(), + }; assert_eq!( by_blocks(markdown), vec![ - ("def f() -> None: ...\n".to_string(), true), - ("from m import f\n".to_string(), true), - ("print(1)\n".to_string(), false), + block("def f() -> None: ...\n", true, "multi-file"), + block("from m import f\n", true, "multi-file"), + block("print(1)\n", false, "standalone"), ] ); } diff --git a/crates/ty_ide/src/semantic_tokens.rs b/crates/ty_ide/src/semantic_tokens.rs index ef32d3be7b..1ffc448e4b 100644 --- a/crates/ty_ide/src/semantic_tokens.rs +++ b/crates/ty_ide/src/semantic_tokens.rs @@ -941,7 +941,7 @@ impl<'db> SemanticTokenVisitor<'db> { | DefinitionKind::TypeMatchCapture(_) => { Some((SemanticTokenType::TypeParameter, modifiers)) } - DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(parameter)) => { + DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(parameter, _)) => { let parsed = parsed_module(db, definition.python_file(db)); let ty = parameter.node(&parsed.load(db)).inferred_type(&model); diff --git a/crates/ty_python_core/src/builder.rs b/crates/ty_python_core/src/builder.rs index 6af822d206..e9ffc4a85d 100644 --- a/crates/ty_python_core/src/builder.rs +++ b/crates/ty_python_core/src/builder.rs @@ -43,8 +43,8 @@ use crate::definition::{ ImportFromDefinitionNodeRef, ImportFromSubmoduleDefinitionNodeRef, LambdaParameterDefinitionNodeRef, LoopHeaderDefinitionNodeRef, LoopStmtRef, MatchPatternDefinitionNodeRef, NestedBindingExecution, NestedBindingsDefinitionKind, - ParameterDefinitionNodeRef, StarImportDefinitionNodeRef, TypeMatchCaptureDefinitionNodeRef, - WithItemDefinitionNodeRef, + ParameterDefinitionNodeRef, ParameterName, StarImportDefinitionNodeRef, + TypeMatchCaptureDefinitionNodeRef, WithItemDefinitionNodeRef, repeats_underscore, }; use crate::expression::{Expression, ExpressionKind}; use crate::fluid::{FluidUse, FluidUseRole}; @@ -4319,8 +4319,12 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { } fn declare_parameters(&mut self, parameters: &'ast ast::Parameters) { + let repeats_underscore = repeats_underscore(parameters, self.source_type); + let name = |parameter: &ast::Parameter| { + ParameterName::new(parameter.name.as_str(), repeats_underscore) + }; for parameter in parameters.iter_non_variadic_params() { - self.declare_parameter(parameter); + self.declare_parameter(parameter, name(¶meter.parameter)); } if let Some(vararg) = parameters.vararg.as_ref() { let symbol = self.add_symbol(vararg.name.id().clone()); @@ -4329,7 +4333,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { .mark_parameter(); self.add_definition( symbol.into(), - ParameterDefinitionNodeRef::VariadicPositionalParameter(vararg), + ParameterDefinitionNodeRef::VariadicPositionalParameter(vararg, name(vararg)), ); } if let Some(kwarg) = parameters.kwarg.as_ref() { @@ -4339,17 +4343,21 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { .mark_parameter(); self.add_definition( symbol.into(), - ParameterDefinitionNodeRef::VariadicKeywordParameter(kwarg), + ParameterDefinitionNodeRef::VariadicKeywordParameter(kwarg, name(kwarg)), ); } } - fn declare_parameter(&mut self, parameter: &'ast ast::ParameterWithDefault) { + fn declare_parameter( + &mut self, + parameter: &'ast ast::ParameterWithDefault, + name: ParameterName, + ) { let symbol = self.add_symbol(parameter.name().id().clone()); let definition = self.add_definition( symbol.into(), - ParameterDefinitionNodeRef::Parameter(parameter), + ParameterDefinitionNodeRef::Parameter(parameter, name), ); self.current_place_table_mut() @@ -4368,13 +4376,17 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { parameters: &'ast ast::Parameters, lambda: &'ast ast::ExprLambda, ) { + let repeats_underscore = repeats_underscore(parameters, self.source_type); + let name = |parameter: &ast::Parameter| { + ParameterName::new(parameter.name.as_str(), repeats_underscore) + }; let mut index = 0; for parameter in ¶meters.posonlyargs { - self.declare_lambda_parameter(index, parameter, lambda); + self.declare_lambda_parameter(index, parameter, name(¶meter.parameter), lambda); index += 1; } for parameter in ¶meters.args { - self.declare_lambda_parameter(index, parameter, lambda); + self.declare_lambda_parameter(index, parameter, name(¶meter.parameter), lambda); index += 1; } if let Some(vararg) = parameters.vararg.as_ref() { @@ -4387,13 +4399,16 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { LambdaParameterDefinitionNodeRef { index, lambda, - parameter: ParameterDefinitionNodeRef::VariadicPositionalParameter(vararg), + parameter: ParameterDefinitionNodeRef::VariadicPositionalParameter( + vararg, + name(vararg), + ), }, ); index += 1; } for parameter in ¶meters.kwonlyargs { - self.declare_lambda_parameter(index, parameter, lambda); + self.declare_lambda_parameter(index, parameter, name(¶meter.parameter), lambda); index += 1; } if let Some(kwarg) = parameters.kwarg.as_ref() { @@ -4406,7 +4421,10 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { LambdaParameterDefinitionNodeRef { index, lambda, - parameter: ParameterDefinitionNodeRef::VariadicKeywordParameter(kwarg), + parameter: ParameterDefinitionNodeRef::VariadicKeywordParameter( + kwarg, + name(kwarg), + ), }, ); } @@ -4416,6 +4434,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { &mut self, index: usize, parameter: &'ast ast::ParameterWithDefault, + name: ParameterName, lambda: &'ast ast::ExprLambda, ) { let symbol = self.add_symbol(parameter.name().id().clone()); @@ -4425,7 +4444,7 @@ impl<'db, 'ast> SemanticIndexBuilder<'db, 'ast> { LambdaParameterDefinitionNodeRef { index, lambda, - parameter: ParameterDefinitionNodeRef::Parameter(parameter), + parameter: ParameterDefinitionNodeRef::Parameter(parameter, name), }, ); diff --git a/crates/ty_python_core/src/definition.rs b/crates/ty_python_core/src/definition.rs index 3a8776141c..34fd10b8a3 100644 --- a/crates/ty_python_core/src/definition.rs +++ b/crates/ty_python_core/src/definition.rs @@ -7,7 +7,7 @@ use ruff_python_ast::find_node::covering_node; use ruff_python_ast::helpers::is_untyped_declaration_marker; use ruff_python_ast::name::Name; use ruff_python_ast::traversal::suite; -use ruff_python_ast::{self as ast, AnyNodeRef, Expr}; +use ruff_python_ast::{self as ast, AnyNodeRef, Expr, PySourceType}; use ruff_text_size::{Ranged, TextRange, TextSize}; use smallvec::SmallVec; @@ -613,35 +613,37 @@ pub(crate) struct ComprehensionDefinitionNodeRef<'ast, 'db> { #[derive(Copy, Clone, Debug)] pub(crate) enum ParameterDefinitionNodeRef<'ast> { - VariadicPositionalParameter(&'ast ast::Parameter), - VariadicKeywordParameter(&'ast ast::Parameter), - Parameter(&'ast ast::ParameterWithDefault), + VariadicPositionalParameter(&'ast ast::Parameter, ParameterName), + VariadicKeywordParameter(&'ast ast::Parameter, ParameterName), + Parameter(&'ast ast::ParameterWithDefault, ParameterName), } impl ParameterDefinitionNodeRef<'_> { fn into_owned(self, parsed: &ParsedModuleRef) -> ParameterDefinitionNodeKind { match self { - Self::VariadicPositionalParameter(parameter) => { - ParameterDefinitionNodeKind::VariadicPositionalParameter(AstNodeRef::new( - parsed, parameter, - )) + Self::VariadicPositionalParameter(parameter, name) => { + ParameterDefinitionNodeKind::VariadicPositionalParameter( + AstNodeRef::new(parsed, parameter), + name, + ) } - Self::VariadicKeywordParameter(parameter) => { - ParameterDefinitionNodeKind::VariadicKeywordParameter(AstNodeRef::new( - parsed, parameter, - )) + Self::VariadicKeywordParameter(parameter, name) => { + ParameterDefinitionNodeKind::VariadicKeywordParameter( + AstNodeRef::new(parsed, parameter), + name, + ) } - Self::Parameter(parameter) => { - ParameterDefinitionNodeKind::Parameter(AstNodeRef::new(parsed, parameter)) + Self::Parameter(parameter, name) => { + ParameterDefinitionNodeKind::Parameter(AstNodeRef::new(parsed, parameter), name) } } } fn key(self) -> DefinitionNodeKey { match self { - Self::VariadicPositionalParameter(node) => node.into(), - Self::VariadicKeywordParameter(node) => node.into(), - Self::Parameter(node) => (&node.parameter).into(), + Self::VariadicPositionalParameter(node, _) => node.into(), + Self::VariadicKeywordParameter(node, _) => node.into(), + Self::Parameter(node, _) => (&node.parameter).into(), } } } @@ -1396,54 +1398,96 @@ impl<'db> ComprehensionDefinitionKind<'db> { } } +/// basedpython: whether the name a parameter binds in its function's body is the parameter's alone +#[derive(Clone, Copy, Debug, PartialEq, Eq, get_size2::GetSize)] +pub enum ParameterName { + /// the name is the parameter's alone, as every parameter's name is in python, so the + /// parameter's annotation declares the name's type + Own, + /// the parameter is one of several written `_`. the lowering gives each of them a name of + /// its own, so the `_` the body binds is none of theirs and has no declared type, and + /// `_ = 5` there is checked against none of their annotations: + /// + /// ```by + /// def f(_: int, _: str) -> None: + /// _ = 5 # an undeclared local, as in `def f(_: int, _2: str, /)` + /// ``` + RepeatedUnderscore, +} + +impl ParameterName { + /// the name a parameter written as `name` binds, in a list of parameters that + /// [`repeats_underscore`] or not + pub(crate) fn new(name: &str, repeats_underscore: bool) -> Self { + if repeats_underscore && name == "_" { + Self::RepeatedUnderscore + } else { + Self::Own + } + } +} + +/// whether `parameters` writes `_` more than once, which basedpython allows and python refuses +pub(crate) fn repeats_underscore(parameters: &ast::Parameters, source_type: PySourceType) -> bool { + source_type.is_basedpython() + && parameters + .iter() + .filter(|parameter| parameter.name() == "_") + .nth(1) + .is_some() +} + #[derive(Clone, Debug, get_size2::GetSize)] pub enum ParameterDefinitionNodeKind { - VariadicPositionalParameter(AstNodeRef), - VariadicKeywordParameter(AstNodeRef), - Parameter(AstNodeRef), + VariadicPositionalParameter(AstNodeRef, ParameterName), + VariadicKeywordParameter(AstNodeRef, ParameterName), + Parameter(AstNodeRef, ParameterName), } impl ParameterDefinitionNodeKind { fn target_range(&self, module: &ParsedModuleRef) -> TextRange { match self { - Self::VariadicPositionalParameter(parameter) => parameter.node(module).name.range(), - Self::VariadicKeywordParameter(parameter) => parameter.node(module).name.range(), - Self::Parameter(parameter) => parameter.node(module).parameter.name.range(), + Self::VariadicPositionalParameter(parameter, _) => parameter.node(module).name.range(), + Self::VariadicKeywordParameter(parameter, _) => parameter.node(module).name.range(), + Self::Parameter(parameter, _) => parameter.node(module).parameter.name.range(), } } fn full_range(&self, module: &ParsedModuleRef) -> TextRange { match self { - Self::VariadicPositionalParameter(parameter) => parameter.node(module).range(), - Self::VariadicKeywordParameter(parameter) => parameter.node(module).range(), - Self::Parameter(parameter) => parameter.node(module).parameter.range(), + Self::VariadicPositionalParameter(parameter, _) => parameter.node(module).range(), + Self::VariadicKeywordParameter(parameter, _) => parameter.node(module).range(), + Self::Parameter(parameter, _) => parameter.node(module).parameter.range(), } } - fn category(&self, module: &ParsedModuleRef) -> DefinitionCategory { + /// whether the name the parameter binds in the body is the parameter's alone + pub fn name(&self) -> ParameterName { match self { - // a parameter always binds a value, but is only a declaration if annotated - Self::VariadicPositionalParameter(parameter) - | Self::VariadicKeywordParameter(parameter) => { - if parameter.node(module).annotation.is_some() { - DefinitionCategory::DeclarationAndBinding - } else { - DefinitionCategory::Binding - } - } - // presence of a default is irrelevant, same logic as for a no-default parameter - Self::Parameter(parameter_with_default) => { - if parameter_with_default - .node(module) - .parameter - .annotation - .is_some() - { - DefinitionCategory::DeclarationAndBinding - } else { - DefinitionCategory::Binding - } + Self::VariadicPositionalParameter(_, name) + | Self::VariadicKeywordParameter(_, name) + | Self::Parameter(_, name) => *name, + } + } + + fn category(&self, module: &ParsedModuleRef) -> DefinitionCategory { + // a parameter always binds a value, but is only a declaration if annotated, and the + // name it binds is its own. presence of a default is irrelevant + let annotated = match self { + Self::VariadicPositionalParameter(parameter, _) + | Self::VariadicKeywordParameter(parameter, _) => { + parameter.node(module).annotation.is_some() } + Self::Parameter(parameter_with_default, _) => parameter_with_default + .node(module) + .parameter + .annotation + .is_some(), + }; + if annotated && self.name() == ParameterName::Own { + DefinitionCategory::DeclarationAndBinding + } else { + DefinitionCategory::Binding } } } diff --git a/crates/ty_python_core/src/lib.rs b/crates/ty_python_core/src/lib.rs index 53597371f4..907617de3d 100644 --- a/crates/ty_python_core/src/lib.rs +++ b/crates/ty_python_core/src/lib.rs @@ -1532,14 +1532,17 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs): .unwrap(); assert_matches!( args_binding.kind(&db), - DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicPositionalParameter(_)) + DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicPositionalParameter( + _, + _ + )) ); let kwargs_binding = use_def .first_public_binding(function_table.symbol_id("kwargs").expect("symbol exists")) .unwrap(); assert_matches!( kwargs_binding.kind(&db), - DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicKeywordParameter(_)) + DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicKeywordParameter(_, _)) ); } @@ -1575,7 +1578,7 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs): DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { index: _, lambda: _, - parameter: ParameterDefinitionNodeKind::Parameter(_) + parameter: ParameterDefinitionNodeKind::Parameter(_, _) }) ); } @@ -1587,7 +1590,7 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs): DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { index: 3, lambda: _, - parameter: ParameterDefinitionNodeKind::VariadicPositionalParameter(_) + parameter: ParameterDefinitionNodeKind::VariadicPositionalParameter(_, _) }) ); let kwargs_binding = use_def @@ -1598,7 +1601,7 @@ def f(a: str, /, b: str, c: int = 1, *args, d: int = 2, **kwargs): DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { index: 5, lambda: _, - parameter: ParameterDefinitionNodeKind::VariadicKeywordParameter(_) + parameter: ParameterDefinitionNodeKind::VariadicKeywordParameter(_, _) }) ); } diff --git a/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md b/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md index 936ec1cdfc..97fbbbb6ce 100644 --- a/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md +++ b/crates/ty_python_semantic/resources/lint_docs/invalid-repeated-underscore.md @@ -1,6 +1,7 @@ ## What it does -Checks for a parameter list that repeats `_` in a shape the lowering has no python spelling for. +Checks for a parameter list that repeats `_` in a shape the lowering has no python spelling for, and +for a read of `_` that may be one of those parameters. ## Why is this bad? @@ -55,3 +56,22 @@ class Override(Base): override def f(self, _: int, _: int) -> int: return x ``` + +Python binds `_` to only one of the parameters, the first, while ty reads a body as binding each `_` +in turn. A read of `_` would mean a different parameter to each, so it is reported, wherever the +parameters may be the binding it reads — in the body, in a nested function, lambda or comprehension, +and in `_ += 1`: + +```by +def first(_: int, _: str) -> str: + # error: [invalid-repeated-underscore] + return _ +``` + +A read that only a binding the body makes reaches reads that binding, and `del _` reads no value: + +```by +def rebound(_: int, _: int) -> int: + _ = 5 + return _ +``` diff --git a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md index 3250523d90..9caefd1c09 100644 --- a/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md +++ b/crates/ty_python_semantic/resources/mdtest/anonymous_named_tuple.md @@ -97,8 +97,12 @@ c: (name: str, age: int) = (name=1, age=1) ### Plain tuple literal assigns to anonymous named tuple parameter +A plain tuple standing where an anonymous named tuple is expected is checked against its fields, and +is that named tuple, so its fields are read by name. + ```by -def f(x: (name: str, age: int)) -> None: ... +def f(x: (name: str, age: int)) -> str: + return x.name f(("asdf", 1)) ``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md b/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md index 3e8d3d6eb4..8971798e04 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_keyword_variadic.md @@ -380,6 +380,20 @@ class B[**P = ...]: ... class C[**P = ()]: ... ``` +## an unresolved default is reported once + +a default naming nothing is an unresolved reference, which says all there is to say about it: what +the name would have to be is never in question when there is no name + +```toml +[environment] +python-version = "3.13" +``` + +```by +class A[**Q = Missing]: ... # error: [unresolved-reference] +``` + ## a class with two packs every keyword argument names a field of a class's first pack, so no specialization spells a second @@ -412,13 +426,14 @@ def h(cb: (**kwargs: str) -> int): ## `(**TD)` unpacks a `TypedDict` a bare name in the `**` position is an unpack — the same reading a parameter pack gets. the -`TypedDict`'s keys become keyword parameters, matching `def f(**kwargs: Unpack[TD])` +`TypedDict`'s keys become keyword parameters, matching `def f(**kwargs: Unpack[TD])`. a dict literal +type is closed, so no other keyword is accepted ```by type TD = {"a": int} def f(fn: (**TD) -> None): - reveal_type(fn) # revealed: (*, a: int, **kwargs: object) -> None + reveal_type(fn) # revealed: (*, a: int) -> None fn(a=1) fn() # error: [missing-argument] fn(a="wrong") # error: [invalid-argument-type] @@ -439,10 +454,10 @@ def labelled(fn: (**kwargs: Unpack[TD]) -> None): ... def anonymous(fn: (**: Unpack[TD]) -> None): ... -reveal_type(base) # revealed: def base(*, a: int, **kwargs: object) -reveal_type(bare) # revealed: def bare(fn: (*, a: int, **kwargs: object) -> None) -reveal_type(labelled) # revealed: def labelled(fn: (*, a: int, **kwargs: object) -> None) -reveal_type(anonymous) # revealed: def anonymous(fn: (*, a: int, **kwargs: object) -> None) +reveal_type(base) # revealed: def base(*, a: int) +reveal_type(bare) # revealed: def bare(fn: (*, a: int) -> None) +reveal_type(labelled) # revealed: def labelled(fn: (*, a: int) -> None) +reveal_type(anonymous) # revealed: def anonymous(fn: (*, a: int) -> None) ``` ## `(**P)` unpacks a protocol's data members diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md index 1790ac0172..4d534708e9 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_sound_types.md @@ -1953,11 +1953,7 @@ def to(n): def fro(n): return map(to, n) -# the mutual case settles a level shallower than the single-function one above, which is the -# recursion being recognised on a different round rather than anything the program says. here it -# is shallower still: the marker is the whole answer, because the depth bound is what recognised -# the recursion and the bound replaces the value rather than folding into it -reveal_type(to([])) # revealed: Divergent +reveal_type(to([])) # revealed: map[Divergent] ``` ### recursion that grows a tuple diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_super.md b/crates/ty_python_semantic/resources/mdtest/basedpython_super.md index d21b02ef67..84b91dc6e6 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_super.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_super.md @@ -74,21 +74,64 @@ class B(A): super.does_not_exist # error: [unresolved-attribute] ``` -## `super[T]` where `T` is not in the MRO falls through +## `super[T]` looks past a generic base -`super[Other]` in a class whose MRO does not contain `Other` produces no bound super, so the -attribute lookup falls back to lookup on the `super` class itself. +The class before `T` in the MRO may be a generic class's specialization. The lookup starts after +that class: + +```by +class Box[T]: + def value(self) -> T: + raise NotImplementedError + +class Named: + def name(self) -> str: + return "named" + +class C(Box[int], Named): + def g(self) -> None: + reveal_type(super[Box].value()) # revealed: int + reveal_type(super[Named].name()) # revealed: str +``` + +## `super[T]` needs a class in the MRO + +`super[T]` finds the class before `T` in the MRO of the class it is written in when it runs. A class +that is not in that MRO raises `ValueError` there, so it is reported, as `super(T, self)` with a `T` +that `self` is no instance of is: ```by class A: pass class Other: - pass + x: int = 1 class B(A): def f(self): - super[Other].x # error: [unresolved-attribute] + super[Other].x # error: [invalid-super-argument] "`Other` is not in the MRO of `B`" +``` + +The class it is written in has no class before it, so it names no class to look up past: + +```by +class C(A): + def f(self): + super[C].f # error: [invalid-super-argument] "`C` comes first in its own MRO, so `super[C]` has no class to look up past" +``` + +`T` has to be the very class in the MRO. A value of a `type[...]` type may be any subclass, and a +specialization of a generic class is not a class at all: + +```by +class Box[T]: + pass + +class D(Box[int]): + def f(self, cls: type[A]): + super[cls].x # error: [invalid-super-argument] "`type[A]` is not known to be a class in the MRO of `D`" + super[Box[int]].x # error: [invalid-super-argument] "`types.GenericAlias` instance `Box[int]` is not a valid class" + super[1].x # error: [invalid-super-argument] "`1` is not a valid class" ``` ## chained calls work the same as the desugared form @@ -131,6 +174,95 @@ def outside() -> int: return super.a ``` +## a function nested in a method passes its own receiver + +`super` reads its implicit arguments where it is written, as zero-argument `super()` does: the +receiver is the first parameter of the function or lambda around it. A function nested in a method +passes its own first parameter, which has to be an instance of the class. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + def inner(other: B) -> int: + return super.f() + def wrong(n: int) -> int: + # error: [invalid-super-argument] + return super.f() + return inner(self) +``` + +## a lambda passes its own receiver + +A lambda reads its own first parameter too, not the method's, so one with no parameter has no +receiver to pass. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + # error: [unavailable-implicit-super-arguments] "has no parameter to read the receiver from" + g = lambda: super.f() + return 1 +``` + +## a comprehension has no receiver + +A comprehension runs as a function of its own, whose first argument is the iterator it walks, so it +has no receiver to pass either. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + # error: [unavailable-implicit-super-arguments] "in a comprehension" + return [super.f() for _ in range(1)][0] +``` + +## `super[T]` anywhere but an attribute's base is a subscript + +`super[T]` is the keyword only as an attribute's base. Anywhere else it is the subscript of the +name, as python reads it: of the builtin class, which takes no subscript. + +```by +class A: + def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + # error: [not-subscriptable] + pivot = super[A] + return 1 +``` + +## `super[T]` anywhere but an attribute's base reads the module's binding + +A module that binds the name `super` itself is subscripted as what it binds, while `super.f` stays +the keyword. + +```by +super = {1: "one"} + +class A: + def f(self) -> int: + return 1 + +class B(A): + def f(self) -> int: + reveal_type(super[1]) # revealed: str + return super.f() +``` + ## `super` in a property accessor An accessor is a method like any other, so `super` stays available there. @@ -146,3 +278,37 @@ class B(A): assert B().a == 2 ``` + +## `super` in a `NamedTuple` method + +A `NamedTuple` class cannot use `super()` in its methods, and the keyword is written as a `super()` +call, so it is reported the same way. + +```by +from typing import NamedTuple + +class Point(NamedTuple): + x: int + + def describe(self) -> str: + # error: [super-call-in-named-tuple-method] + return super.__repr__() +``` + +## an attribute of `super` is not assigned or deleted + +A `super()` object has no attributes of its own, and does not pass an assignment or a deletion on to +the class it reads past, so python raises `AttributeError` for either. The keyword is that object +wherever it is an attribute's base, so a write through it is reported as one through `super()` is. + +```by +class A: + x: int = 0 + +class B(A): + def f(self): + super.x = 1 # error: [invalid-assignment] "Cannot assign to attribute `x`" + super.x += 1 # error: [invalid-assignment] "Cannot assign to attribute `x`" + del super.x # error: [invalid-assignment] "Cannot delete attribute `x`" + del super().x # error: [invalid-assignment] "Cannot delete attribute `x`" +``` diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md index 53a1fa41fb..7487238713 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md @@ -834,6 +834,26 @@ def f(check: (x: object) -> (a is int)): reveal_type(a) # revealed: int ``` +## a narrowing return reads back as the guard it was written as + +`TypeIs[int]` does not say which place a predicate narrows, and an assertion returns `None`, so a +signature shows the guard its return annotation wrote — parenthesized in a callable type, as it is +written there. + +```by +def is_int_second(first: object, second: object) -> second is int: + return isinstance(second, int) + +def ensure(x: int | None, y: str | None) -> asserts x is int and not y: + if x is None or y: + raise ValueError() + +def f(check: (first: object, second: object) -> (second is int)): + reveal_type(is_int_second) # revealed: def is_int_second(first: object, second: object) -> second is int + reveal_type(check) # revealed: (first: object, second: object) -> (second is int) + reveal_type(ensure) # revealed: def ensure(x: int | None, y: str | None) -> asserts x is int and not y +``` + ## a callable narrowing another parameter is not such a callable @@ -1002,6 +1022,73 @@ class Sub(Base): return isinstance(second, int) ``` +## an implementation asserts what the protocol asserts + + + +A call through a protocol narrows what the protocol's member asserts, once the call returns, so a +class standing in for it has to assert the same thing about the same argument. Asserting it about +another argument leaves the one the caller relies on unchecked. + +```by +from typing import Protocol + +class Checker(Protocol): + def check(self, first: int | None, second: int | None) -> asserts first is int: ... + +class SecondChecker: + def check(self, first: int | None, second: int | None) -> asserts second is int: + if second is None: + raise ValueError() + +class Silent: + def check(self, first: int | None, second: int | None) -> None: + pass + +def use(checker: Checker): ... + +# error: [invalid-argument-type] +use(SecondChecker()) +# error: [invalid-argument-type] +use(Silent()) +``` + +An assertion at least as narrow is enough, and so is asserting more than the protocol does. + +```by +class FirstChecker: + def check(self, first: int | None, second: int | None) -> asserts first is int and second is int: + if first is None or second is None: + raise ValueError() + +class TruthyChecker: + def check(self, first: int | None, second: int | None) -> asserts first: + assert first + +use(FirstChecker()) +# error: [invalid-argument-type] +use(TruthyChecker()) +``` + +## a function assigned over a method asserts what the method asserts + +A function stored on a subclass under the method's name is an override like a `def` there, but it +brings its own assertions rather than taking on the method's, so they are compared. + +```by +class Base: + def check(self, first: int | None, second: int | None) -> asserts first is int: + if first is None: + raise ValueError() + +def check_second(self: object, first: int | None, second: int | None) -> asserts second is int: + if second is None: + raise ValueError() + +class Sub(Base): + check = check_second # error: [invalid-method-override] +``` + ## a callable type's predicate is parenthesized Unparenthesized, `is` takes the whole callable type. diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_type_modifiers.md b/crates/ty_python_semantic/resources/mdtest/basedpython_type_modifiers.md index 24950af389..291a2d35a2 100644 --- a/crates/ty_python_semantic/resources/mdtest/basedpython_type_modifiers.md +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_type_modifiers.md @@ -295,13 +295,17 @@ def f(a: literal str, b: final int, c: literal list[*]) -> None: ## modifiers in a type parameter's bound, constraints and default -A bound is a type expression like any other, and so is each element of a constraint tuple. +A bound is a type expression like any other, and so is each element of a tuple bound and each member +of a type mapping's constraints. ```by def bounded[T: literal str](t: T) -> T: return t -def constrained[T: (literal str, literal int)](t: T) -> T: +def in_a_tuple[T: (literal str, literal int)](t: T) -> T: + return t + +def constrained[T in (literal str, literal int)](t: T) -> T: return t def defaulted[T: literal int = literal int](t: T) -> T: @@ -365,6 +369,17 @@ def f(p: (m: literal str)) -> None: reveal_type(p.m) # revealed: LiteralString ``` +## a modifier in a typed dict literal field + +A typed dict literal's values are the types of its fields, so each carries a modifier as any other +type does, and so does the `**: T` type of its extra items. + +```by +def f(p: {"m": literal str, "n": final int, **: final str}) -> None: + reveal_type(p["m"]) # revealed: LiteralString + reveal_type(p["n"]) # revealed: final int +``` + ## a modifier on a `cast` target The right operand of a cast is the type being cast to, so it is a type expression. diff --git a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md index 8140a3d525..cd1666d1bd 100644 --- a/crates/ty_python_semantic/resources/mdtest/cycle/basic.md +++ b/crates/ty_python_semantic/resources/mdtest/cycle/basic.md @@ -1228,6 +1228,39 @@ def check(): reveal_type(rec()) # revealed: float ``` +## a float literal a recursive function recomputes from itself, called from the module body + +The answer does not depend on where checking starts. A call from the module body asks for `rec`'s +signature before anything has looked at `rec`'s body, so the body's own call to `rec` re-enters the +signature rather than the return type. While the signature is still being worked out it returns the +same marker the return type starts from, so the union still records that it is defined in terms of +itself, and its literals are given up after a few rounds. + +```by +from typing import reveal_type + +def rec(n: int): + return 0.1 if n == 0 else min(0.1, rec(n - 1) * 2) + +reveal_type(rec(3)) # revealed: float +``` + +## a float literal a recursive method recomputes from itself + +A method that calls itself through `self` reaches its own signature the same way, wherever checking +starts. + +```by +from typing import reveal_type + +class Doubler: + def rec(self, n: int): + return 0.1 if n == 0 else self.rec(n - 1) * 2 + +def check(): + reveal_type(Doubler().rec(3)) # revealed: float +``` + ## a complex literal a loop recomputes from itself Complex literals fold the same way and are held the same way, so they need the same bound. diff --git a/crates/ty_python_semantic/resources/mdtest/deprecated.md b/crates/ty_python_semantic/resources/mdtest/deprecated.md index b8d7c7a17c..b287ab74e9 100644 --- a/crates/ty_python_semantic/resources/mdtest/deprecated.md +++ b/crates/ty_python_semantic/resources/mdtest/deprecated.md @@ -997,10 +997,11 @@ Binding `super()` to a class instead returns the property object without invokin super(Child, Child).value ``` -Deleting an attribute on a `super()` object does not invoke the parent's property deleter. +Deleting an attribute on a `super()` object does not invoke the parent's property deleter: it raises +`AttributeError`, since a `super()` object has no attributes of its own to delete. ```py -del super(Child, Child()).value +del super(Child, Child()).value # error: [invalid-assignment] ``` The same holds when the receiver may be either a `super()` object or an ordinary instance. @@ -1011,7 +1012,7 @@ class Ordinary: def delete_union(flag: bool): target = super(Child, Child()) if flag else Ordinary() - del target.value + del target.value # error: [invalid-assignment] ``` ## Metaclass properties diff --git a/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md b/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md index f99572055e..13d1cc8be6 100644 --- a/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md +++ b/crates/ty_python_semantic/resources/mdtest/diagnostics/version_related_syntax_errors.md @@ -74,6 +74,75 @@ type Pair = tuple[int, int] assert first([1, 2]) == 1 ``` +### A custom string tag + +A string tag's template is written as a t-string, which python only has from 3.14, but the template +is lowered for any target: + +```toml +[environment] +python-version = "3.10" +``` + +```by +def tagged(template: object) -> object: + return template + +greeting = tagged"hello {1 + 1}" +``` + +### A bracketless `except` + +The types of an `except` clause that leaves out their parentheses are written as the parenthesized +tuple they are, which every version parses: + +```toml +[environment] +python-version = "3.13" +``` + +```by +def parse(text: str) -> int: + try: + return int(text) + except ValueError, TypeError: + return 0 + +assert parse("x") == 0 +``` + +### A template literal type + +An f-string in a type expression is a template literal type, which is written as the `str` or +`Literal[...]` it means, so none of the f-string's own syntax reaches the python. It may reuse its +quote inside a replacement field before 3.12: + +```toml +[environment] +python-version = "3.11" +``` + +```by +def greet(greeting: f"{"hi" | "hello"} {str}") -> str: + return greeting + +assert greet("hi there") == "hi there" +``` + +### An f-string the python keeps + +An f-string value is written into the python as it stands, so it reuses its quote only from 3.12: + +```toml +[environment] +python-version = "3.11" +``` + +```by +name = "world" +greeting = f"{"hello"} {name}" # error: [invalid-syntax] "Cannot reuse outer quote character in f-strings on Python 3.11 (syntax was added in Python 3.12)" +``` + ### A `match` statement A `match` statement is lowered to a chain of `if` statements whose tests bind what the patterns @@ -116,46 +185,23 @@ def describe(x: int) -> str: ### A destructuring pattern A destructuring pattern — `let P := v`, an `if let` clause, or a pattern written as a `for` or -`with` target or as a parameter — is written as a `match` statement, so it runs wherever one does. -From 3.8 that is the lowered `match`: +`with` target or as a parameter — is written as a `match` statement, but is not lowered where a +`match` statement is. It needs python 3.10 whatever the target: ```toml [environment] -python-version = "3.8" -``` - -```by -def total(pair: tuple[int, int] | None) -> int: - if let (a, b) := pair: - return a + b - return 0 - -def first(pair: tuple[int, int]) -> int: - let (a, _) := pair - return a - -assert total((1, 2)) == 3 -assert first((4, 5)) == 4 -``` - -### A destructuring pattern before 3.8 - -Before 3.8 no `match` can be written, so neither can a destructuring pattern: - -```toml -[environment] -python-version = "3.7" +python-version = "3.9" ``` ```by def total(pair: tuple[int, int] | None) -> int: - # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.9 (syntax was added in Python 3.10)" if let (a, b) := pair: return a + b return 0 def first(pair: tuple[int, int]) -> int: - # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.9 (syntax was added in Python 3.10)" let (a, _) := pair return a @@ -168,8 +214,12 @@ class Point: def sum_all(points: list[Point]) -> int: n = 0 - # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.7 (syntax was added in Python 3.10)" + # error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.9 (syntax was added in Python 3.10)" for Point(x, y) in points: n += x + y return n + +# error: [invalid-syntax] "Cannot use a destructuring pattern, which is written as a `match` statement, on Python 3.9 (syntax was added in Python 3.10)" +def norm(Point(x, y): Point) -> int: + return x + y ``` diff --git a/crates/ty_python_semantic/resources/mdtest/literal/float.md b/crates/ty_python_semantic/resources/mdtest/literal/float.md index 1580c1a4b5..f6173b00f9 100644 --- a/crates/ty_python_semantic/resources/mdtest/literal/float.md +++ b/crates/ty_python_semantic/resources/mdtest/literal/float.md @@ -59,7 +59,7 @@ def f(nan: float.nan) -> None: ```by def f() -> float.inf: - raise NotImplementedError + return float("inf") reveal_type(f()) # revealed: inf ``` diff --git a/crates/ty_python_semantic/resources/mdtest/protocols.md b/crates/ty_python_semantic/resources/mdtest/protocols.md index 23a5b08719..77ca5fada8 100644 --- a/crates/ty_python_semantic/resources/mdtest/protocols.md +++ b/crates/ty_python_semantic/resources/mdtest/protocols.md @@ -8103,7 +8103,7 @@ protocol Named(Sized): name: str class Person: - name: str + name: str = "Ada" def __len__(self) -> int: return 0 diff --git a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md index 9b2f576c73..c2eaefe6ff 100644 --- a/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md +++ b/crates/ty_python_semantic/resources/mdtest/repeated_underscore.md @@ -5,8 +5,10 @@ name is `_`, the parameters are allowed. Python itself refuses a repeated parame transpiler gives every `_` after the first a name of its own (`_2`, `_3`, ...) that the module spells nowhere. Those names are the transpiler's rather than the author's, so a call cannot spell them: the numbered parameters are positional-only, and the signature ty checks a call against is the -one the transpiled python has. The signature shows each of them as the `_` the author wrote. -References to `_` inside the body resolve to the first parameter. +one the transpiled python has. The signature shows each of them as the `_` the author wrote. Reading +`_` where it may be one of them is an error, since python and ty would each read a different one. +Writing `_` is checked against none of their annotations, since none of them is the `_` the body +binds. A module that spells `_2` anywhere has its `_`s numbered past it, so the examples below that pass a numbered name by keyword declare the function in a module of its own. @@ -15,7 +17,7 @@ numbered name by keyword declare the function in a module of its own. ```by def f(_, _): - reveal_type(_) # revealed: _@f + return 1 def g(_, _, _, x: int): reveal_type(x) # revealed: int @@ -285,6 +287,293 @@ python-version = "3.7" def pair(_: int, _: str) -> None: ... ``` +## reading a repeated `_` is an error + +Python binds `_` to only one of the parameters, the lowering keeping the first `_` under that name, +while every other `_` shadows the one before it as ty reads the body. A read of `_` would mean the +first parameter to python and the last to ty, so it is an error rather than either: + +```by +def f(_: int, _: str) -> str: + # error: [invalid-repeated-underscore] "reading a repeated `_` parameter of `f`" + return _ +``` + +Having no one meaning, the read has no one type either, so it is typed `Unknown` rather than as the +last `_`: + +```by +def g(_: int, _: str) -> None: + # error: [invalid-repeated-underscore] + reveal_type(_) # revealed: Unknown +``` + +## a single `_` is read as usual + +A parameter list that names one parameter `_` binds it like any other parameter, and a module's own +`_` is a name like any other too: + +```by +def f(_: int) -> int: + return _ + +_ = 2 + +assert f(1) == 1 +assert _ == 2 +``` + +## a write to `_` is checked against no parameter + +The lowering gives every `_` a parameter name of its own, so none of their annotations declares the +`_` the body binds. A write to `_` is an assignment to an undeclared local, and a read that only it +reaches sees the value written: + +```by +def f(_: int, _: str) -> int: + _ = 5 + reveal_type(_) # revealed: 5 + return _ + +assert f(1, "a") == 5 +``` + +A starred `_` declares nothing either, though its annotation gives the parameter a tuple type: + +```by +def rest(_: int, *_: str) -> int: + _ = 5 + return _ + +assert rest(1, "a", "b") == 5 +``` + +## every binding of `_` is such a write + +Unpacking, a `for` loop, an assignment expression and a `with` statement all bind `_` without a +declared type to check it against: + +```by +import tempfile + +def unpack(_: int, _: str) -> int: + a, _ = 1, 2 + return a + _ + +def loop(_: int, _: str) -> int: + total = 0 + for _ in range(3): + total += _ + return total + +def walrus(_: int, _: str) -> int: + if (_ := 3) > 0: + return _ + return 0 + +def with_(_: int, _: str) -> bool: + with tempfile.TemporaryFile() as _: + return _.closed + +assert unpack(1, "a") == 3 +assert loop(1, "a") == 3 +assert walrus(1, "a") == 3 +assert not with_(1, "a") +``` + +A lambda's `_`s declare nothing, whether or not the lambda is typed: + +```by +untyped = lambda _, _: (_ := 9) +typed = lambda (_: int, _: str) -> int: (_ := 9) + +assert untyped(1, "a") == 9 +assert typed(1, "a") == 9 +``` + +So a nested function that declares `nonlocal _` writes it unchecked too: + +```by +def outer(_: int, _: str) -> int: + def inner() -> int: + nonlocal _ + _ = 7 + return _ + + return inner() + +assert outer(1, "a") == 7 +``` + +## an annotated `_` declares it + +An annotation the body writes for `_` is the one declaration of the name, as it is for any local, so +a write after it is checked against it: + +```by +def f(_: int, _: str) -> bytes: + _: bytes = b"a" + # error: [invalid-assignment] "Object of type `5` is not assignable to `bytes`" + _ = 5 + return _ +``` + +## a read that only a rebinding of `_` reaches + +The body may bind `_` itself. A read that only that binding reaches reads it, whatever the +parameters are: + +```by +def f(_: int, _: int) -> int: + _ = 5 + return _ + +assert f(1, 2) == 5 +``` + +## a read that one path from the parameters still reaches + +A rebinding on some paths leaves the parameters reaching the read on the others, so the read is an +error: + +```by +def f(_: int, _: int, flag: bool) -> int: + if flag: + _ = 5 + # error: [invalid-repeated-underscore] + return _ +``` + +## an augmented assignment reads `_` + +`_ += 1` reads `_` before it binds it: + +```by +def f(_: int, _: int) -> None: + # error: [invalid-repeated-underscore] + _ += 1 +``` + +The refused read is typed `Unknown`, so the operator is not checked against the last `_` as well: a +`str` has no `+` with an `int`, but the read is the one error here: + +```by +def g(_: int, _: str) -> None: + # error: [invalid-repeated-underscore] + _ += 1 +``` + +## `del _` reads no value + +`del _` unbinds the name rather than reading its value, so whichever parameter python and ty take it +to be, both are left with `_` unbound: + +```by +def f(_: int, _: str) -> None: + del _ + +f(1, "a") +``` + +## a nested scope that reads it + +A nested function, a lambda and a comprehension read a name they do not bind from the function +around them, so each read of `_` there is an error too: + +```by +def f(_: int, _: str) -> None: + def g() -> str: + # error: [invalid-repeated-underscore] "reading a repeated `_` parameter of `f`" + return _ + + # error: [invalid-repeated-underscore] + h = lambda: _ + + # error: [invalid-repeated-underscore] + xs = [_ for x in range(3)] +``` + +A nested function reads the name when it is called, by which point the function around it may have +bound `_` again, so its read is an error wherever a binding of the parameters remains: + +```by +def later(_: int, _: int) -> None: + def g() -> int: + # error: [invalid-repeated-underscore] + return _ + _ = 5 +``` + +## a nested scope that binds `_` itself + +A nested scope that binds `_` reads its own: + +```by +def f(_: int, _: str) -> list[int]: + def g(_: bytes) -> bytes: + return _ + + assert g(b"a") == b"a" + return [_ for _ in range(3)] + +assert f(1, "a") == [0, 1, 2] +``` + +## `nonlocal _` and `global _` + +A nested function that declares `nonlocal _` reads the parameters, unless it binds `_` on every path +to the read first: + +```by +def f(_: int, _: int) -> None: + def reads() -> int: + nonlocal _ + # error: [invalid-repeated-underscore] + return _ + + def rebinds() -> int: + nonlocal _ + _ = 5 + return _ +``` + +One that declares `global _` reads the module's `_`: + +```by +_ = 5 + +def g(_: int, _: str) -> int: + def module() -> int: + global _ + return _ + + return module() + +assert g(1, "a") == 5 +``` + +## a read the parser writes + +`init(let _: int, let _: int)` stands for a `self._ = _` for each parameter, which the author never +wrote. The lowering writes each of them reading its own parameter, by the name it gives that +parameter, so neither is a read of a repeated `_`: + +```by +class Pair: + init(let _: int, let _: int) + +assert Pair(1, 2)._ == 2 +``` + +## a lambda's own repeated `_` + +A lambda's parameters are read by the same rule: + +```by +# error: [invalid-repeated-underscore] "reading a repeated `_` parameter of a lambda" +first = lambda _, _: _ +``` + ## an override takes the base method's names A method that overrides one takes the names the overridden method gives those positions, so a call @@ -341,27 +630,32 @@ Override().f() ## an override of a method whose `_`s are numbered A base that repeats `_` numbers its own. The first is `_` in the base, and the override's `_` in -that position takes that name as it takes any other, so a read of `_` in the override's body is its -first parameter, as it is in any definition that repeats `_`. The numbered ones are positional-only, -so no call passes one by name, and the override numbers its own there: +that position takes that name as it takes any other. The numbered ones are positional-only, so no +call passes one by name, and the override numbers its own there: ```by class Base: def f(self, _: int, _: int) -> int: - return _ + return 1 class Override(Base): override def f(self, _: int, _: int) -> int: - return _ * 10 + return 10 reveal_type(Override.f) # revealed: def f(self, _: int, _: int, /) -> int assert Override().f(1, 2) == 10 ``` +## a variadic `_` in an override of a method whose `_`s are numbered + A variadic `_` has no counterpart in the base, and is numbered after the names the others take: ```by +class Base: + def f(self, _: int, _: int) -> int: + return 1 + class Variadic(Base): override def f(self, _: int, _: int, *_: int) -> int: return 20 @@ -417,7 +711,7 @@ class Middle(Base): class Override(Middle): override def f(self, _: int, _: int) -> int: - return _ * 30 + return 30 reveal_type(Override.f) # revealed: def f(self, x: int, y: int) -> int @@ -436,11 +730,11 @@ class Base: class Override(Base): override def f(self, _: int, _: int) -> int: - return _ + return 35 reveal_type(Override.f) # revealed: def f(self, x: int, _: int) -> int -assert Override().f(1, _=2) == 1 +assert Override().f(1, _=2) == 35 ``` ## an override of a base in another module @@ -460,7 +754,7 @@ from base import Base class Override(Base): override def f(self, _: int, _: int) -> int: - return _ * 40 + return 40 reveal_type(Override.f) # revealed: def f(self, x: int, y: int) -> int @@ -478,7 +772,7 @@ class Base[T]: class Specialized(Base[int]): override def f(self, _: int, _: int) -> int: - return _ * 50 + return 50 class Generic[T](Base[T]): override def f(self, _: T, _: T) -> int: @@ -534,3 +828,37 @@ class Override(Base): override def f(self, _: int, _: int) -> int: return x ``` + +## an override's body + +An override's `_`s take the names of the base's parameters, but the body still spells them all `_`, +so a read of `_` there is an error as it is anywhere else: + +```by +class Base: + def f(self, x: int, y: str) -> str: + return y + +class Override(Base): + override def f(self, _: int, _: str) -> str: + # error: [invalid-repeated-underscore] + return _ +``` + +## `del _` in an override's body + +`del _` reads no value in an override's body either, though its `_`s take the names of the base's +parameters: + +```by +class Base: + def f(self, x: int, y: str) -> str: + return y + +class Deletes(Base): + override def f(self, _: int, _: str) -> str: + del _ + return "deleted" + +assert Deletes().f(1, "a") == "deleted" +``` diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" index dd3b8f7ab4..c342afb6b0 100644 --- "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_a_callable_narrowing\342\200\246_(831b7fe0c360003b).snap" @@ -39,7 +39,7 @@ error[invalid-argument-type]: Argument to function `want_second` is incorrect --> src/mdtest_snippet.by:11:13 | 11 | want_second(is_int_first) - | ^^^^^^^^^^^^ Expected `(first: object, second: object) -> TypeIs[int]`, found `def is_int_first(first: object, second: object) -> TypeIs[int]` + | ^^^^^^^^^^^^ Expected `(first: object, second: object) -> (second is int)`, found `def is_int_first(first: object, second: object) -> first is int` info: the return type narrows `first`, where `second` is expected info: Function defined here --> src/mdtest_snippet.by:1:5 @@ -54,7 +54,7 @@ error[invalid-argument-type]: Argument to function `want_first` is incorrect --> src/mdtest_snippet.by:17:16 | 17 | want_first(second_checker) - | ^^^^^^^^^^^^^^ Expected `(first: object, second: object) -> TypeIs[int]`, found `(first: object, second: object) -> TypeIs[int]` + | ^^^^^^^^^^^^^^ Expected `(first: object, second: object) -> (first is int)`, found `(first: object, second: object) -> (second is int)` info: the return type narrows `second`, where `first` is expected info: Function defined here --> src/mdtest_snippet.by:12:5 diff --git "a/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_an_implementation_as\342\200\246_(b79e5a772d01ad8a).snap" "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_an_implementation_as\342\200\246_(b79e5a772d01ad8a).snap" new file mode 100644 index 0000000000..2c34d97a89 --- /dev/null +++ "b/crates/ty_python_semantic/resources/mdtest/snapshots/basedpython_type_gua\342\200\246_-_basedpython___narrowi\342\200\246_-_an_implementation_as\342\200\246_(b79e5a772d01ad8a).snap" @@ -0,0 +1,101 @@ +--- +source: crates/mdtest/src/lib.rs +expression: snapshot +--- + +--- +mdtest name: basedpython_type_guards.md - basedpython: narrowing return annotations - an implementation asserts what the protocol asserts +mdtest path: crates/ty_python_semantic/resources/mdtest/basedpython_type_guards.md +--- + +# Source files + +## mdtest_snippet.by + +``` + 1 | from typing import Protocol + 2 | + 3 | class Checker(Protocol): + 4 | def check(self, first: int | None, second: int | None) -> asserts first is int: ... + 5 | + 6 | class SecondChecker: + 7 | def check(self, first: int | None, second: int | None) -> asserts second is int: + 8 | if second is None: + 9 | raise ValueError() +10 | +11 | class Silent: +12 | def check(self, first: int | None, second: int | None) -> None: +13 | pass +14 | +15 | def use(checker: Checker): ... +16 | +17 | # error: [invalid-argument-type] +18 | use(SecondChecker()) +19 | # error: [invalid-argument-type] +20 | use(Silent()) +21 | class FirstChecker: +22 | def check(self, first: int | None, second: int | None) -> asserts first is int and second is int: +23 | if first is None or second is None: +24 | raise ValueError() +25 | +26 | class TruthyChecker: +27 | def check(self, first: int | None, second: int | None) -> asserts first: +28 | assert first +29 | +30 | use(FirstChecker()) +31 | # error: [invalid-argument-type] +32 | use(TruthyChecker()) +``` + +# Diagnostics + +``` +error[invalid-argument-type]: Argument to function `use` is incorrect + --> src/mdtest_snippet.by:18:5 + | +18 | use(SecondChecker()) + | ^^^^^^^^^^^^^^^ Expected `Checker`, found `final SecondChecker` +info: type `SecondChecker` is not assignable to protocol `Checker` +info: └── protocol member `check` is incompatible +info: └── the function asserts nothing about `first`, where `int` is expected +info: Function defined here + --> src/mdtest_snippet.by:15:5 + | +15 | def use(checker: Checker): ... + | ^^^ ---------------- Parameter declared here + +``` + +``` +error[invalid-argument-type]: Argument to function `use` is incorrect + --> src/mdtest_snippet.by:20:5 + | +20 | use(Silent()) + | ^^^^^^^^ Expected `Checker`, found `final Silent` +info: type `Silent` is not assignable to protocol `Checker` +info: └── protocol member `check` is incompatible +info: └── the function asserts nothing about `first`, where `int` is expected +info: Function defined here + --> src/mdtest_snippet.by:15:5 + | +15 | def use(checker: Checker): ... + | ^^^ ---------------- Parameter declared here + +``` + +``` +error[invalid-argument-type]: Argument to function `use` is incorrect + --> src/mdtest_snippet.by:32:5 + | +32 | use(TruthyChecker()) + | ^^^^^^^^^^^^^^^ Expected `Checker`, found `final TruthyChecker` +info: type `TruthyChecker` is not assignable to protocol `Checker` +info: └── protocol member `check` is incompatible +info: └── the function asserts `first` is truthy, where `int` is expected +info: Function defined here + --> src/mdtest_snippet.by:15:5 + | +15 | def use(checker: Checker): ... + | ^^^ ---------------- Parameter declared here + +``` diff --git a/crates/ty_python_semantic/resources/mdtest/typed_dict_literal.md b/crates/ty_python_semantic/resources/mdtest/typed_dict_literal.md index 7ff492b9ed..4f703a1777 100644 --- a/crates/ty_python_semantic/resources/mdtest/typed_dict_literal.md +++ b/crates/ty_python_semantic/resources/mdtest/typed_dict_literal.md @@ -41,15 +41,35 @@ def h(x: {"only": int}) -> None: reveal_type(x["only"]) # revealed: int ``` +### Closed by default + +A dict literal type is closed, as the `TypedDict` it lowers to is (`closed=True`): it has no keys +but the ones it names, so a value with another key is rejected. + +```by +def f(name: str) -> None: + a: {"name": str} = {"name": name, "age": 1} # error: [invalid-key] "Unknown key "age"" +``` + +A closed type is not a supertype of a type with more keys, since the extra key would be lost. + +```by +def g(wider: {"name": str, "age": int}) -> None: + narrower: {"name": str} = wider # error: [invalid-assignment] +``` + ### Extra-items marker `**: T` -A `**: T` entry in a dict literal type lowers to `extra_items=T` on the synthesized `TypedDict`. -ty's extra-items semantics are still TODO, so the declared fields still type-check but extra keys -aren't yet enforced. +A `**: T` entry in a dict literal type lowers to `extra_items=T` on the synthesized `TypedDict`: a +key it does not name is allowed, with a value of type `T`. ```by -def f(x: {"name": str, **: int}) -> None: - reveal_type(x["name"]) # revealed: str +def f(name: str) -> None: + a: {"name": str, **: int} = {"name": name, "age": 1} + reveal_type(a) # revealed: {"name": str, **: int} + reveal_type(a["name"]) # revealed: str + + b: {"name": str, **: int} = {"name": name, "age": "x"} # error: [invalid-argument-type] ``` ## Display diff --git a/crates/ty_python_semantic/resources/mdtest/typeof.md b/crates/ty_python_semantic/resources/mdtest/typeof.md index 2d5ec10057..d6246951e2 100644 --- a/crates/ty_python_semantic/resources/mdtest/typeof.md +++ b/crates/ty_python_semantic/resources/mdtest/typeof.md @@ -17,7 +17,7 @@ reveal_type(a) # revealed: 1 ```by class C: - x: str + x: str = "" c: C = C() diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index f058eb8d17..84c29e5161 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -22,6 +22,10 @@ use ruff_db::diagnostic::{Annotation, Diagnostic, DiagnosticId, Severity, Span}; use ruff_db::files::File; use ruff_db::parsed::parsed_module; use ruff_db::source::{SourceTextError, source_text}; +use ruff_python_ast as ast; +use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, walk_expr}; +use ruff_python_parser::{UnsupportedSyntaxError, UnsupportedSyntaxErrorKind}; +use ruff_text_size::TextRange; use rustc_hash::FxHasher; pub use semantic_model::{ Completion, DjangoLookupArgument, ExpectedStringLiteralCompletion, ExtensionOperatorRewrite, @@ -519,24 +523,86 @@ fn check_file_inner( .map(|error| Diagnostic::invalid_syntax(source_file, &error.error, error)), ); - diagnostics.extend(parsed_ref.unsupported_syntax_errors().iter().map(|error| { - let mut error = Diagnostic::invalid_syntax(source_file, error, error); - add_inferred_python_version_hint_to_diagnostic( - db, - source_file, - &mut error, - "parsing syntax", - ); - error - })); + let (type_diagnostics, template_literal_types) = check_types_with(db, file, external); - diagnostics.extend(check_types_with(db, file, external)); + let lowered_fstrings = outermost_template_literal_types( + parsed_ref.syntax(), + parsed_ref.unsupported_syntax_errors(), + &template_literal_types, + ); + diagnostics.extend( + parsed_ref + .unsupported_syntax_errors() + .iter() + .filter(|error| !is_written_in(error, &lowered_fstrings)) + .map(|error| { + let mut error = Diagnostic::invalid_syntax(source_file, error, error); + add_inferred_python_version_hint_to_diagnostic( + db, + source_file, + &mut error, + "parsing syntax", + ); + error + }), + ); + + diagnostics.extend(type_diagnostics); diagnostics.sort_unstable_by(|a, b| a.rendering_sort_key(db).cmp(&b.rendering_sort_key(db))); Ok(diagnostics.into_boxed_slice()) } +/// basedpython: the f-strings standing outside any other f-string that are template literal +/// types, when one holds a pep 701 f-string error +/// +/// which syntax the lowering rewrites is the parser's to say, in +/// `UnsupportedSyntaxErrorKind::is_lowered_by_basedpython`, and it holds a pep 701 f-string to +/// python 3.12. a template literal type is the exception, which only the checker can tell apart: +/// the lowering writes the `str` or `Literal[...]` it means in place of the whole f-string, so +/// none of the f-string's syntax reaches the python. an f-string nested in one goes with it, and +/// one nested in an f-string the python keeps is kept, so the outermost f-string decides +fn outermost_template_literal_types( + module: &ast::ModModule, + errors: &[UnsupportedSyntaxError], + template_literal_types: &[TextRange], +) -> Vec { + struct Outermost(Vec); + + impl<'a> SourceOrderVisitor<'a> for Outermost { + fn visit_expr(&mut self, expr: &'a ast::Expr) { + if let ast::Expr::FString(fstring) = expr { + self.0.push(fstring.range); + } else { + walk_expr(self, expr); + } + } + } + + if template_literal_types.is_empty() + || !errors + .iter() + .any(|error| matches!(error.kind, UnsupportedSyntaxErrorKind::Pep701FString(_))) + { + return Vec::new(); + } + let mut outermost = Outermost(Vec::new()); + outermost.visit_body(&module.body); + outermost + .0 + .retain(|range| template_literal_types.contains(range)); + outermost.0 +} + +/// whether `error` is a pep 701 f-string error written in one of `fstrings` +fn is_written_in(error: &UnsupportedSyntaxError, fstrings: &[TextRange]) -> bool { + matches!(error.kind, UnsupportedSyntaxErrorKind::Pep701FString(_)) + && fstrings + .iter() + .any(|fstring| fstring.contains_range(error.range)) +} + #[derive(Debug, Clone, get_size2::GetSize)] pub struct IOErrorDiagnostic { file: File, diff --git a/crates/ty_python_semantic/src/semantic_model.rs b/crates/ty_python_semantic/src/semantic_model.rs index 0c74355d2a..0e366f4576 100644 --- a/crates/ty_python_semantic/src/semantic_model.rs +++ b/crates/ty_python_semantic/src/semantic_model.rs @@ -158,6 +158,14 @@ impl<'db> SemanticModel<'db> { crate::types::visibility::reads_class_scope_binding(self.db, self.file, reference) } + /// basedpython: see `crate::types::visibility::name_resolution` + pub fn name_resolution( + &self, + reference: &ast::ExprName, + ) -> Option { + crate::types::visibility::name_resolution(self.db, self.file, reference) + } + /// basedpython: the module-level names this file declares `private` — the /// set `private-import` reads, and whose references the lowering renames pub fn private_module_symbols(&self) -> Vec { diff --git a/crates/ty_python_semantic/src/types.rs b/crates/ty_python_semantic/src/types.rs index 1e1bd4dc78..7473b5e07b 100644 --- a/crates/ty_python_semantic/src/types.rs +++ b/crates/ty_python_semantic/src/types.rs @@ -19,7 +19,7 @@ use ruff_db::parsed::parsed_module; use ruff_python_ast as ast; use ruff_python_ast::helpers::{MemberVisibility, TypeModifier}; use ruff_python_ast::name::Name; -use ruff_text_size::Ranged; +use ruff_text_size::{Ranged, TextRange}; use smallvec::smallvec_inline; use ty_module_resolver::{ ImportingFile, KnownModule, Module, ModuleName, file_to_module, resolve_module, @@ -71,6 +71,7 @@ pub(crate) use self::signatures::Signature; pub use self::signatures::{ParameterDefault, ParameterKind}; pub(crate) use self::subclass_of::{SubclassOfInner, SubclassOfType}; pub(crate) use self::type_expansion::expand_type; +pub use self::visibility::NameResolution; pub(crate) use crate::diagnostic::add_inferred_python_version_hint_to_diagnostic; use crate::django_settings; use crate::place::{ @@ -279,10 +280,11 @@ mod property_tests; pub(crate) mod subscript; pub fn check_types(db: &dyn Db, file: ProgramFile<'_>) -> Vec { - check_types_with(db, file, Vec::new()) + check_types_with(db, file, Vec::new()).0 } -/// [`check_types`], with lint diagnostics worked out elsewhere folded in. +/// [`check_types`], with lint diagnostics worked out elsewhere folded in, and the range of +/// every f-string inferred as a template literal type. /// /// See [`crate::check_file_with`] for what `external` is and why its suppressions /// are applied here rather than where it was computed. @@ -290,7 +292,7 @@ pub(crate) fn check_types_with( db: &dyn Db, file: ProgramFile<'_>, external: Vec, -) -> Vec { +) -> (Vec, Vec) { let source_file = file.file(db); let _span = tracing::trace_span!("check_types", ?source_file).entered(); tracing::debug!("Checking file '{path}'", path = source_file.path(db)); @@ -326,6 +328,7 @@ pub(crate) fn check_types_with( report_external(db, source_file, external, &mut diagnostics); + let template_literal_types = diagnostics.template_literal_types().to_vec(); let diagnostics = check_suppressions(db, file.python_file(db), diagnostics); let elapsed = start.elapsed(); @@ -336,7 +339,7 @@ pub(crate) fn check_types_with( ); } - diagnostics + (diagnostics, template_literal_types) } /// Apply `file`'s suppression comments to diagnostics computed outside this crate. diff --git a/crates/ty_python_semantic/src/types/bound_super.rs b/crates/ty_python_semantic/src/types/bound_super.rs index 66ced38cad..6ac959c545 100644 --- a/crates/ty_python_semantic/src/types/bound_super.rs +++ b/crates/ty_python_semantic/src/types/bound_super.rs @@ -3,7 +3,10 @@ use crate::ProgramEnvironment; use itertools::{Either, Itertools}; use ruff_db::diagnostic::Diagnostic; +use ruff_db::parsed::ParsedModuleRef; use ruff_python_ast::{AnyNodeRef, name::Name}; +use ty_python_core::SemanticIndex; +use ty_python_core::scope::{NodeWithScopeKind, ScopeId}; use crate::{ Db, DisplaySettings, @@ -11,11 +14,15 @@ use crate::{ types::{ BoundTypeVarInstance, ClassBase, ClassType, DivergentType, DynamicType, IntersectionBuilder, KnownClass, MemberLookupErrorKind, MemberLookupPolicy, - MemberLookupResult, SpecialFormType, SubclassOfInner, SubclassOfType, Type, - TypeVarBoundOrConstraints, UnionBuilder, UnsafeUnionType, + MemberLookupResult, SpecialFormType, StaticClassLiteral, SubclassOfInner, SubclassOfType, + Type, TypeVarBoundOrConstraints, UnionBuilder, UnsafeUnionType, binding_type, + class::CodeGeneratorKind, constraints::ConstraintSet, context::InferContext, - diagnostic::{INVALID_SUPER_ARGUMENT, UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS}, + diagnostic::{ + INVALID_SUPER_ARGUMENT, SUPER_CALL_IN_NAMED_TUPLE_METHOD, + UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, + }, member_lookup_result, relation::EquivalenceChecker, signatures::{Parameter, Parameters, Signature}, @@ -24,6 +31,54 @@ use crate::{ }, }; +/// Report `super` used in a method of `enclosing_class` when that is a `NamedTuple`, whose methods +/// cannot use it, and return whether it was one. `super()` with no arguments or with two is +/// refused there, and so is the basedpython `super` keyword, which is written as one of them. +pub(crate) fn report_super_in_named_tuple_method<'db>( + context: &InferContext<'db, '_>, + enclosing_class: StaticClassLiteral<'db>, + node: AnyNodeRef, +) -> bool { + let db = context.db(); + if !CodeGeneratorKind::NamedTuple.matches(db, enclosing_class.into()) { + return false; + } + if let Some(builder) = context.report_lint(&SUPER_CALL_IN_NAMED_TUPLE_METHOD, node) { + builder.into_diagnostic(format_args!( + "Cannot use `super()` in a method of NamedTuple class `{}`", + enclosing_class.name(db) + )); + } + true +} + +/// The receiver zero-argument `super()` reads where it is written in `scope`: the first parameter +/// of the function or lambda `scope` is, as python takes the first argument of the frame it runs +/// in. `None` where `scope` is no function — a class body, a module, a comprehension — or one +/// with no parameter. +pub(crate) fn implicit_super_receiver<'db>( + db: &'db dyn Db, + index: &SemanticIndex<'db>, + scope: ScopeId<'db>, + module: &ParsedModuleRef, +) -> Option> { + let first_param = match scope.node(db) { + NodeWithScopeKind::Function(function) => function.node(module).parameters.iter().next(), + NodeWithScopeKind::Lambda(lambda) => lambda + .node(module) + .parameters + .as_ref() + .into_iter() + .flatten() + .next(), + _ => None, + }?; + Some(binding_type( + db, + index.expect_single_definition(first_param), + )) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TypeVarOwnerContext<'db> { Bare(BoundTypeVarInstance<'db>), diff --git a/crates/ty_python_semantic/src/types/class/known.rs b/crates/ty_python_semantic/src/types/class/known.rs index 53f6088174..f7bf97634a 100644 --- a/crates/ty_python_semantic/src/types/class/known.rs +++ b/crates/ty_python_semantic/src/types/class/known.rs @@ -3,12 +3,13 @@ use crate::{ place::{DefinedPlace, Definedness, Place, known_module_symbol}, types::{ Binding, ClassLiteral, ClassType, GenericContext, KnownInstanceType, StaticClassLiteral, - SubclassOfType, Type, binding_type, - bound_super::{BoundSuperError, BoundSuperType}, - class::CodeGeneratorKind, + SubclassOfType, Type, + bound_super::{ + BoundSuperError, BoundSuperType, implicit_super_receiver, + report_super_in_named_tuple_method, + }, constraints::{ConstraintSet, ConstraintSetBuilder}, context::InferContext, - diagnostic::SUPER_CALL_IN_NAMED_TUPLE_METHOD, infer::nearest_enclosing_class, known_instance::DeprecatedInstance, }, @@ -21,7 +22,7 @@ use std::{ sync::{LazyLock, Mutex}, }; use ty_module_resolver::{ImportingFile, KnownModule, file_to_module}; -use ty_python_core::{SemanticIndex, Truthiness, scope::NodeWithScopeKind}; +use ty_python_core::{SemanticIndex, Truthiness}; /// Non-exhaustive enumeration of known classes (e.g. `builtins.int`, `typing.Any`, ...) to allow /// for easier syntax when interacting with very common classes. @@ -2309,48 +2310,23 @@ impl KnownClass { return; }; - // Check if the enclosing class is a `NamedTuple`, which forbids the use of `super()`. - if CodeGeneratorKind::NamedTuple - .matches(context.db(), enclosing_class.into()) - { - if let Some(builder) = context - .report_lint(&SUPER_CALL_IN_NAMED_TUPLE_METHOD, call_expression) - { - builder.into_diagnostic(format_args!( - "Cannot use `super()` in a method of NamedTuple class `{}`", - enclosing_class.name(db) - )); - } + if report_super_in_named_tuple_method( + context, + enclosing_class, + call_expression.into(), + ) { overload.set_return_type(Type::unknown()); return; } - // The type of the first parameter if the given scope is function-like (i.e. function or lambda). - // `None` if the scope is not function-like, or has no parameters. - let first_param = match scope.node(db) { - NodeWithScopeKind::Function(f) => { - f.node(module).parameters.iter().next() - } - NodeWithScopeKind::Lambda(l) => l - .node(module) - .parameters - .as_ref() - .into_iter() - .flatten() - .next(), - _ => None, - }; - - let Some(first_param) = first_param else { + let Some(first_param) = implicit_super_receiver(db, index, scope, module) + else { BoundSuperError::UnavailableImplicitArguments .report_diagnostic(context, call_expression.into()); overload.set_return_type(Type::unknown()); return; }; - let definition = index.expect_single_definition(first_param); - let first_param = binding_type(context.db(), definition); - let bound_super = BoundSuperType::build( db, context.program_environment(), @@ -2365,24 +2341,16 @@ impl KnownClass { overload.set_return_type(bound_super); } [Some(pivot_class_type), Some(owner_type)] => { - // Check if the enclosing class is a `NamedTuple`, which forbids the use of `super()`. if let Some(enclosing_class) = nearest_enclosing_class(context.db(), index, scope) + && report_super_in_named_tuple_method( + context, + enclosing_class, + call_expression.into(), + ) { - if CodeGeneratorKind::NamedTuple - .matches(context.db(), enclosing_class.into()) - { - if let Some(builder) = context - .report_lint(&SUPER_CALL_IN_NAMED_TUPLE_METHOD, call_expression) - { - builder.into_diagnostic(format_args!( - "Cannot use `super()` in a method of NamedTuple class `{}`", - enclosing_class.name(db) - )); - } - overload.set_return_type(Type::unknown()); - return; - } + overload.set_return_type(Type::unknown()); + return; } let bound_super = BoundSuperType::build( diff --git a/crates/ty_python_semantic/src/types/class/typed_dict.rs b/crates/ty_python_semantic/src/types/class/typed_dict.rs index 27b97d43b0..2e7e32f51e 100644 --- a/crates/ty_python_semantic/src/types/class/typed_dict.rs +++ b/crates/ty_python_semantic/src/types/class/typed_dict.rs @@ -860,11 +860,15 @@ pub enum DynamicTypedDictAnchor<'db> { /// `packs` holds the keyword-variadic packs spliced in with `{**Kwargs}`, /// which contribute no fields until they are specialized. Applying a type /// mapping resolves each one and folds its fields into `schema`. + /// + /// `openness` is closed, or the extra items a `**: T` entry declares, as the + /// `TypedDict` the transpiler writes for the literal says Synthesized { scope: ScopeId<'db>, range: TextRange, schema: TypedDictSchema<'db>, packs: Box<[Type<'db>]>, + openness: TypedDictOpenness<'db>, }, } @@ -894,6 +898,7 @@ impl<'db> DynamicTypedDictAnchor<'db> { range, schema, packs, + openness, } => Some(Self::Synthesized { scope: *scope, range: *range, @@ -902,6 +907,7 @@ impl<'db> DynamicTypedDictAnchor<'db> { .iter() .map(|pack| pack.recursive_type_normalized_impl(db, env, div, true)) .collect::>>()?, + openness: openness.recursive_type_normalized_impl(db, env, div, nested)?, }), } } @@ -965,6 +971,7 @@ impl<'db> DynamicTypedDictLiteral<'db> { range, schema, packs, + openness, } = self.anchor(db) else { return self; @@ -998,8 +1005,9 @@ impl<'db> DynamicTypedDictLiteral<'db> { } } let pending: Box<[Type<'db>]> = pending.into_boxed_slice(); + let mapped_openness = openness.apply_type_mapping_impl(db, env, type_mapping, tcx, visitor); - if &mapped == schema && &pending == packs { + if &mapped == schema && &pending == packs && mapped_openness == *openness { return self; } @@ -1011,6 +1019,7 @@ impl<'db> DynamicTypedDictLiteral<'db> { range: *range, schema: mapped, packs: pending, + openness: mapped_openness, }, self.typed_dict_module(db), ) @@ -1089,10 +1098,8 @@ impl<'db> DynamicTypedDictLiteral<'db> { DynamicTypedDictAnchor::Definition(definition) => { deferred_functional_typed_dict_openness(db, *definition) } - DynamicTypedDictAnchor::ScopeOffset { openness, .. } => *openness, - // basedpython synthesized `{"key": T}` type literals behave like a - // default (implicitly open) `TypedDict` - DynamicTypedDictAnchor::Synthesized { .. } => TypedDictOpenness::ImplicitlyOpen, + DynamicTypedDictAnchor::ScopeOffset { openness, .. } + | DynamicTypedDictAnchor::Synthesized { openness, .. } => *openness, } } diff --git a/crates/ty_python_semantic/src/types/context.rs b/crates/ty_python_semantic/src/types/context.rs index efc73d3065..5a3f9c7ed0 100644 --- a/crates/ty_python_semantic/src/types/context.rs +++ b/crates/ty_python_semantic/src/types/context.rs @@ -293,6 +293,13 @@ impl<'db, 'ast> InferContext<'db, 'ast> { .mark_extension_module_used(module); } + /// basedpython: record that `range` is an f-string inferred as a template literal type + pub(super) fn record_template_literal_type(&self, range: TextRange) { + self.diagnostics + .borrow_mut() + .record_template_literal_type(range); + } + /// Prevents diagnostic construction for this inference context. pub(super) fn suppress_diagnostics(&mut self) { debug_assert!(!self.diagnostics_suppressed); diff --git a/crates/ty_python_semantic/src/types/dedicated/pytest/fixtures.rs b/crates/ty_python_semantic/src/types/dedicated/pytest/fixtures.rs index aee22dfc0c..83e2b69e19 100644 --- a/crates/ty_python_semantic/src/types/dedicated/pytest/fixtures.rs +++ b/crates/ty_python_semantic/src/types/dedicated/pytest/fixtures.rs @@ -624,7 +624,8 @@ fn fixture_request_for_parameter<'db>( db: &'db dyn Db, definition: Definition<'db>, ) -> Option> { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(_)) = definition.kind(db) + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(_, _)) = + definition.kind(db) else { return None; }; diff --git a/crates/ty_python_semantic/src/types/definition_resolution.rs b/crates/ty_python_semantic/src/types/definition_resolution.rs index 8c5998290c..136ad05a8c 100644 --- a/crates/ty_python_semantic/src/types/definition_resolution.rs +++ b/crates/ty_python_semantic/src/types/definition_resolution.rs @@ -473,12 +473,12 @@ fn declared_annotation<'a>( match definition.kind(db) { DefinitionKind::Parameter(parameter) => match parameter { ty_python_core::definition::ParameterDefinitionNodeKind::VariadicPositionalParameter( - parameter, + parameter, _ ) | ty_python_core::definition::ParameterDefinitionNodeKind::VariadicKeywordParameter( - parameter, + parameter, _ ) => parameter.node(parsed).annotation.as_deref(), - ty_python_core::definition::ParameterDefinitionNodeKind::Parameter(parameter) => { + ty_python_core::definition::ParameterDefinitionNodeKind::Parameter(parameter, _) => { parameter.node(parsed).parameter.annotation.as_deref() } }, diff --git a/crates/ty_python_semantic/src/types/diagnostic.rs b/crates/ty_python_semantic/src/types/diagnostic.rs index 014b260512..c0ee7d8ff2 100644 --- a/crates/ty_python_semantic/src/types/diagnostic.rs +++ b/crates/ty_python_semantic/src/types/diagnostic.rs @@ -24,7 +24,7 @@ use crate::types::infer::UnsupportedComparisonError; use crate::types::overrides::MethodKind; use crate::types::protocol_class::ProtocolMember; use crate::types::reified_infer::{ErasedTargetReason, ReifiedInferenceError}; -use crate::types::repeated_underscore::UnderscoreRefusal; +use crate::types::repeated_underscore::{RepeatedUnderscoreRead, UnderscoreRefusal}; use crate::types::runtime_reads::RuntimeReadCheck; use crate::types::special_form::TypeQualifier; use crate::types::string_annotation::{ @@ -601,7 +601,7 @@ declare_lint! { declare_lint! { #[doc = include_str!("../../resources/lint_docs/invalid-repeated-underscore.md")] pub(crate) static INVALID_REPEATED_UNDERSCORE = { - summary: "detects a repeated `_` parameter the lowering has no python spelling for", + summary: "detects a repeated `_` parameter the lowering has no python spelling for, or a read of one", status: LintStatus::stable("0.0.83"), default_level: Level::Error, ty_compat: TyCompat::BasedPython, @@ -4225,6 +4225,10 @@ struct InferenceUses { /// the sites that write module-level names into code that runs, whose names are checked /// once the file is inferred ([`crate::types::runtime_reads`]) runtime_read_checks: Vec, + /// every f-string inferred as a template literal type. the transpiler writes each as the + /// `str` or `Literal[...]` it means, so none of the f-string's own syntax reaches the + /// python, whatever version that python is for + template_literal_types: Vec, } /// reports a bare call of the reified generic function `name` whose specialization cannot be @@ -4492,9 +4496,22 @@ impl TypeCheckDiagnostics { uses.scope_reads.extend(&other_uses.scope_reads); uses.runtime_read_checks .extend_from_slice(&other_uses.runtime_read_checks); + uses.template_literal_types + .extend_from_slice(&other_uses.template_literal_types); } } + /// basedpython: records that `range` is an f-string inferred as a template literal type + pub(super) fn record_template_literal_type(&mut self, range: TextRange) { + self.uses_mut().template_literal_types.push(range); + } + + pub(crate) fn template_literal_types(&self) -> &[TextRange] { + self.uses + .as_ref() + .map_or(&[], |uses| &uses.template_literal_types) + } + pub(super) fn extend_diagnostics(&mut self, diagnostics: impl IntoIterator) { self.diagnostics.extend(diagnostics); } @@ -4556,6 +4573,7 @@ impl TypeCheckDiagnostics { uses.extension_modules.shrink_to_fit(); uses.scope_reads.shrink_to_fit(); uses.runtime_read_checks.shrink_to_fit(); + uses.template_literal_types.shrink_to_fit(); } self.diagnostics.shrink_to_fit(); } @@ -4875,14 +4893,17 @@ fn assignment_declaration_annotation<'db>( assignment.annotation(context.module()), DeclarationKind::Regular, )), - DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(parameter)) => parameter - .node(context.module()) - .parameter - .annotation - .as_deref() - .map(|annotation| (annotation, DeclarationKind::Regular)), + DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(parameter, _)) => { + parameter + .node(context.module()) + .parameter + .annotation + .as_deref() + .map(|annotation| (annotation, DeclarationKind::Regular)) + } DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicPositionalParameter( parameter, + _, )) => parameter .node(context.module()) .annotation @@ -4890,6 +4911,7 @@ fn assignment_declaration_annotation<'db>( .map(|annotation| (annotation, DeclarationKind::VariadicParameter)), DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicKeywordParameter( parameter, + _, )) => parameter .node(context.module()) .annotation @@ -6365,6 +6387,20 @@ pub(super) fn report_invalid_repeated_underscore( diagnostic.help(refusal.help()); } +/// basedpython: report a read of `_` that may read one of a parameter list's repeated `_`s — +/// see [`RepeatedUnderscoreRead`]. the transpiler refuses the same read +pub(super) fn report_repeated_underscore_read( + context: &InferContext, + name: &ast::ExprName, + read: &RepeatedUnderscoreRead, +) { + let Some(builder) = context.report_lint(&INVALID_REPEATED_UNDERSCORE, name) else { + return; + }; + let mut diagnostic = builder.into_diagnostic(read.message()); + diagnostic.help(read.help()); +} + /// basedpython: what is wrong with a `decorator def` the lowering cannot expand. pub(super) enum InvalidDecoratorDef<'a> { /// written in a class body, where the keyword is not available at all diff --git a/crates/ty_python_semantic/src/types/display.rs b/crates/ty_python_semantic/src/types/display.rs index 51410ed476..4c6a63b1df 100644 --- a/crates/ty_python_semantic/src/types/display.rs +++ b/crates/ty_python_semantic/src/types/display.rs @@ -28,7 +28,8 @@ use crate::types::function::{FunctionType, OverloadLiteral}; use crate::types::generics::{GenericContext, Specialization}; use crate::types::protocol_class::{InlineProtocolMemberForm, ProtocolInterface}; use crate::types::signatures::{ - CallableSignature, Parameter, Parameters, ParametersKind, Signature, + CallableSignature, NarrowingGuard, NarrowingGuardKind, Parameter, Parameters, ParametersKind, + Signature, }; use crate::types::tuple::{RepeatedUnit, TupleSpec, VariableLengthTuple, VariableSegment}; use crate::types::typevar::{BoundTypeVarIdentity, BoundTypeVarInstance}; @@ -2407,6 +2408,16 @@ impl<'db> FmtDetailed<'db> for DisplayRepresentation<'_, 'db> { .fmt_detailed(f)?; wrote_any = true; } + if let Some(extra_items) = typeddict.openness(self.db).explicit_extra_items() { + if wrote_any { + f.write_str(", ")?; + } + f.write_str("**: ")?; + extra_items + .declared_ty + .display_with(self.db, env, self.settings.clone()) + .fmt_detailed(f)?; + } return f.write_char('}'); } match defining_class { @@ -3582,6 +3593,7 @@ impl<'db> Signature<'db> { generic_context: self.generic_context.as_ref(), parameters: self.parameters(), return_ty: self.return_ty, + narrowing_guards: &self.narrowing_guards, db, env, settings, @@ -3594,6 +3606,9 @@ pub(crate) struct DisplaySignature<'a, 'db> { generic_context: Option<&'a GenericContext<'db>>, parameters: &'a Parameters<'db>, return_ty: Type<'db>, + /// basedpython: the guards the return annotation wrote, which name the place a `TypeIs` + /// narrows and the places an assertion narrows + narrowing_guards: &'a [NarrowingGuard<'db>], db: &'db dyn Db, env: &'a ProgramEnvironment<'db>, settings: DisplaySettings<'db>, @@ -3711,6 +3726,47 @@ impl<'db> FmtDetailed<'db> for DisplaySignature<'_, 'db> { // `def f()` and `def f() -> None` are the same declaration. A callable *type* has no // such default and always states what it returns, or `() -> None` would read as `()` let omit_none_return = is_declaration && self.return_ty.is_none(self.db); + if !self.settings.hide_return_type && basedpython_display_enabled() { + // basedpython: a narrowing return reads back as the guard that wrote it, since the + // `TypeIs[int]` python spells it with does not say which place it narrows, and an + // assertion's `None` says nothing at all + let assertions: Vec<_> = self + .narrowing_guards + .iter() + .filter(|guard| guard.is_written_assertion()) + .collect(); + if !assertions.is_empty() { + f.write_str(" -> asserts ")?; + for (index, guard) in assertions.into_iter().enumerate() { + if index > 0 { + f.write_str(" and ")?; + } + self.fmt_assertion(guard, &settings, &mut f)?; + } + return self.close_top(&mut f); + } + if let Type::TypeIs(type_is) = self.return_ty + && type_is.place_name(db).is_none() + && let Some(guard) = self + .narrowing_guards + .iter() + .find(|guard| matches!(guard.kind, NarrowingGuardKind::Predicate)) + { + // a callable type's return binds tighter than a comparison, so its predicate is + // parenthesized, as it is written + f.write_str(if is_declaration { " -> " } else { " -> (" })?; + fmt_guard_place(guard, &mut f)?; + f.write_str(" is ")?; + type_is + .type_argument(db) + .display_with(db, env, settings.singleline()) + .fmt_detailed(&mut f)?; + if !is_declaration { + f.write_char(')')?; + } + return self.close_top(&mut f); + } + } if !self.settings.hide_return_type && !omit_none_return { f.write_str(" -> ")?; @@ -3734,12 +3790,54 @@ impl<'db> FmtDetailed<'db> for DisplaySignature<'_, 'db> { } } + self.close_top(&mut f) + } +} + +impl<'db> DisplaySignature<'_, 'db> { + fn close_top(&self, f: &mut TypeWriter<'_, '_, 'db>) -> fmt::Result { if self.parameters.is_top() { f.write_str("]")?; } - Ok(()) } + + /// basedpython: one written assertion, as the return annotation spells it after `asserts` + fn fmt_assertion( + &self, + guard: &NarrowingGuard<'db>, + settings: &DisplaySettings<'db>, + f: &mut TypeWriter<'_, '_, 'db>, + ) -> fmt::Result { + match guard.kind { + NarrowingGuardKind::Asserts { is_positive } => { + if !is_positive { + f.write_str("not ")?; + } + fmt_guard_place(guard, f) + } + NarrowingGuardKind::AssertsType { is_positive, ty } => { + fmt_guard_place(guard, f)?; + f.write_str(if is_positive { " is " } else { " is not " })?; + ty.display_with(self.db, self.env, settings.singleline()) + .fmt_detailed(f) + } + NarrowingGuardKind::Predicate + | NarrowingGuardKind::InferredPredicate { .. } + | NarrowingGuardKind::InferredAssertion { .. } => Ok(()), + } + } +} + +/// basedpython: the place a guard narrows, `x` or `self.data` +fn fmt_guard_place(guard: &NarrowingGuard<'_>, f: &mut TypeWriter<'_, '_, '_>) -> fmt::Result { + f.set_invalid_type_annotation(); + f.write_str(&guard.name)?; + for member in &guard.members { + f.write_char('.')?; + f.write_str(member)?; + } + Ok(()) } impl Display for DisplaySignature<'_, '_> { diff --git a/crates/ty_python_semantic/src/types/function.rs b/crates/ty_python_semantic/src/types/function.rs index a41f434bc0..97620a92dc 100644 --- a/crates/ty_python_semantic/src/types/function.rs +++ b/crates/ty_python_semantic/src/types/function.rs @@ -3032,12 +3032,7 @@ impl<'db> FunctionType<'db> { /// would depend on the function's AST and rerun for every change in that file. #[salsa::tracked( returns(ref), - cycle_initial=|db, id, function: FunctionType<'db>| { - let env = ProgramEnvironment::from_scope( - function.literal(db).last_definition.body_scope(db), - ); - CallableSignature::cycle_initial(db, &env, id) - }, + cycle_initial=|_, id, _| CallableSignature::cycle_initial(id), cycle_fn=|db, cycle, previous, value: CallableSignature<'db>, function: FunctionType<'db>| { let env = ProgramEnvironment::from_scope( function.literal(db).last_definition.body_scope(db), diff --git a/crates/ty_python_semantic/src/types/ide_support.rs b/crates/ty_python_semantic/src/types/ide_support.rs index 0a6347e7f5..1989b001b0 100644 --- a/crates/ty_python_semantic/src/types/ide_support.rs +++ b/crates/ty_python_semantic/src/types/ide_support.rs @@ -29,7 +29,9 @@ use crate::types::list_members::all_end_of_scope_members; use crate::types::literal::LiteralValueTypeKind; use crate::types::overrides::{OverriddenDeclaration, OverrideStep, overridden_declarations}; use crate::types::receivers; -use crate::types::repeated_underscore::{UnderscoreLowering, UnderscoreRefusal}; +use crate::types::repeated_underscore::{ + RepeatedUnderscoreRead, UnderscoreLowering, UnderscoreRefusal, repeated_underscore_read, +}; use crate::types::signatures::{ Parameter, ParameterKind, ParametersKind, ReturnCallableTypeVarScope, Signature, is_placeholder_default, @@ -3778,7 +3780,7 @@ fn inherited_expression<'db>( db: &'db dyn Db, definition: Definition<'db>, ) -> Option> { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = definition.kind(db) else { return None; @@ -4098,6 +4100,35 @@ pub fn repeated_underscore_lowering( .clone() } +/// basedpython: whether the body of `function` refers to the `_` its parameters bind — reads +/// or deletes it, or holds a scope that reads it from the function or declares it `nonlocal` +pub fn refers_to_own_underscore( + model: &SemanticModel<'_>, + function: &ast::StmtFunctionDef, +) -> bool { + let db = model.db(); + let index = semantic_index(db, db.program_file(model.file())); + index + .try_node_scope(NodeWithScopeRef::Function(function)) + .is_some_and(|scope| { + crate::types::repeated_underscore::refers_to_own_underscore(index, scope) + }) +} + +/// basedpython: whether `name` is a read of `_` that may read one of a parameter list's +/// repeated `_`s — the read ty reports, and the transpiler refuses +pub fn repeated_underscore_read_of( + model: &SemanticModel<'_>, + name: &ast::ExprName, +) -> Option { + repeated_underscore_read( + model.db(), + &model.program_environment(), + model.program_file(), + name, + ) +} + /// basedpython: the return type recovered for a `def` that leaves its annotation out. /// /// `None` is what such a `def` already means, so recovering it says nothing the source did not diff --git a/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs b/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs index dab47fcbbd..96e417a1d5 100644 --- a/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs +++ b/crates/ty_python_semantic/src/types/ide_support/unused_bindings.rs @@ -85,11 +85,11 @@ fn parameter_is_context(kind: &DefinitionKind<'_>, parsed: &ParsedModuleRef) -> return false; }; match parameter { - ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter) - | ParameterDefinitionNodeKind::VariadicKeywordParameter(parameter) => { + ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter, _) + | ParameterDefinitionNodeKind::VariadicKeywordParameter(parameter, _) => { parameter.node(parsed).is_context } - ParameterDefinitionNodeKind::Parameter(parameter) => { + ParameterDefinitionNodeKind::Parameter(parameter, _) => { parameter.node(parsed).parameter.is_context } } diff --git a/crates/ty_python_semantic/src/types/infer/builder.rs b/crates/ty_python_semantic/src/types/infer/builder.rs index c11b6f40c7..5707a2e089 100644 --- a/crates/ty_python_semantic/src/types/infer/builder.rs +++ b/crates/ty_python_semantic/src/types/infer/builder.rs @@ -83,16 +83,16 @@ use crate::types::diagnostic::{ INACCESSIBLE_MEMBER, INEFFECTIVE_FINAL, INVALID_ARGUMENT_TYPE, INVALID_ASSIGNMENT, INVALID_ATTRIBUTE_ACCESS, INVALID_DECLARATION, INVALID_ENUM_MEMBER_ANNOTATION, INVALID_FIELD_LOOKUP, INVALID_LEGACY_TYPE_VARIABLE, INVALID_NEWTYPE, INVALID_PARAMSPEC, - INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, INVALID_TYPE_ALIAS_TYPE, INVALID_TYPE_FORM, - INVALID_TYPE_VARIABLE_CONSTRAINTS, INVALID_TYPE_VARIABLE_DEFAULT, INVALID_VARIANCE_DECLARATION, - NARROWING_GUARD_AS_VALUE, NON_EXHAUSTIVE_STATEMENT_EXPRESSION, NON_OVERLAPPING_CAST, - NON_OVERLAPPING_TYPE_TEST, OPTIONAL_OBJECT_CONVERSION, POSSIBLY_MISSING_IMPLICIT_CALL, - POSSIBLY_MISSING_SUBMODULE, PRIVATE_CONSTRUCTOR, REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, - TRAILING_LAMBDA_PARAMETERS, TypeCheckDiagnostics, UNANNOTATED_MODEL_FIELD, - UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, UNDEFINED_REVEAL, UNRESOLVED_ATTRIBUTE, - UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, UNSOUND_ASSIGNMENT, UNSOUND_CAST, UNSOUND_YIELD, - UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, YieldKind, - autofix_with_notimplementederror, display_required_elements, + INVALID_REGEX, INVALID_REIFIED_TYPE_PARAM, INVALID_SUPER_ARGUMENT, INVALID_TYPE_ALIAS_TYPE, + INVALID_TYPE_FORM, INVALID_TYPE_VARIABLE_CONSTRAINTS, INVALID_TYPE_VARIABLE_DEFAULT, + INVALID_VARIANCE_DECLARATION, NARROWING_GUARD_AS_VALUE, NON_EXHAUSTIVE_STATEMENT_EXPRESSION, + NON_OVERLAPPING_CAST, NON_OVERLAPPING_TYPE_TEST, OPTIONAL_OBJECT_CONVERSION, + POSSIBLY_MISSING_IMPLICIT_CALL, POSSIBLY_MISSING_SUBMODULE, PRIVATE_CONSTRUCTOR, + REFUTABLE_DESTRUCTURING, REFUTABLE_UNPACKING, TRAILING_LAMBDA_PARAMETERS, TypeCheckDiagnostics, + UNANNOTATED_MODEL_FIELD, UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, UNDEFINED_REVEAL, + UNRESOLVED_ATTRIBUTE, UNRESOLVED_GLOBAL, UNRESOLVED_REFERENCE, UNSOUND_ASSIGNMENT, + UNSOUND_CAST, UNSOUND_YIELD, UNSUPPORTED_OPERATOR, UNUSED_AWAITABLE, USED_UNDERSCORE_NAME, + YieldKind, autofix_with_notimplementederror, display_required_elements, hint_if_stdlib_attribute_exists_on_other_versions, refutable_unpacking_applies, report_attempted_protocol_instantiation, report_bad_dunder_delattr_call, report_bad_dunder_delete_call, report_bool_as_int, report_bool_as_int_assignment, @@ -107,7 +107,8 @@ use crate::types::diagnostic::{ report_match_pattern_against_non_runtime_checkable_protocol, report_match_pattern_against_typed_dict, report_mismatched_type_name, report_possibly_missing_attribute, report_possibly_unresolved_reference, - report_restricted_constructor, report_too_many_positional_patterns_for_class_pattern, + report_repeated_underscore_read, report_restricted_constructor, + report_too_many_positional_patterns_for_class_pattern, report_unfilled_decoration_context_parameters, report_unplaceable_starred_class_pattern, report_unsound_assignment, report_unsound_yield, report_unspecialized_reified_call, report_unspecialized_reified_construction, report_unsupported_augmented_assignment, @@ -145,6 +146,7 @@ use crate::types::regex; use crate::types::reified_infer; use crate::types::repeated_underscore::{ LoweredParameters, WrittenNames, lower_repeated_underscores, parameter_slots, + repeated_underscore_read, }; use crate::types::runtime_reads::{ReifiedGeneric, RuntimeReadSite}; use crate::types::set_theoretic::RecursivelyDefined; @@ -162,7 +164,9 @@ use crate::types::trailing_lambda::{ use crate::types::tuple::promotion::TupleSizePromotionConstraints; use crate::types::tuple::{Tuple, TupleLength, TupleSpecBuilder, TupleType, VariableSegment}; use crate::types::type_alias::{ManualPEP695TypeAliasType, PEP695TypeAliasType}; -use crate::types::typed_dict::{TypedDictAssignmentKind, TypedDictKeyAssignment}; +use crate::types::typed_dict::{ + TypedDictAssignmentKind, TypedDictKeyAssignment, TypedDictOpenness, +}; use crate::types::typevar::{ BoundTypeVarIdentity, TypeVarConstraints, TypeVarIdentity, TypeVarInstance, TypeVarSet, }; @@ -1631,7 +1635,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { self.infer_comprehension_definition(comprehension, definition); } DefinitionKind::Parameter( - ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter), + ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter, _), ) => { self.infer_variadic_positional_parameter_definition( parameter.node(self.module()), @@ -1640,6 +1644,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } DefinitionKind::Parameter(ParameterDefinitionNodeKind::VariadicKeywordParameter( parameter, + _, )) => { self.infer_variadic_keyword_parameter_definition( parameter.node(self.module()), @@ -1648,6 +1653,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter( parameter_with_default, + _, )) => { self.infer_parameter_definition( parameter_with_default.node(self.module()), @@ -1657,7 +1663,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { index, lambda, - parameter: ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter), + parameter: ParameterDefinitionNodeKind::VariadicPositionalParameter(parameter, _), }) => { self.infer_variadic_positional_lambda_parameter_definition( *index, @@ -1667,7 +1673,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ); } DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { - parameter: ParameterDefinitionNodeKind::VariadicKeywordParameter(parameter), + parameter: ParameterDefinitionNodeKind::VariadicKeywordParameter(parameter, _), .. }) => { self.infer_variadic_keyword_lambda_parameter_definition( @@ -1678,7 +1684,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { index, lambda, - parameter: ParameterDefinitionNodeKind::Parameter(parameter_with_default), + parameter: ParameterDefinitionNodeKind::Parameter(parameter_with_default, _), }) => { self.infer_lambda_parameter_definition( *index, @@ -4194,11 +4200,21 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { true } - Type::Dynamic(..) - | Type::Divergent(_) - | Type::Never - | Type::ModuleLiteral(..) - | Type::BoundSuper(..) => true, + // a `super` object has no attributes of its own to delete, and does not pass a deletion + // on to the class it reads past, so python raises `AttributeError` for any attribute + Type::BoundSuper(_) => { + if emit_diagnostics + && let Some(builder) = self.context.report_lint(&INVALID_ASSIGNMENT, target) + { + builder.into_diagnostic(format_args!( + "Cannot delete attribute `{attribute}` on type `{}`", + object_ty.display(db, env), + )); + } + false + } + + Type::Dynamic(..) | Type::Divergent(_) | Type::Never | Type::ModuleLiteral(..) => true, } } @@ -4254,7 +4270,10 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { .. }, ) => { - let object_ty = self.infer_expression(object, TypeContext::default()); + let object_ty = match self.infer_super_keyword(object) { + Some(keyword) => keyword, + None => self.infer_expression(object, TypeContext::default()), + }; self.report_undeclared_protocol_attribute(attr_expr); if let Some(infer_assigned_ty) = infer_assigned_ty { @@ -6341,6 +6360,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ast::Expr::Name(name) => { let previous_value = self.infer_name_load(name, TypeContext::default()); self.check_underscore_name_load(name); + let previous_value = self.check_repeated_underscore_read(name, previous_value); self.store_expression_type(target, previous_value); Ok(previous_value) } @@ -8681,19 +8701,20 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { let db = self.db(); let mut schema = TypedDictSchema::default(); let mut packs: Vec> = Vec::new(); + // closed unless a `**: T` entry admits extra items, as the `TypedDict` the transpiler + // writes for the literal is + let mut openness = TypedDictOpenness::Closed; let mut hasher = DefaultHasher::new(); for item in &dict.items { let Some(key_expr) = item.key.as_ref() else { - // `**: T` extra-items marker — encoded as `Starred(Starred(T))`. - // TODO: wire this into `TypedDictSchema.extra_items` once - // PEP 728 support lands. for now we accept the syntax and - // infer the type so user-visible diagnostics still fire, but - // we don't yet enforce extra-key matching + // `**: T` extra-items marker — encoded as `Starred(Starred(T))` if let ast::Expr::Starred(outer) = &item.value && let ast::Expr::Starred(inner) = outer.value.as_ref() { - let _ = self.infer_type_expression(&inner.value); + let extra_ty = self.infer_type_expression(&inner.value); "**".hash(&mut hasher); + extra_ty.display(db, env).to_string().hash(&mut hasher); + openness = TypedDictOpenness::extra(db, extra_ty, false); continue; } // basedpython `{**Kwargs}`: the pack contributes no fields until it is @@ -8728,6 +8749,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { range: dict.range(), schema, packs: packs.into_boxed_slice(), + openness, }; let td = DynamicTypedDictLiteral::new(db, class_name, anchor, TypingModule::Typing); Type::ClassLiteral(ClassLiteral::DynamicTypedDict(td)).to_instance_approximation(db, env) @@ -14700,6 +14722,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ExprContext::Load => { let ty = self.infer_name_load(name, tcx); self.check_underscore_name_load(name); + let ty = self.check_repeated_underscore_read(name, ty); // basedpython: a `type def` has no runtime existence — the // declaration is erased when transpiling — so naming one in a value // position would emit python that raises `NameError` @@ -14825,52 +14848,77 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } - /// basedpython: compute the bound-super type for `super` / `super[T]` - /// when used as an attribute base. returns `None` when `value` does not - /// match the sugar form, when there is no enclosing class, or when the - /// supplied target class is not in the enclosing class' MRO. - fn basedpython_super_value_type(&mut self, value: &ast::Expr) -> Option> { - let env = self.program_environment(); - let db = self.db(); - - let target_class: Option> = match value { - ast::Expr::Name(n) if n.id.as_str() == "super" => None, - ast::Expr::Subscript(s) => { - let ast::Expr::Name(n) = s.value.as_ref() else { - return None; - }; - if n.id.as_str() != "super" { - return None; - } - Some(self.expression_type(s.slice.as_ref()).to_class_type(db)?) + /// basedpython: infer `value` as the `super` keyword, when it is one: `super` or `super[T]` + /// standing as an attribute's base. `None` for any other expression, which is left for the + /// caller to infer: anywhere else `super` is the name and `super[T]` its subscript, as python + /// reads them. + /// + /// the keyword is given the bound super the lowering's `super()` call evaluates to, so an + /// attribute is read, assigned and deleted through the same object in the checker as at run + /// time. `super[T]` names the class the attribute is looked up past + fn infer_super_keyword(&mut self, value: &ast::Expr) -> Option> { + if !self.is_basedpython_file() { + return None; + } + let ty = match value { + ast::Expr::Name(name) if name.id.as_str() == "super" => { + self.super_keyword_bound(value, None) + } + ast::Expr::Subscript(subscript) + if subscript + .value + .as_name_expr() + .is_some_and(|name| name.id.as_str() == "super") => + { + self.infer_expression(&subscript.value, TypeContext::default()); + let target = self.infer_expression(&subscript.slice, TypeContext::default()); + self.super_keyword_bound(value, Some(target)) } _ => return None, }; + self.store_expression_type(value, ty); + Some(ty) + } - // `super` needs the implicit arguments a method supplies — the `__class__` - // cell and the receiver. directly in a class body there is neither, so - // `super().x` there raises at runtime; report it rather than lower it - let reaches_a_function = self - .index - .ancestor_scopes(self.scope().file_scope_id(db)) - .take_while(|(_, ancestor)| ancestor.node().as_class().is_none()) - .any(|(_, ancestor)| ancestor.node().as_function().is_some()); - // both unavailable cases answer `Unknown` rather than declining: falling - // through would let the ordinary attribute load report a second, less - // useful `Class `super` has no attribute …` on top of this one - if !reaches_a_function { + /// basedpython: the bound super the `super` keyword `value` stands for, looked up past + /// `target`, the class `super[T]` names, reporting the keyword where python could not build + /// one + fn super_keyword_bound(&mut self, value: &ast::Expr, target: Option>) -> Type<'db> { + let env = self.program_environment(); + let db = self.db(); + + // `super` reads its implicit arguments where it is written, as zero-argument `super()` + // does and as the lowering passes them: the `__class__` cell of the class around it, and + // the first parameter of the function or lambda it is written in. a class body or a + // comprehension is no such function, so `super().x` there raises at runtime; report it + // rather than lower it + // + // each unavailable case answers `Unknown` rather than declining: falling through would + // let the ordinary attribute load report a second, less useful `Class `super` has no + // attribute …` on top of this one + let scope = self.scope(); + let unavailable = match scope.node(db) { + NodeWithScopeKind::Function(_) | NodeWithScopeKind::Lambda(_) => None, + NodeWithScopeKind::ListComprehension(_) + | NodeWithScopeKind::SetComprehension(_) + | NodeWithScopeKind::DictComprehension(_) + | NodeWithScopeKind::GeneratorExpression(_) => Some( + "Cannot determine implicit arguments for `super` in a comprehension, \ + which reads them from the function python runs it as", + ), + _ => Some("Cannot determine implicit arguments for `super` outside a method"), + }; + if let Some(message) = unavailable { if let Some(builder) = self .context .report_lint(&UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, value) { - builder.into_diagnostic( - "Cannot determine implicit arguments for `super` outside a method", - ); + builder.into_diagnostic(message); } - return Some(Type::unknown()); + return Type::unknown(); } - let Some(enclosing) = nearest_enclosing_class(db, self.index, self.scope()) else { + let Some(enclosing) = nearest_enclosing_class(db, self.index, scope) else { // in a function, but no class encloses it — `super` has a `__class__` // cell to read only inside a method, so this is the same failure if let Some(builder) = self @@ -14882,31 +14930,122 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { no class encloses this function", ); } - return Some(Type::unknown()); + return Type::unknown(); + }; + if crate::types::bound_super::report_super_in_named_tuple_method( + &self.context, + enclosing, + value.into(), + ) { + return Type::unknown(); + } + let Some(owner_type) = crate::types::bound_super::implicit_super_receiver( + db, + self.index, + scope, + self.module(), + ) else { + if let Some(builder) = self + .context + .report_lint(&UNAVAILABLE_IMPLICIT_SUPER_ARGUMENTS, value) + { + builder.into_diagnostic( + "Cannot determine implicit arguments for `super`: \ + the function it is written in has no parameter to read the receiver from", + ); + } + return Type::unknown(); }; let enclosing_class = ClassLiteral::Static(enclosing).default_specialization(db); - let owner_type = Type::instance(db, env, enclosing_class); - let pivot_class_type = match target_class { + let pivot_class_type = match target { None => Type::from(enclosing_class), - Some(target) => { - let target_lit = target.class_literal(db); - let mut prev: Option> = None; - let mut found: Option> = None; - for entry in enclosing_class.class_literal(db).iter_mro(db) { - if let crate::types::ClassBase::Class(c) = entry { - if c.class_literal(db) == target_lit { - found = prev; - break; - } - prev = Some(c); - } + Some(target) if target.is_dynamic() => return target, + // the pivot is the class python finds in the MRO, which is never a specialization + Some(target) => match self.super_keyword_pivot(value, enclosing_class, target) { + Some(pivot) => Type::ClassLiteral(pivot.class_literal(db)), + None => return Type::unknown(), + }, + }; + + match crate::types::bound_super::BoundSuperType::build( + db, + env, + pivot_class_type, + owner_type, + ) { + Ok(bound_super) => bound_super, + Err(error) => { + error.report_diagnostic(&self.context, value.into()); + Type::unknown() + } + } + } + + /// basedpython: the class `super[T]` looks up past, the one before `target` in the MRO of + /// `enclosing`, the class it is written in, reporting `value` where there is none + /// + /// the lowering finds that class at run time, as `C.__mro__[C.__mro__.index(T) - 1]`, which + /// raises `ValueError` for a `T` that is not in the MRO, and reads past `object` for `C` + /// itself. a value of a `type[...]` type may be a subclass the MRO does not hold, so only a + /// class literal names a class in it + fn super_keyword_pivot( + &self, + value: &ast::Expr, + enclosing: ClassType<'db>, + target: Type<'db>, + ) -> Option> { + let db = self.db(); + let env = self.program_environment(); + let target_literal = match target { + Type::ClassLiteral(literal) => literal, + Type::SubclassOf(_) => { + if let Some(builder) = self.context.report_lint(&INVALID_SUPER_ARGUMENT, value) { + builder.into_diagnostic(format_args!( + "`{}` is not known to be a class in the MRO of `{}`", + target.display(db, env), + enclosing.name(db), + )); + } + return None; + } + _ => { + crate::types::bound_super::BoundSuperError::InvalidPivotClassType { + pivot_class: target, } - Type::from(found?) + .report_diagnostic(&self.context, value.into()); + return None; } }; - crate::types::bound_super::BoundSuperType::build(db, env, pivot_class_type, owner_type).ok() + let mut previous: Option> = None; + for entry in enclosing.class_literal(db).iter_mro(db) { + if let crate::types::ClassBase::Class(class) = entry { + if class.class_literal(db) == target_literal { + if previous.is_none() + && let Some(builder) = + self.context.report_lint(&INVALID_SUPER_ARGUMENT, value) + { + builder.into_diagnostic(format_args!( + "`{}` comes first in its own MRO, so `super[{}]` has no class to \ + look up past", + enclosing.name(db), + target_literal.name(db), + )); + } + return previous; + } + previous = Some(class); + } + } + if let Some(builder) = self.context.report_lint(&INVALID_SUPER_ARGUMENT, value) { + builder.into_diagnostic(format_args!( + "`{}` is not in the MRO of `{}`", + target_literal.name(db), + enclosing.name(db), + )); + } + None } /// Resolve `receiver` to the type the next link of a basedpython optional chain must be @@ -14960,8 +15099,12 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { &mut self, attribute: &ast::ExprAttribute, ) -> Result, Type<'db>> { - let value_type = - self.infer_maybe_standalone_expression(&attribute.value, TypeContext::default()); + let value_type = match self.infer_super_keyword(&attribute.value) { + Some(keyword) => keyword, + None => { + self.infer_maybe_standalone_expression(&attribute.value, TypeContext::default()) + } + }; let (value_type, in_chain) = self.basedpython_chain_receiver(&attribute.value, value_type); self.infer_attribute_load_chained(attribute, value_type, in_chain) } @@ -15068,16 +15211,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } - // basedpython: `super.x` and `super[T].x` sugar in `.by` files. - // overrides value_type with the corresponding bound-super type so that - // attribute lookup is performed against the MRO, mirroring what the - // transpile produces (`super().x` / `super(, self).x`) - if self.is_basedpython_file() - && let Some(super_value_type) = self.basedpython_super_value_type(value) - { - value_type = super_value_type; - } - // basedpython safe variance: a private member does not specialize. It keeps the type it // was declared with, and through any receiver but the class's own that type is erased to // what such a view actually knows — the parameter's bound @@ -15581,7 +15714,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { member_type } ExprContext::Store => { - self.infer_expression(value, TypeContext::default()); + if self.infer_super_keyword(value).is_none() { + self.infer_expression(value, TypeContext::default()); + } self.validate_member_visibility(attribute, self.expression_type(value)); Type::Never } @@ -15635,6 +15770,26 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { }); } + /// basedpython: report a read of `_` that may read one of a parameter list's repeated + /// `_`s, which python and ty bind to different parameters, and give the type the read + /// `name` has, given that ty resolved it to `ty` + /// + /// a refused read has no one meaning, so it is `Unknown` rather than whichever `_` ty took + /// it to be: what it goes on to be used in is not checked against a parameter python never + /// binds there + fn check_repeated_underscore_read(&self, name: &ast::ExprName, ty: Type<'db>) -> Type<'db> { + let Some(read) = repeated_underscore_read( + self.db(), + self.program_environment(), + self.program_file(), + name, + ) else { + return ty; + }; + report_repeated_underscore_read(&self.context, name, &read); + Type::unknown() + } + /// basedpython: reports a read of the name `name` when a leading underscore /// marks it unused — see [`USED_UNDERSCORE_NAME`] fn check_underscore_name_load(&self, name: &ast::ExprName) { diff --git a/crates/ty_python_semantic/src/types/infer/builder/function.rs b/crates/ty_python_semantic/src/types/infer/builder/function.rs index e61d4142aa..548e1832b2 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/function.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/function.rs @@ -2023,11 +2023,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { )); } - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(declared_ty), - ); + self.add_parameter(parameter.into(), definition, declared_ty); } else if let Some(function) = self.enclosing_trailing_lambda() { // basedpython: the implicit `it` parameter of a trailing lambda // block is context-typed from the called expression — the sole @@ -2217,11 +2213,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { _ => Type::homogeneous_tuple(db, self.program_environment(), annotated_type), }; - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(ty), - ); + self.add_parameter(parameter.into(), definition, ty); } else { let inferred_ty = Type::homogeneous_tuple(db, self.program_environment(), Type::unknown()); @@ -2569,11 +2561,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { &[KnownClass::Str.to_instance(db, env), annotated_type], ) }; - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(ty), - ); + self.add_parameter(parameter.into(), definition, ty); } else { let inferred_ty = KnownClass::Dict.to_specialized_instance( db, @@ -2586,6 +2574,33 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } } + /// bind a parameter to `ty`, which is also the declared type of the parameter's name when the + /// parameter is a declaration + /// + /// basedpython: an annotated parameter is one unless its name is not its own. a parameter + /// list that repeats `_` lowers each `_` to a parameter of its own name, so none of their + /// annotations declares the `_` the body binds + fn add_parameter( + &mut self, + node: AnyNodeRef<'ast>, + definition: Definition<'db>, + ty: Type<'db>, + ) { + if definition + .kind(self.db()) + .category(self.context.in_stub(), self.module()) + .is_declaration() + { + self.add_declaration_with_binding( + node, + definition, + &DeclaredAndInferredType::are_the_same_type(ty), + ); + } else { + self.add_binding(node, definition).insert(self, ty); + } + } + /// Set initial declared type (if annotated) and inferred type for a lambda-parameter symbol, /// in the lambda body scope. pub(super) fn infer_lambda_parameter_definition( @@ -2626,21 +2641,8 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { // basedpython typed lambdas (`lambda (a: int) -> int: ...`) make the // parameter a `DeclarationAndBinding` because the annotation is set, - // matching annotated function parameters. The plain `add_binding` - // path looks up `declarations_by_binding` which is only populated for - // pure bindings — for declared+bound parameters we go through - // `add_declaration_with_binding` instead, otherwise the lookup - // panics with "no entry found for key". - if parameter.annotation.is_some() { - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(ty), - ); - } else { - self.add_binding(parameter.into(), definition) - .insert(self, ty); - } + // matching annotated function parameters + self.add_parameter(parameter.into(), definition, ty); } /// Set initial declared/inferred types for a `*args` variadic positional parameter @@ -2664,19 +2666,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { Type::Dynamic(DynamicType::UnknownLambdaParameter), ) }; - // see `infer_lambda_parameter_definition` — annotated `*args` is a - // `DeclarationAndBinding`, which doesn't populate - // `declarations_by_binding`, so `add_binding` would panic - if parameter.annotation.is_some() { - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(ty), - ); - } else { - self.add_binding(parameter.into(), definition) - .insert(self, ty); - } + self.add_parameter(parameter.into(), definition, ty); } /// Set initial declared/inferred types for a `**kwargs` keyword-variadic parameter @@ -2697,16 +2687,7 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { ], ); - if parameter.annotation.is_some() { - self.add_declaration_with_binding( - parameter.into(), - definition, - &DeclaredAndInferredType::are_the_same_type(inferred_ty), - ); - } else { - self.add_binding(parameter.into(), definition) - .insert(self, inferred_ty); - } + self.add_parameter(parameter.into(), definition, inferred_ty); } /// Returns the annotated type of the lambda parameter at the given index in the provided diff --git a/crates/ty_python_semantic/src/types/infer/builder/post_inference/typeguard.rs b/crates/ty_python_semantic/src/types/infer/builder/post_inference/typeguard.rs index 446ea119af..d546032ab0 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/post_inference/typeguard.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/post_inference/typeguard.rs @@ -327,22 +327,10 @@ fn check_assertion_established<'db>( definition.map_or_else(outer, |definition| binding_type(db, definition)) }; - let claim = match guard.kind { - NarrowingGuardKind::Asserts { is_positive: true } => "truthy".to_string(), - NarrowingGuardKind::Asserts { is_positive: false } => "falsy".to_string(), - NarrowingGuardKind::AssertsType { - is_positive: true, - ty, - } => format!("`{}`", ty.display(db, env)), - NarrowingGuardKind::AssertsType { - is_positive: false, - ty, - } => format!("not `{}`", ty.display(db, env)), - // an assertion recovered from a body says what that body established, which is a type - NarrowingGuardKind::InferredAssertion { ty } => format!("`{}`", ty.display(db, env)), - // a predicate narrows where the call is tested rather than once it returns, so there is - // no assertion for a body to establish - NarrowingGuardKind::Predicate | NarrowingGuardKind::InferredPredicate { .. } => return, + // a predicate narrows where the call is tested rather than once it returns, so there is no + // assertion for a body to establish + let Some(claim) = guard.kind.assertion_claim(db, env) else { + return; }; // a member is read off whatever the place above it holds, so rebinding that place leaves what diff --git a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs index d0c0d371c8..373b1fe1b3 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/subscript.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/subscript.rs @@ -396,20 +396,6 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { is_type_decoration: _, } = subscript; - // basedpython: `super[T]` is sugar for `super(, self)` - // and must not be type-checked as a regular subscript expression. infer - // the slice as a type so `reveal_type` etc. behave, then return the - // pivot class type — the attribute lookup against the bound super is - // performed in `infer_attribute_load` - if self.is_basedpython_file() - && let ast::Expr::Name(name) = value.as_ref() - && name.id.as_str() == "super" - { - self.infer_expression(value, TypeContext::default()); - let slice_ty = self.infer_expression(slice, TypeContext::default()); - return slice_ty; - } - match ctx { ExprContext::Load => self .infer_subscript_load(subscript, tcx) diff --git a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs index df79e40699..24463b8dce 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/type_expression.rs @@ -2710,6 +2710,7 @@ impl<'db> TypeInferenceBuilder<'db, '_> { /// that `str()` fills. a conversion or a format spec would change what fills /// it, so both are rejected rather than quietly ignored. fn infer_template_literal_type_expression(&mut self, fstring: &ast::ExprFString) -> Type<'db> { + self.context.record_template_literal_type(fstring.range()); let mut parts: Vec> = Vec::new(); for part in &fstring.value { match part { diff --git a/crates/ty_python_semantic/src/types/infer/builder/typevar.rs b/crates/ty_python_semantic/src/types/infer/builder/typevar.rs index 1651bf832c..a5861026d3 100644 --- a/crates/ty_python_semantic/src/types/infer/builder/typevar.rs +++ b/crates/ty_python_semantic/src/types/infer/builder/typevar.rs @@ -1068,7 +1068,9 @@ impl<'db, 'ast> TypeInferenceBuilder<'db, 'ast> { } _ => false, }; - if same_kind { + // a name that resolves to nothing is already reported as unresolved. a `ParamSpec` + // default keeps the second report upstream makes for it + if same_kind || (kind.is_keyword_variadic() && ty.is_unknown()) { return; } } diff --git a/crates/ty_python_semantic/src/types/inferred_signature.rs b/crates/ty_python_semantic/src/types/inferred_signature.rs index 5967ebffe0..3123c6bbfb 100644 --- a/crates/ty_python_semantic/src/types/inferred_signature.rs +++ b/crates/ty_python_semantic/src/types/inferred_signature.rs @@ -285,7 +285,7 @@ pub(crate) fn inferred_parameter_default<'db>( parameter: Definition<'db>, ) -> Option> { let env = &ProgramEnvironment::from_definition(parameter); - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = parameter.kind(db) else { return None; @@ -1181,7 +1181,7 @@ fn comparison_dunder(op: ast::CmpOp) -> Option<&'static str> { /// The name a parameter definition binds. fn parameter_definition_name<'db>(db: &'db dyn Db, parameter: Definition<'db>) -> Option { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = parameter.kind(db) else { return None; diff --git a/crates/ty_python_semantic/src/types/normal_completion.rs b/crates/ty_python_semantic/src/types/normal_completion.rs index b5862bac78..d79f94b8f0 100644 --- a/crates/ty_python_semantic/src/types/normal_completion.rs +++ b/crates/ty_python_semantic/src/types/normal_completion.rs @@ -599,7 +599,7 @@ impl<'db> CalleeResolver<'_, 'db> { parameter: &ParameterDefinitionNodeKind, ) -> Option> { let db = self.db; - let ParameterDefinitionNodeKind::Parameter(parameter) = parameter else { + let ParameterDefinitionNodeKind::Parameter(parameter, _) = parameter else { return None; }; let file = definition.program_file(db); diff --git a/crates/ty_python_semantic/src/types/relation_error.rs b/crates/ty_python_semantic/src/types/relation_error.rs index 3184fc182d..390afc01b8 100644 --- a/crates/ty_python_semantic/src/types/relation_error.rs +++ b/crates/ty_python_semantic/src/types/relation_error.rs @@ -178,6 +178,13 @@ pub(crate) enum ErrorContext<'db> { source: String, target: String, }, + /// basedpython: `target` asserts something about a place that `source` does not establish: + /// it asserts nothing about it, or the claims in `source`, which together are not as narrow + WeakerAssertion { + place: String, + source: Vec, + target: String, + }, IncompatibleParameterTypes { source: Type<'db>, target: Type<'db>, @@ -548,6 +555,22 @@ impl<'db> ErrorContext<'db> { Self::NarrowsAnotherArgument { source, target } => { format!("the return type narrows {source}, where {target} is expected") } + Self::WeakerAssertion { + place, + source, + target, + } => { + if source.is_empty() { + format!( + "the function asserts nothing about {place}, where {target} is expected" + ) + } else { + format!( + "the function asserts {place} is {source}, where {target} is expected", + source = source.join(" and ") + ) + } + } Self::IncompatibleParameterTypes { source, target, diff --git a/crates/ty_python_semantic/src/types/repeated_underscore.rs b/crates/ty_python_semantic/src/types/repeated_underscore.rs index 9e6dc7d538..0440897452 100644 --- a/crates/ty_python_semantic/src/types/repeated_underscore.rs +++ b/crates/ty_python_semantic/src/types/repeated_underscore.rs @@ -9,16 +9,35 @@ //! //! what each parameter is called in the python, and how many of them a call reaches by //! position alone, is decided here once — for ty's signature of the definition and for the -//! transpiler's output alike +//! transpiler's output alike. so is which reads of `_` are refused: python binds the name to +//! one of the parameters and ty to another, so a read of a repeated `_` has no one meaning use std::collections::HashSet; +use ruff_db::parsed::{ParsedModuleRef, parsed_module}; use ruff_python_ast as ast; use ruff_python_ast::name::Name; use ruff_python_stdlib::identifiers::is_identifier_continuation; -use ty_python_core::{FileScopeId, SemanticIndex}; +use ty_python_core::definition::{ + Definition, DefinitionKind, DefinitionState, LambdaParameterDefinitionNodeKind, ParameterName, +}; +use ty_python_core::place::PlaceExpr; +use ty_python_core::scope::NodeWithScopeKind; +use ty_python_core::symbol::Symbol; +use ty_python_core::{ + BindingWithConstraintsIterator, FileScopeId, ProgramFile, SemanticIndex, semantic_index, + use_def_map, +}; +use crate::place::{ + ConsideredDefinitions, RequiresExplicitReExport, place_by_id, place_from_bindings, +}; +use crate::place_load::{ + PlaceLoadMode, PlaceLoadResolutionStep, PlaceLoadSourceKind, resolve_place_load, +}; +use crate::reachability::evaluate_reachability_with_cache; use crate::types::signatures::Parameter; +use crate::{Db, ProgramEnvironment}; /// the names a module spells anywhere in the text its author wrote — every run of /// identifier characters, in code, strings and comments alike @@ -592,17 +611,36 @@ fn inherited_names<'a>( /// whether a parameter named `name` would change what the body of the function whose /// scope is `scope` reads: the body mentions the name itself, or a scope nested in it reads /// the name without binding it, which python resolves through the function -/// -/// a nested scope that binds the name reads its own, but one nested in *that* is still -/// asked, which can only answer yes where the answer did not matter pub(crate) fn reads_from_enclosing_scope( index: &SemanticIndex, scope: FileScopeId, name: &str, ) -> bool { - if index.place_table(scope).symbol_by_name(name).is_some() { - return true; - } + index.place_table(scope).symbol_by_name(name).is_some() + || read_by_nested_scope(index, scope, name) +} + +/// whether the body of the function whose scope is `scope` refers to the `_` its parameters +/// bind: it reads or deletes `_` itself, or a scope nested in it reads `_` without binding +/// it or declares it `nonlocal` +/// +/// an override's `_`s take the names of the method it overrides, so the python binds `_` on +/// entry for such a body, to the first of them. without it, a nested `nonlocal _` has no +/// binding to find and python rejects the module +pub(crate) fn refers_to_own_underscore(index: &SemanticIndex, scope: FileScopeId) -> bool { + index + .place_table(scope) + .symbol_by_name("_") + .is_some_and(Symbol::is_used) + || read_by_nested_scope(index, scope, "_") +} + +/// whether a scope nested in `scope` reads `name` without binding it, or declares it +/// `nonlocal`, which python resolves through `scope` +/// +/// a nested scope that binds the name reads its own, but one nested in *that* is still +/// asked, which can only answer yes where the answer did not matter +fn read_by_nested_scope(index: &SemanticIndex, scope: FileScopeId, name: &str) -> bool { let mut pending: Vec = index.child_scopes(scope).map(|(id, _)| id).collect(); while let Some(scope) = pending.pop() { if let Some(symbol) = index.place_table(scope).symbol_by_name(name) @@ -616,6 +654,171 @@ pub(crate) fn reads_from_enclosing_scope( false } +/// a read of `_` that may read one of a parameter list's repeated `_`s +/// +/// python binds `_` to one of them and ty to another, the lowering keeping the first `_` +/// and ty's bindings reaching the read from the last, so the read is refused rather than +/// answered two ways: +/// +/// ```by +/// def f(_: int, _: str) -> str: +/// return _ # python returns the `int` +/// ``` +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepeatedUnderscoreRead { + /// the `def` the parameters belong to, `None` for a lambda + function: Option, +} + +impl RepeatedUnderscoreRead { + /// what is wrong, in the words a diagnostic and a transpile error both use + pub fn message(&self) -> String { + match &self.function { + Some(function) => format!("reading a repeated `_` parameter of `{function}`"), + None => "reading a repeated `_` parameter of a lambda".to_owned(), + } + } + + /// how to write the definition so it is accepted + pub fn help(&self) -> &'static str { + "several parameters are named `_`, so a read cannot say which: give the one the body \ + reads a name of its own" + } +} + +/// whether `name`, a read in `file`, may read one of a parameter list's repeated `_`s, and +/// whose they are when it does +/// +/// the read is resolved the way inference resolves it: from the bindings that reach it, +/// outward through every scope the name is free in, until a source binds it on every path. +/// a read that only a rebinding of `_` reaches reads that rebinding, and is accepted, and so +/// is a read the parser wrote: the `self._ = _` an `init(let _: int, let _: int)` stands for +/// reads each parameter by the name the lowering gives it +/// +/// `del _` is not a read. it takes no value, and leaves the name unbound in either answer +pub(crate) fn repeated_underscore_read<'db>( + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + file: ProgramFile<'db>, + name: &ast::ExprName, +) -> Option { + if name.id != "_" || !file.file(db).source_type(db).is_basedpython() { + return None; + } + let index = semantic_index(db, file); + let expr = ast::ExprRef::Name(name); + // a name the index never saw has no reaching bindings to resolve: one parsed out of a + // string annotation is such + let scope = index.try_expression_scope_id(&expr)?.to_scope_id(db, file); + index.try_use_id(expr)?; + let module = parsed_module(db, file.python_file(db)).load(db); + if module + .generated_names() + .get(name.range, name.id.clone()) + .is_some() + { + return None; + } + + let resolution = resolve_place_load( + db, + index, + scope, + PlaceExpr::from_expr_name(name), + PlaceLoadMode::AtExpression(expr), + ); + for step in resolution { + match step { + PlaceLoadResolutionStep::Source(source) => { + let bound = match source.kind { + PlaceLoadSourceKind::Bindings(bindings) => { + if let Some(read) = repeated_parameter_in(db, &module, bindings.clone()) { + return Some(read); + } + place_from_bindings(db, env, bindings) + .place + .is_definitely_bound() + } + PlaceLoadSourceKind::DefinitionsFromOwningScope { scope, id } => { + let bindings = use_def_map(db, scope).reachable_bindings(id); + if let Some(read) = repeated_parameter_in(db, &module, bindings) { + return Some(read); + } + place_by_id( + db, + scope, + id, + RequiresExplicitReExport::No, + ConsideredDefinitions::AllReachable, + ) + .place + .is_definitely_bound() + } + // a value a debugger read out of the frame is the one the read has + PlaceLoadSourceKind::Observed(_) => true, + // a name python supplies without a binding is no parameter, and none of + // them is a `_`, so the sources after it are asked as inference asks them + PlaceLoadSourceKind::Implicit(_) => false, + }; + if bound { + return None; + } + } + PlaceLoadResolutionStep::MemberResolutionCondition(_) => {} + PlaceLoadResolutionStep::Exhausted(_) => return None, + } + } + None +} + +/// the repeated `_`s whose parameter definition is among `bindings`, when one is and may be +/// reached +fn repeated_parameter_in<'db>( + db: &'db dyn Db, + module: &ParsedModuleRef, + bindings: BindingWithConstraintsIterator<'_, 'db>, +) -> Option { + let predicates = bindings.predicates(); + let reachability_constraints = bindings.reachability_constraints(); + bindings + .filter(|binding| { + !evaluate_reachability_with_cache( + db, + None, + reachability_constraints, + predicates, + binding.reachability_constraint, + ) + .is_always_false() + }) + .find_map(|binding| match binding.binding { + DefinitionState::Defined(definition) => repeated_parameter(db, module, definition), + _ => None, + }) +} + +/// the parameter list `definition` is a repeated `_` of, when it is one +fn repeated_parameter<'db>( + db: &'db dyn Db, + module: &ParsedModuleRef, + definition: Definition<'db>, +) -> Option { + let (DefinitionKind::Parameter(parameter) + | DefinitionKind::LambdaParameter(LambdaParameterDefinitionNodeKind { parameter, .. })) = + definition.kind(db) + else { + return None; + }; + if parameter.name() != ParameterName::RepeatedUnderscore { + return None; + } + let function = match definition.scope(db).node(db) { + NodeWithScopeKind::Function(function) => Some(function.node(module).name.id.clone()), + _ => None, + }; + Some(RepeatedUnderscoreRead { function }) +} + /// the name each of `parameters`, a parameter list written as each one's slot and source /// name, binds in the python it lowers to, where no parameter takes a name from an /// overridden method diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 61bce9b464..afa860337b 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -21,7 +21,7 @@ use itertools::{Either, EitherOrBoth, Itertools}; use rustc_hash::{FxHashMap, FxHashSet}; use smallvec::{SmallVec, smallvec_inline}; -use super::{DynamicType, Type, TypeVarVariance, UnionType, semantic_index}; +use super::{DynamicType, IntersectionType, Type, TypeVarVariance, UnionType, semantic_index}; use crate::types::UnpackedKwargs; use crate::types::callable::{CallableTypeKind, walk_callable_type}; use crate::types::constraints::{ @@ -195,15 +195,19 @@ impl<'db> CallableSignature<'db> { Self::single(Signature::bottom()) } - pub(crate) fn cycle_initial( - db: &'db dyn Db, - env: &ProgramEnvironment<'db>, - id: salsa::Id, - ) -> Self { - Self::single(Signature::new( - Parameters::bottom(), - Type::divergent(id).bottom_materialization(db, env), - )) + /// the signature a function has while the cycle computing it is still in progress + /// + /// basedpython: its return type is the cycle's own unmaterialized marker, the same value + /// [`inferred_return_type`](crate::types::inferred_signature::inferred_return_type) starts + /// from. a body that calls its own function reaches the return type through whichever of the + /// two queries the cycle re-enters first, and that depends on where checking started: a call + /// from the module body enters through this signature, a check of the function's own body + /// through the return type. upstream starts this one from a bottom-materialized marker, which + /// behaves as `Never` and so drops out of the first union it meets — and with it the only + /// record that the return type was defined in terms of itself. a float doubled on every round + /// then never widens, and the query iterates until salsa gives up + pub(crate) fn cycle_initial(id: salsa::Id) -> Self { + Self::single(Signature::new(Parameters::bottom(), Type::divergent(id))) } fn is_cycle_initial(&self) -> bool { @@ -770,6 +774,32 @@ impl<'db> NarrowingGuardKind<'db> { NarrowingGuardKind::Predicate | NarrowingGuardKind::InferredPredicate { .. } => None, } } + + /// What an assertion guard claims about its place, in words: `truthy`, `` `int` ``, `` not + /// `str` ``. `None` for a predicate, which claims nothing once the call returns. + pub(crate) fn assertion_claim( + self, + db: &'db dyn Db, + env: &ProgramEnvironment<'db>, + ) -> Option { + Some(match self { + NarrowingGuardKind::Asserts { is_positive: true } => "truthy".to_string(), + NarrowingGuardKind::Asserts { is_positive: false } => "falsy".to_string(), + NarrowingGuardKind::AssertsType { + is_positive: true, + ty, + } => format!("`{}`", ty.display(db, env)), + NarrowingGuardKind::AssertsType { + is_positive: false, + ty, + } => format!("not `{}`", ty.display(db, env)), + // an assertion recovered from a body says what that body established, which is a type + NarrowingGuardKind::InferredAssertion { ty } => format!("`{}`", ty.display(db, env)), + NarrowingGuardKind::Predicate | NarrowingGuardKind::InferredPredicate { .. } => { + return None; + } + }) + } } impl<'db> NarrowingGuard<'db> { @@ -1246,25 +1276,29 @@ impl<'db> Signature<'db> { None => NarrowedArgument::Nothing, }); }; - Some( - match self - .parameters - .iter() - .enumerate() - .find(|(_, parameter)| parameter.name() == Some(&guard.name)) - { - Some((index, parameter)) => { - NarrowedArgument::of_parameter(index, parameter, &guard.members) - } - None if guard.root_is_first_parameter => NarrowedArgument::Receiver { - members: &guard.members, - }, - None => NarrowedArgument::Place { - name: &guard.name, - members: &guard.members, - }, + Some(self.guarded_argument(guard)) + } + + /// basedpython: what a call through this signature reaches with `guard`, which names a + /// parameter of it, the receiver it was bound to, or a place in the calling scope + fn guarded_argument<'a>(&'a self, guard: &'a NarrowingGuard<'db>) -> NarrowedArgument<'a> { + match self + .parameters + .iter() + .enumerate() + .find(|(_, parameter)| parameter.name() == Some(&guard.name)) + { + Some((index, parameter)) => { + NarrowedArgument::of_parameter(index, parameter, &guard.members) + } + None if guard.root_is_first_parameter => NarrowedArgument::Receiver { + members: &guard.members, }, - ) + None => NarrowedArgument::Place { + name: &guard.name, + members: &guard.members, + }, + } } /// Returns the binding referenced by a direct `P.args` or `P.kwargs` variadic parameter. @@ -3453,6 +3487,75 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { self.check_tuple_spec_pair(db, &target_tuple, &source_tuple) } + /// basedpython: whether `source` establishes everything `target` asserts. + /// + /// an assertion narrows its place once the call returns, so a caller calling through `target` + /// relies on each place it names being narrowed at least that far. the function actually + /// called has to assert something about the same argument, and what it establishes there — + /// every assertion it makes about it at once — has to be the narrower type + fn check_assertion_pair( + &self, + db: &'db dyn Db, + source: &Signature<'db>, + target: &Signature<'db>, + ) -> ConstraintSet<'db, 'c> { + let env = self.env; + if target.deferred_assertions.is_none() + && !target + .narrowing_guards + .iter() + .any(|guard| guard.kind.is_assertion()) + { + return self.always(); + } + let target_guards = target.all_narrowing_guards(db); + let source_guards = source.all_narrowing_guards(db); + let mut result = self.always(); + for target_guard in target_guards.iter() { + let Some(asserted) = target_guard.kind.asserted_type(db, env) else { + continue; + }; + let place = target.guarded_argument(target_guard); + let matching: Vec<&NarrowingGuard<'db>> = source_guards + .iter() + .filter(|source_guard| { + source_guard.kind.is_assertion() + && place.is_narrowed_by(&source.guarded_argument(source_guard)) + }) + .collect(); + let established = matching + .iter() + .filter_map(|source_guard| source_guard.kind.asserted_type(db, env)) + .reduce(|left, right| IntersectionType::from_two_elements(db, env, left, right)); + let holds = established.is_some_and(|established| { + !result + .intersect( + db, + self.constraints, + self.check_type_pair(db, established, asserted), + ) + .is_never_satisfied(db, env) + }); + if !holds { + if let Some(context) = self.report_context() { + context.push(ErrorContext::WeakerAssertion { + place: place.to_string(), + source: matching + .iter() + .filter_map(|source_guard| source_guard.kind.assertion_claim(db, env)) + .collect(), + target: target_guard + .kind + .assertion_claim(db, env) + .unwrap_or_default(), + }); + } + return self.never(); + } + } + result + } + fn check_paramspec_return_pair( &self, db: &'db dyn Db, @@ -3828,6 +3931,16 @@ impl<'c, 'db> TypeRelationChecker<'_, 'c, 'db> { return_type_checks = false; } + if return_type_checks { + let assertions = self.check_assertion_pair(db, source, target); + if result + .intersect(db, self.constraints, assertions) + .is_never_satisfied(db, env) + { + return_type_checks = false; + } + } + // basedpython: a variadic parameter that takes whole repetitions of several types, // `*args: *((int, str) * int)`, cannot be written out as a list of parameters, so the two // signatures are compared through the tuples of positional arguments they accept. @@ -7396,7 +7509,7 @@ impl<'db> ParameterDefault<'db> { heap_size=ruff_memory_usage::heap_size )] fn parameter_default_type<'db>(db: &'db dyn Db, parameter: Definition<'db>) -> Type<'db> { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = parameter.kind(db) else { return Type::unknown(); @@ -7425,7 +7538,7 @@ fn parameter_default_type<'db>(db: &'db dyn Db, parameter: Definition<'db>) -> T /// call that leaves the argument out receives #[salsa::tracked(returns(copy), heap_size=ruff_memory_usage::heap_size)] pub(crate) fn is_placeholder_default<'db>(db: &'db dyn Db, parameter: Definition<'db>) -> bool { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = parameter.kind(db) else { return false; @@ -7664,7 +7777,7 @@ fn parameter_declared_type<'db>( db: &'db dyn Db, parameter: Definition<'db>, ) -> Option<(Definition<'db>, Type<'db>)> { - let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node)) = + let DefinitionKind::Parameter(ParameterDefinitionNodeKind::Parameter(node, _)) = parameter.kind(db) else { return None; diff --git a/crates/ty_python_semantic/src/types/typed_dict.rs b/crates/ty_python_semantic/src/types/typed_dict.rs index b09b096212..0b1383389c 100644 --- a/crates/ty_python_semantic/src/types/typed_dict.rs +++ b/crates/ty_python_semantic/src/types/typed_dict.rs @@ -125,7 +125,7 @@ impl<'db> TypedDictOpenness<'db> { matches!(self, Self::Closed) } - fn apply_type_mapping_impl<'a>( + pub(crate) fn apply_type_mapping_impl<'a>( self, db: &'db dyn Db, env: &ProgramEnvironment<'db>, diff --git a/crates/ty_python_semantic/src/types/visibility.rs b/crates/ty_python_semantic/src/types/visibility.rs index 9029779088..6dc7fe9bc4 100644 --- a/crates/ty_python_semantic/src/types/visibility.rs +++ b/crates/ty_python_semantic/src/types/visibility.rs @@ -354,6 +354,39 @@ pub(crate) fn resolves_to_module_scope<'db>( file: ProgramFile<'db>, reference: &ast::ExprName, ) -> Option { + name_resolution(db, file, reference).map(|resolution| { + matches!( + resolution, + NameResolution::Module | NameResolution::DeclaredGlobal + ) + }) +} + +/// basedpython: where python finds the binding of a name +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameResolution { + /// a pep 695 type-parameter list binds it + TypeParameter, + /// a scope on the way to its binding declares it `global` + DeclaredGlobal, + /// the module binds it, reached with no `global` declaration + Module, + /// a function, lambda, comprehension, class body or type alias binds it + Local, + /// no scope binds it: it is a builtin, or unbound + Unbound, +} + +/// basedpython: where python finds the binding of the name `reference` reads or writes. +/// `None` when ty did not index the reference +/// +/// a class body is flow-sensitive here as python is: a read the body has not yet bound +/// resolves outward, and one it binds on some paths only is the class's +pub(crate) fn name_resolution<'db>( + db: &'db dyn Db, + file: ProgramFile<'db>, + reference: &ast::ExprName, +) -> Option { let index = crate::semantic_index(db, file); let reference_scope = index.try_expression_scope_id(&ast::ExprRef::from(reference))?; for (ancestor_id, scope) in index.ancestor_scopes(reference_scope) { @@ -369,21 +402,29 @@ pub(crate) fn resolves_to_module_scope<'db>( }; // `global count` hands the name to the module, whatever this scope does if symbol.is_global() { - return Some(true); + return Some(NameResolution::DeclaredGlobal); } // a binding, or a bare annotation, makes the name local to its scope if !(symbol.is_bound() || symbol.is_declared()) { continue; } - if scope.kind() == ScopeKind::Class && reference.ctx.is_load() { - match class_scope_binding(db, scope_id, reference) { - ClassBinding::Unbound => continue, - ClassBinding::Bound | ClassBinding::Maybe => return Some(false), - } + if scope.kind() == ScopeKind::Class + && reference.ctx.is_load() + && class_scope_binding(db, scope_id, reference) == ClassBinding::Unbound + { + continue; } - return Some(scope.kind() == ScopeKind::Module); + return Some(match scope.kind() { + ScopeKind::Module => NameResolution::Module, + ScopeKind::TypeParams => NameResolution::TypeParameter, + ScopeKind::Class + | ScopeKind::Function + | ScopeKind::Lambda + | ScopeKind::Comprehension + | ScopeKind::TypeAlias => NameResolution::Local, + }); } - Some(false) + Some(NameResolution::Unbound) } /// basedpython: whether the name `reference` reads is read in a class body's own diff --git a/crates/ty_server/src/server/api/requests/transpile.rs b/crates/ty_server/src/server/api/requests/transpile.rs index edf845e259..07302153f7 100644 --- a/crates/ty_server/src/server/api/requests/transpile.rs +++ b/crates/ty_server/src/server/api/requests/transpile.rs @@ -138,7 +138,7 @@ impl BackgroundDocumentRequestHandler for TranspileRequestHandler { let fragment = params.source.is_some(); let transpiled = transpile_detached(move || { if fragment { - by_transforms::transpile(source, &config) + by_transforms::transpile_analysed_at(source, &config, config.min_version) } else { by_transforms::transpile_typed(&*owned, file, &config, None) .map_err(|error| error.to_string()) diff --git a/crates/ty_server/tests/e2e/transpile.rs b/crates/ty_server/tests/e2e/transpile.rs index 28400dcef6..6458c90012 100644 --- a/crates/ty_server/tests/e2e/transpile.rs +++ b/crates/ty_server/tests/e2e/transpile.rs @@ -88,6 +88,40 @@ fn transpiling_a_fragment_answers_rather_than_panicking() -> Result<()> { Ok(()) } +/// A fragment is analysed at the project's python version, as the whole document is. +/// `frozendict` is a builtin from 3.15, past ty's newest python, so the conversion its +/// annotation makes is only written where the analysis knows it. +#[test] +fn a_fragment_is_analysed_at_the_project_s_version() -> Result<()> { + let workspace_root = SystemPath::new("src"); + let main = SystemPath::new("src/main.by"); + let content = "a: frozendict[str, str] = {}\n"; + + let mut server = TestServerBuilder::new()? + .with_workspace(workspace_root, None)? + .with_file(main, content)? + .with_file( + SystemPath::new("ty.toml"), + "[environment]\npython-version = \"3.15\"\n", + )? + .build() + .wait_until_workspaces_are_initialized(); + + server.open_text_document(main, content, 1); + let uri = server.file_uri(main); + + let response = server + .send_request_await::(params(&uri, Some(content))) + .expect("the server answered nothing at all"); + + let generated = response.source.expect("no generated source"); + assert!( + generated.contains("= frozendict()"), + "unexpected output:\n{generated}" + ); + Ok(()) +} + /// The whole-document path, which goes through the project database — and then /// still builds one of its own, because a pre-pass rewrote the source it lowers. /// `enum class` is such a pre-pass. diff --git a/docs/basedpython/development/compilation/runtime.md b/docs/basedpython/development/compilation/runtime.md index a719a95d9d..740c4f1020 100644 --- a/docs/basedpython/development/compilation/runtime.md +++ b/docs/basedpython/development/compilation/runtime.md @@ -122,9 +122,16 @@ is installed in the instance's dict word, which is what lets python's own attrib machinery put a stray name straight into it. a field read pays nothing for any of this; a field write pays one test of that word. -two differences remain, both loud: `type(vars(o))` names the mapping rather than -`dict`, and `del o.__dict__[f]` for a *field* raises where python removes it, -because a layout field has no unbound state to return to. +one difference remains, and it is loud: `type(vars(o))` names the mapping rather than +`dict`. + +a field is deleted as python deletes an attribute — `del o.f` from this module or from any +other, `del o.__dict__[f]` or `o.__dict__.clear()` — because every field has the state an +instance `__init__` never ran on is in, and a delete puts it back there. a read then +raises `AttributeError` in python's words, and a later write sets the field again. a +compiled function that has seen a field asks again once python code can have run, since +that code may have deleted it. a clear lets go of what it removed only once every name is +gone, as python's does, so a `__del__` that runs then finds the object already cleared. the invariant is checked once more where every attribute write passes, rather than only where the fields are worked out: a write of a name nothing in the @@ -484,6 +491,33 @@ bump([Acc(1)]) `//=`, `%=`, `**=`, `<<=` and `>>=` ask whether the left operand is an exact `int` before they take the plain operator's path, which is one more test on a short +such a value can have a `__del__`, so it is let go of when python lets go of it. a +tuple of `int`s nothing reads goes on holding its elements until python's would have +gone, and lets them go last to first; a name an element was taken out of a tuple into +holds it for as long as the name does: + +```python +def ints(f) -> tuple[int, int]: + return f(1), f(2) + + +def discarded(f) -> None: + ints(f) # the second element's `__del__` runs first, as in python + + +def kept(f) -> int: + head, part = split(f) + total = part + 1 + log("kept") # `part`'s `__del__` runs after this, as in python + return total +``` + +the tuple is held as the struct of its elements rather than as an object, so holding it +allocates nothing. what it still costs is a reference taken and given back per element: +the `tuples` benchmark, which builds `pair = (whole, part)` in its loop and reads only +the elements, retires 15% more instructions for it (10.2M to 11.7M a call), and `pairs` +2.4% more. no other benchmark's compiled functions changed + ## floats, strings, tuples - **`float`** is an unboxed `double` wherever it is not stored in an @@ -657,10 +691,75 @@ so the substitution is made everywhere a twin can still be held: | a retained interpreted definition's defaults | `By_RemapTwinDefaults`, one call per handle | | a declined class's own methods and attributes | the same walk, one step in | -a value that merely **reaches** a twin — an instance whose type it is, a list -holding one — is not moved and cannot be. those stay as the body left them, and -that limit is the reason the rule is a substitution of *the twin itself* rather -than a deep rewrite. +an instance the body built with one of those classes is moved too: an instance of the +emitted type is allocated without running a constructor, and the state the twin's +instance holds is handed over field by field, the way `object.__setattr__` writes it — +past any `__setattr__` the class wrote, so a `frozen data class` moves like any other. +the move is remembered, so every holder of the one instance ends up holding the one +replacement. a key of a dict moves the same way, and the dict is keyed again in the order +it held its keys, so a table the body built answers a key the compiled class makes: + +```python +frozen data class Pair: + x: int + y: int + + +ORIGIN: Final = Pair(0, 0) +TABLE = {Pair(1, 2): "a"} + +isinstance(ORIGIN, Pair) # both: True +TABLE[Pair(1, 2)] # both: "a" +``` + +a member of a set moves too. one whose replacement hashes as it did takes its own entry, +so the set iterates in the order it did. where one hashes differently — an instance hashed +on its identity — the set is built again from its members and iterates in whatever order +that gives, as python's did in the order the members' addresses gave it. a `frozenset` is +written in place where every member keeps its hash, and is otherwise settled as a copy, as +a tuple is, so a module-level name holding one goes on holding the original. a subclass of +`dict` or `set` moves the same way where every write it has is its base's own. + +what is still left as the body left it is a key of a dict, or a member of a set, whose class +writes one of its own — `OrderedDict` keeps its order in a list beside the table, and a +write past that would leave the two disagreeing — and a member of a `frozenset` subclass. +those still hold the interpreted class's instance: + +```python +ORDERED = OrderedDict({Pair(1, 2): "a"}) + +ORDERED[Pair(1, 2)] # python: "a"; compiled: KeyError +``` + +### what the body wrote over a definition's name + +installing a definition is only right while its name still holds what the definition +made. a write the source spells out — `f = g`, `del f` — is seen at build time and the +definition stays interpreted. a write through the namespace object is not: +`globals()["f"] = g`, `globals().update(...)`, a helper handed `globals()`, an `exec`, a +star import of an accelerator module — `operator`'s trailing `from _operator import *` is +the shape. so init asks the name itself, once the body has run: + +- a function is published only where its name still holds the function its `def` made, + told apart by its code object, which is a constant of the module's own. one the body + wrote over keeps what was written, and every compiled call of it goes through the name +- a class is installed only where its name still holds the class its statement built. one + the body wrote over leaves the **whole module** as its interpreted definition built it, + because compiled code reaches a class without going through its name + +```python +def f(n: int) -> int: + return n + 1 + + +def call(n: int) -> int: + return f(n) + + +globals()["f"] = lambda n: n - 100 + +call(1) # both: -99 +``` ### checking that what was installed is what was reported @@ -688,6 +787,17 @@ hands the interpreted definition back to stand as the class — in both cases th name holding that definition is the answer, not a defect, and the check records it rather than raising. +the one construction that cannot be handed back alone is a class on a base the module +emitted, beside one from outside or beside a second base of its own — `class ConvertingDict(dict, ConvertingMixin)` in `logging.config`. it is built by calling its +metaclass, and from 3.12 python refuses to lay out a `dict` and a class of ours as one +instance, as it refuses two classes of ours. the definition handed back stands on the +interpreted copy of the base while the base's name already means the emitted type, so +the import stands the **whole module** down there instead: before anything has been +decorated or adopted, every name it installed is given back its interpreted definition, +and the module runs as the interpreted one. whether the bases can be laid out as one is +the interpreter's own rule over whatever their names hold at import, so the build cannot +know it beforehand. + recording is the other half, and it is what the check is for. `--annotate` says which classes a module *meant* to compile; it cannot say which ones an import stood a type under, and for most of this project's life the report was read as @@ -882,6 +992,37 @@ sys.setrecursionlimit(100) mod.nested_depth(98) # python: RecursionError; compiled: 98 ``` +a construction counts the frame of the `__init__` it runs, as python's does, a generated one +included — made by compiled code, where it goes round a cycle of compiled calls, as any call +is. on 3.11 it also counts python's call of the type, which that version counts against the +same limit. from 3.13 python also specialises a construction a call site makes often, where +the class has no metaclass or `__new__` of its own and its `__init__` is a plain python +function given every argument by position — and, on 3.13, where its instances keep their +attributes inline — and the specialised form takes a second frame for the return from +`__init__`. a compiled class is never constructed that way, and neither is a class written +on one the module extends, so a recursion that goes round through such a construction +reaches the limit a level later for each one. that includes an `__init__` that builds +another instance of its own class, which python specialises once the call inside it is warm: + +```python +class Node: + def __init__(self, f: Callable[[int], int], n: int) -> None: + self.v = f(n) + + +# from another module, on 3.13 or later +def down(n): + return 0 if n == 0 else mod.Node(down, n - 1).v + 1 + + +sys.setrecursionlimit(100) +down(40) # python: RecursionError; compiled: 40 +``` + +python's own answer moves with how warm the call site is and with how the call is spelled — +`mod.Node(down, n=n - 1)` is not specialised, and counts the one frame — so there is no +one depth to match + `==` and `!=` are one more such call. where the left operand's type has no comparison of its own, python's answer is whatever the right operand's type says, and a compiled module asks that type directly instead of going through `PyObject_RichCompare` — which leaves out @@ -956,6 +1097,47 @@ mod.scaled.__kwdefaults__ = original mod.scaled(2) # python: 20; compiled with watchers: 20; compiled without: 2 ``` +### a field is an attribute of its class + +an emitted instance keeps its fields in its layout, and python reaches a field there +through a descriptor the class publishes under the field's name, as it reaches a slot. +so the class answers for a name python's class only holds on its instances: + +```python +class Cell: + def __init__(self, v: int) -> None: + self.v = v + + +hasattr(Cell, "v") # python: False; compiled: True +"v" in vars(Cell) # python: False; compiled: True, a `getset_descriptor` +``` + +an instance answers the same either way. a descriptor is the one place a generic attribute +read finds a field held outside a dict, so taking them away would mean a read of every +attribute going through a lookup of the class's own + +### the name python's messages give a class with a dunder field + +python's own messages name a class by the name its `class` statement gave it — +`'Box' object is not callable`, `cannot unpack non-iterable Box object` — and a compiled +class answers the same. a class that writes a field spelled as a dunder is the exception: +it is built as a static type, which carries its module in the name those messages print, +since that name is where a static type's `__module__` is read from. its `__module__`, +`repr` and pickling are python's: + +```python +class Tagged: + def __init__(self, v: int) -> None: + self.__tag__ = v + + +mod.Tagged(1)() # python: TypeError: 'Tagged' object is not callable; compiled: TypeError: 'mod.Tagged' object is not callable +``` + +a static type is what keeps the dunder a field: a class built the other way reads its +`__module__` and `__doc__` out of its own dict, where such a field's descriptor would stand + ### a method is not a `function` a class holds each method it compiled as a `by.method_descriptor` rather than a @@ -986,9 +1168,9 @@ mod.Scale().get() # python: 10; compiled: 3 a dunder python calls through a type slot — `__init__`, `__eq__`, `__len__` and the rest — is not one of those methods. the class holds the `wrapper_descriptor` cpython makes for the slot, which is what `object.__init__` is too, so it says nothing about the `def` it was -compiled from: it has no `__annotations__`, `__defaults__` or `__code__`, and what reads -those reads the slot's own description instead. `inspect.signature` of the class itself -finds no signature at all: +compiled from: it has no `__annotations__`, `__defaults__` or `__code__`, its `__doc__` is +the slot's own rather than the docstring, and what reads those reads the slot's own +description instead. `inspect.signature` of the class itself finds no signature at all: ```python class Point: @@ -999,6 +1181,7 @@ class Point: # from another module mod.Point.__init__.__annotations__ # python: {'x': int, 'y': int, 'return': None}; compiled: AttributeError +mod.Point.__init__.__doc__ # python: None; compiled: 'Initialize self. See help(type(self)) for accurate signature.' typing.get_type_hints(mod.Point.__init__) # python: {'x': int, 'y': int, 'return': NoneType}; compiled: {} inspect.signature(mod.Point.__init__) # python: (self, x: int, y: int = 0) -> None; compiled: (self, /, *args, **kwargs) inspect.signature(mod.Point) # python: (x: int, y: int = 0) -> None; compiled: ValueError @@ -1205,6 +1388,75 @@ from the class the decorator handed back. `setattr(Point, "__repr__", f)` after `data class Point`, or a write over one of its fields, leaves the module interpreted, and what the decorator wrote itself does not +### an instance's `__class__` reassigned + +python moves an instance to another class with `o.__class__ = Other` wherever the two lay +their instances out alike, and two classes written in python with no `__slots__` always do. +a compiled class lays its instances out itself, and python decides the move against that +layout: + +- two classes the module extends that share a base and add no field to it are laid out + alike, and the move is taken. a call compiled against the base asks the instance's type + before it reaches a method directly, so it answers as the class the instance now has +- a class and its own base, or a subclass that adds a field, are laid out apart, and the + move is refused with python's + `__class__ assignment: 'Base' object layout differs from 'Circle'` +- a class the module neither extends nor decorates is an immutable type — see + [writing onto a class nothing extends](#writing-onto-a-class-nothing-extends) — and + python refuses to move an instance to or from one, with + `__class__ assignment only supported for mutable types or ModuleType subclasses` + +```python +class Base: + def name(self) -> str: + return "base" + + +class Circle(Base): + def name(self) -> str: + return "circle" + + +class Shape(Base): + def name(self) -> str: + return "shape" + + +class Lone: + pass + + +# from another module +c = mod.Circle() +c.__class__ = mod.Shape # both: c.name() is "shape" +c.__class__ = mod.Base # python: moved; compiled: TypeError +mod.Lone().__class__ = mod.Base # python: moved; compiled: TypeError +``` + +where python refuses too — an instance of a class written in python moved to `int` — the +two refuse with the same words. the refusal is what keeps a method reached directly working +on the layout its class was compiled with + +### the `__new__` a class with an unset field publishes + +a compiled class with an `int` field that may not have been set publishes a `__new__` of +its own, which hands a subclass made outside the module the allocator that marks the field +unset. python's class has none, so `C.__new__` is `object.__new__`, and the entry answers +as that does: it is a builtin bound to `object`, with its docstring and signature, and it +refuses what `object.__new__` refuses, in the same words. what can tell the two apart is +the entry itself: + +```python +class Box: + def __init__(self, v: int) -> None: + self.v = v + + +# from another module +"__new__" in vars(mod.Box) # python: False; compiled: True +mod.Box.__new__ is object.__new__ # python: True; compiled: False +``` + ### an interpreted subclass whose `__init_subclass__` does not chain up a compiled class keeps an `int` field unboxed, and a field no `__init__` has set holds a @@ -1412,6 +1664,32 @@ alive() is not None # python: False; compiled: True, for as long as `kept` live a cell for each captured name would close this, and that is a different shape for every closure a compiled frame makes +### a nested function's closure + +python gives each name a nested function captures a cell, and answers `f.__closure__` with +a tuple of them, or `None` where it captures nothing. a compiled nested function keeps the +names its frame's closures capture in one environment they share — see +[what a closure keeps alive](#what-a-closure-keeps-alive) — and has no cells to hand over. +cells made for the asking would hold the values the names have when they are made, and a +later write to the name, or to a cell's `cell_contents`, would not reach the other. so one +that captures nothing answers `None`, as python's does, and one that captures a name has no +`__closure__` at all: + +```python +def outer(k: int): + def inner(x: int) -> int: + return x + k + + return inner + + +# from another module +mod.outer(1).__closure__ # python: (,); compiled: AttributeError +inspect.getclosurevars(mod.outer(1)) # python: ClosureVars(nonlocals={'k': 1}, ...); compiled: TypeError +``` + +`inspect.getclosurevars` refuses any compiled function, which is not a `types.FunctionType` + ### a frozen field written through `object.__setattr__` python's `frozen data class` refuses a write in `__setattr__` and leaves each field @@ -1426,7 +1704,7 @@ frozen data class Fixed: f = Fixed(1) -object.__setattr__(f, "n", 2) # python: stored; compiled: AttributeError: attribute 'n' of 'mod.Fixed' objects is not writable +object.__setattr__(f, "n", 2) # python: stored; compiled: AttributeError: attribute 'n' of 'Fixed' objects is not writable object.__delattr__(f, "n") # python: deleted; compiled: the same AttributeError ``` @@ -1494,6 +1772,9 @@ f = outer(int) - `__defaults__` and `__kwdefaults__` can be read and not written, and `__code__` is the interpreted definition's code object, which the compiled function does not run +what is said of 3.13 here holds of 3.11 and 3.12 as well, where a function has no +`__type_params__` at all + evaluating the annotations where the `def` stands is a python call, which made a closure with annotations cost about 3,400 more instructions to make on 3.13 than one without — the price of a `def` inside a loop. what the `def` takes now is the values alone, so an @@ -1516,6 +1797,96 @@ f = outer() # python 3.13: prints `y`; compiled: prints nothing f.__annotations__ # compiled: prints `y` ``` +### python 3.11 and 3.12 + +a compiled module answers as the interpreter it was built for answers, on every supported +version — 3.11, 3.12, 3.13 and 3.14. where python itself changed between them, the +generated `==` of a data class, the wording of an error, how an async generator's +awaitables refuse a second caller, a `throw` into a delegation, it is the running +version's rule that is followed. what is left is listed here. + +a class that keeps fields of its own past a base's instance is built by appending +storage to that base's layout, and the way to ask where appended storage is, PEP 697, +is 3.12's. below 3.12 such a class has no construction, and the module holding it stands +down whole at import: every class and function in it is the interpreted definition, and +it answers as python does, at python's speed: + +```python +class Failure(Exception): + def __init__(self, reason: str) -> None: + Exception.__init__(self, reason) + self.reason = reason # a field past `Exception`'s instance + + +def count(xs: list[int]) -> int: + return len(xs) # on 3.11, interpreted along with `Failure` +``` + +python 3.11 counts a call into a builtin against the recursion limit, on every call site +the interpreter has not yet specialised, where 3.12 counts it against the C stack alone. +a compiled module takes the frames python's own functions, methods and `==` take, and +none for its calls into builtins, so a recursion that goes round through one — a key +function, a callback, `functools.reduce` — reaches the limit later on 3.11 than python +does from a cold call site: + +```python +import functools + + +def fold(n: int) -> int: + if n == 0: + return 0 + return functools.reduce(lambda acc, x: acc + fold(x), [n - 1], 1) + + +# from another module, on 3.11 +sys.setrecursionlimit(100) +mod.fold(33) # python: RecursionError; compiled: 33 +``` + +python's own answer moves with how warm the call site is, so there is no one depth to +match; counting each such call would put a cost on every call into a builtin + +a dunder python calls through a type slot is held as the slot's wrapper — see +[a dunder a slot answers](#a-dunder-a-slot-answers) — and calling one by name, +`Base.__init__(self, …)`, `super().__init__(…)` or `Node.__len__(n)`, is a call into that +builtin, which 3.11 counts. the compiled body counts the frame python's `def` would have +pushed as well, so on 3.11 a recursion that goes round through such a call reaches the +limit a level earlier than python's for each one: + +```python +class Node: + def __init__(self, f: Callable[[int], int], n: int) -> None: + self.v = f(n) + + +# from another module, on 3.11 +def down(n): + node = mod.Node.__new__(mod.Node) + mod.Node.__init__(node, down, n - 1) if n else None + return node.v + 1 if n else 0 + + +sys.setrecursionlimit(100) +down(40) # python: 40; compiled: RecursionError +``` + +a data class's `==` compares two tuples of its fields below 3.13, and a tuple finds two +identical objects equal without asking them. a `float` field is kept as a number rather +than as the object it was given, so two instances sharing one NaN object are unequal +where python finds the one object in both. a field of any other type is kept as its +object, and an instance compared with itself is equal on both: + +```python +frozen data class Reading: + value: float + + +# from another module, on 3.11 or 3.12 +nan = float("nan") +mod.Reading(nan) == mod.Reading(nan) # python: True; compiled: False +``` + ## debugging and inspection - **`#line` directives** in the generated C point at `.by` source. gdb, lldb, diff --git a/docs/basedpython/development/compilation/technology.md b/docs/basedpython/development/compilation/technology.md index 2cc55c134e..11e769b8f3 100644 --- a/docs/basedpython/development/compilation/technology.md +++ b/docs/basedpython/development/compilation/technology.md @@ -219,6 +219,13 @@ and `by.h` restates it as an `#error` for a compile that did not come through wording of the unbound-local error now, one `By_IterSend`, and the match-shape tests stand unguarded +above the floor, every version is supported: 3.11, 3.12, 3.13 and 3.14, and the +free-threaded builds of the last two. ci runs the backend's whole suite against each of +them. a compiled module answers as the interpreter it was built for answers, so where +python changed between those versions the emitted C follows the running one, and the few +places it cannot are declined there or written down in +[runtime.md](runtime.md#python-311-and-312) + ### an artefact is pinned to one minor version, and says so itself taking the full API means the runtime header reads layouts that move between diff --git a/docs/basedpython/development/how-transpilation-works.md b/docs/basedpython/development/how-transpilation-works.md index 3f880c4d93..cd1fb9f9b3 100644 --- a/docs/basedpython/development/how-transpilation-works.md +++ b/docs/basedpython/development/how-transpilation-works.md @@ -34,14 +34,14 @@ source (.by) │ └─ rewrite `from typing import X` to `from typing_extensions import X` │ when X is not yet stdlib at the configured min Python version │ - ├─ phase 2b anon-named-tuple cleanup - │ └─ re-run `anon_named_tuple` and the inline-protocol lowering on the - │ post-lowering output to catch spans copied verbatim by other transforms - │ (e.g. the PEP-695 polyfill). bounded to a few iterations. an output that - │ spells no anonymous named tuple and no inline protocol is passed over - │ after a parse, without a db or any type inference: the checker makes an - │ anonymous named tuple type only from such a form, so no plain tuple in - │ it can need constructing as one + ├─ phase 2b synthesized-type cleanup + │ └─ re-run the synthesized-type lowerings on the post-lowering output to + │ catch a typed dict, anonymous named tuple or inline protocol copied + │ verbatim by other transforms (e.g. the PEP-695 polyfill). bounded to a + │ few iterations. an output that spells no anonymous named tuple and no + │ inline protocol is passed over after a parse, without a db or any type + │ inference: the checker makes an anonymous named tuple type only from + │ such a form, so no plain tuple in it can need constructing as one │ ├─ phase 2c lazy-import marking │ └─ lower imports to the `lazy` keyword (3.15+) or a runtime polyfill @@ -49,9 +49,10 @@ source (.by) ├─ phase 2d version polyfill │ └─ rewrite syntax the target python cannot parse. today that is the │ `match` statement, lowered to an `if`/`elif` chain for a target below - │ 3.10. it runs over the *finished* python rather than over `.by`, so a - │ `match` an earlier lowering generated is lowered by the same code as - │ one the author wrote + │ 3.10, and a bracketless `except`, whose types are parenthesized for a + │ target below 3.14. it runs over the *finished* python rather than over + │ `.by`, so a `match` an earlier lowering generated is lowered by the + │ same code as one the author wrote │ └─ phase 3 syntax verification ├─ parse the final output as `.py` — any parse error aborts with a @@ -208,6 +209,16 @@ and gets the single-file db: correct, just blind past the file text edits keyed by `TextRange`. it never mutates the AST, because `inferred_type` binds to the exact parsed node identities +a typed dict literal, an anonymous named tuple type and an inline protocol each +lower to a class, and each can be the type of another's field, so their classes +are named before any pass runs, in one walk that names a type only once every type +inside it has its class (`transforms/synthesized_types.rs`). each class body is +printed through one type-expression lowerer that writes the classes named so far, +and the classes are declared in the order they were named, so each comes after +every class it reads. the passes lowering each kind write those names, and every +other pass that prints a type expression whole is handed them alongside the +symbolic folds, as `type_substitutions` + order matters: passes that target the same offset rely on a fixed sequence (e.g. `type_is` before `identity_swap`, `coalesce` before `none_chain`). the ordered lists live in `run_against_source`. a complete list of transforms is at diff --git a/docs/basedpython/features/anonymous-named-tuple.md b/docs/basedpython/features/anonymous-named-tuple.md index e5f1c317bc..95f60e2590 100644 --- a/docs/basedpython/features/anonymous-named-tuple.md +++ b/docs/basedpython/features/anonymous-named-tuple.md @@ -185,6 +185,22 @@ class _Anon(NamedTuple): it is a subclass of the heterogeneous `tuple[str, int]`, so it can be used anywhere that tuple type is expected +the class is declared ahead of the module, where a named tuple's field annotations are read as the +class is made. a field naming something the module binds — a class or an import of the module — +is written as a string there, which `typing.get_type_hints` resolves once the module has run: + +```by +class Point: ... + +a: (origin: Point, label: str) +``` + +```python +class _Anon(NamedTuple): + origin: "Point" + label: str +``` + ## structural identity two anonymous named tuples with the same field names *and* the same field diff --git a/docs/basedpython/features/polyfills.md b/docs/basedpython/features/polyfills.md index 66ad8a5b65..4e2330c4d5 100644 --- a/docs/basedpython/features/polyfills.md +++ b/docs/basedpython/features/polyfills.md @@ -58,6 +58,8 @@ except (TimeoutError, ConnectionRefusedError): ... ``` +the same goes for `except*` + ### template strings / t-strings (PEP 750) `t'...'` literals produce `string.templatelib.Template` objects. below 3.14 there is no runtime @@ -342,7 +344,7 @@ ______________________________________________________________________ ### the `match` statement (PEP 634) -`match` is grammar, so a target below 3.10 cannot even parse a file containing one. every `match` in the output — the ones you wrote, and the ones other lowerings produce for [`let` destructuring](destructuring.md), [`if let`](if-let.md), [statement expressions](statement-expressions.md) and [enum](enums.md) exhaustiveness — becomes an `if`/`elif` chain whose conditions do the matching: +`match` is grammar, so a target below 3.10 cannot even parse a file containing one. every `match` statement in the output — the ones you wrote, and the ones [statement expressions](statement-expressions.md) and [enum](enums.md) exhaustiveness produce — becomes an `if`/`elif` chain whose conditions do the matching: ```python # python source @@ -370,6 +372,8 @@ two things are not reproduced: a comment written *inside* a multi-line pattern i the lowering binds with assignment expressions, so it needs python 3.8. below that a `match` is reported rather than lowered +[destructuring](destructuring.md) — a `let`, an `if let` or `elif let`, and a pattern written as a `for` or `with` target or as a parameter — is written as a `match` statement too, but is not lowered: it needs a target of 3.10 or later, and an older one is reported + ### `X | Y` at runtime (PEP 604) `int | str` calls `type.__or__`, which arrived in 3.10. in an *annotation* that costs nothing — a target this old always gets `from __future__ import annotations`, so no annotation is ever evaluated — but where the value is really produced it is a `TypeError` at import time. those are spelled the way the target can: diff --git a/docs/basedpython/features/repeated-underscore.md b/docs/basedpython/features/repeated-underscore.md index c5603a7388..5d3cf30375 100644 --- a/docs/basedpython/features/repeated-underscore.md +++ b/docs/basedpython/features/repeated-underscore.md @@ -138,6 +138,48 @@ scope, so an override whose body reads one is refused as ## reading `_` -references to `_` in the body resolve to the first `_` parameter. in an override -that took its names from the base, the first of them is bound to `_` at the start -of the body, so a read of `_` still finds it +the `_`s are there to be ignored. python keeps only one parameter under the name +`_`, so a read of `_` could mean any of them, and reading one is an +`invalid-repeated-underscore` error: + +```by +def pick(_: int, _: str) -> str: + return _ # error +``` + +name the parameter you want to read instead. this covers every read the +parameters may reach: in the body, in `_ += 1`, and in a nested function, lambda +or comprehension that reads `_` from the function around it. an override is no +exception, though its `_`s take the base's names in the python + +a read of `_` that the body has bound again on every path to it reads that +binding, and is fine: + +```by +def count(_: int, _: int) -> int: + _ = 0 + return _ +``` + +so is `del _`, which reads no value, and a single `_` parameter, which is read +like any other parameter + +## writing `_` + +each `_` becomes a parameter of its own name in the python, so none of their +annotations is the type of the `_` the body binds. a write to `_` is checked +against none of them: it is an ordinary local variable, and a read that only the +write reaches has the type of the value written: + +```by +def first(_: int, _: str) -> int: + _ = 5 # not checked against `int` or `str` + return _ +``` + +the same goes for every other way of binding `_`: unpacking, `for _ in`, `:=`, +`with ... as _`, and a nested function's `nonlocal _`. `_ += 1` is no write of +this kind, since it reads `_` first, and is an error like any other read + +an annotation the body writes for `_` declares it, as it would any local, so a +write after `_: bytes = b""` has to be a `bytes` diff --git a/docs/basedpython/features/super.md b/docs/basedpython/features/super.md index 529982004b..645b74a28e 100644 --- a/docs/basedpython/features/super.md +++ b/docs/basedpython/features/super.md @@ -23,10 +23,46 @@ method context (it pulls the enclosing class from the `__class__` cell), so `super.x` outside a method is a runtime error — exactly like the python it lowers to +the receiver is read where `super` is written, as `super()` reads it: the first +parameter of the function or lambda around it. a lambda passes its own first +argument, not the method's, and a lambda with no parameter, or a comprehension, +has no receiver to pass and is reported: + +```by +class B(A): + def f(self): + g = lambda other: super.greet() # `other` is the receiver +``` + +the keyword is the base of an attribute access, and it is the builtin `super` +there even where the module binds the name to something else. read anywhere +else, `super` is the name as python reads it: + +```by +class B(A): + def f(self, super: int): + super.greet(super) # the builtin's `greet`, passed the parameter +``` + +## assigning through `super` + +a `super()` object has no attributes of its own, and does not pass an +assignment or a deletion on to the class it reads past, so python raises +`AttributeError` for `super().x = v` and `del super().x` whatever `x` is. the +keyword is that object, so `super.x = v`, `super.x += v` and `del super.x` are +reported. assign through the receiver instead: + +```by +class B(A): + def f(self): + self.x = 1 # not `super.x = 1` +``` + ## targeted form — `super[T].x` `super[T].x` resolves `x` against the MRO entry that **precedes** `T` in the -enclosing class' MRO +enclosing class' MRO. like the bare form, `super[T]` is the keyword only as the +base of an attribute access; anywhere else it is a subscript of the name example: @@ -44,3 +80,14 @@ class C(A, B): C's MRO is `(C, A, B, object)`; the entry preceding `B` is `A`, so the bound super starts the search after `A` and lands on `B.f` + +`T` has to be a class in that MRO other than the enclosing class itself, which +has nothing before it. anything else is reported, since the lookup would raise +`ValueError` when it runs — so is a value of a `type[...]` type, which may be a +subclass the MRO does not hold: + +```by +class C(A): + def f(self): + super[B].f() # error: `B` is not in the MRO of `C` +``` diff --git a/docs/basedpython/features/template-literal-types.md b/docs/basedpython/features/template-literal-types.md index c5dc1bf052..0804380742 100644 --- a/docs/basedpython/features/template-literal-types.md +++ b/docs/basedpython/features/template-literal-types.md @@ -179,3 +179,11 @@ the emitted annotation is read back from the checker, so it cannot disagree with what basedpython decided the pattern meant. the lowering is one-way: a `str` annotation carries no evidence that it was ever a pattern, so [reverse transpilation](../development/reverse-transforms.md) leaves it alone + +none of the f-string reaches the python, so neither does its syntax. a hole may +reuse the f-string's own quotes whatever python the output is for, which an +f-string *value* can only do from python 3.12: + +```by +kind: f"{"get" | "set"}-{int}" # fine on a python 3.11 target +``` diff --git a/docs/basedpython/features/type-modifiers.md b/docs/basedpython/features/type-modifiers.md index 0d48de448e..c76f888da6 100644 --- a/docs/basedpython/features/type-modifiers.md +++ b/docs/basedpython/features/type-modifiers.md @@ -163,8 +163,9 @@ anywhere a type expression appears: a variable annotation, a parameter, a return type, a [type alias](../development/how-transpilation-works.md) value, a type parameter's bound or default, a [`cast`](cast.md) target, an [inline protocol](inline-protocol.md) member, an -[anonymous named tuple](anonymous-named-tuple.md) field, and nested inside a -subscript, a `Callable[[…], R]` parameter list or a parenthesis +[anonymous named tuple](anonymous-named-tuple.md) field, a +[typed dict literal](typed-dict-literal.md) value or its `**: T` extra items, and +nested inside a subscript, a `Callable[[…], R]` parameter list or a parenthesis each member of a [type mapping](type-mappings.md) is its own type position, so a modifier may be written inside one: `T in (literal str, literal int)` @@ -220,6 +221,21 @@ produced python. every other modifier is erased: | `d: final int` | `d: int` | | `e: final str` | `e: str` | +it is written as `LiteralString` wherever the type is, a tuple type or a type +parameter's bound and constraints among them, and so the `TypeVar` a type +parameter is [polyfilled](polyfills.md) to below 3.12 is bound by it too: + +```by +def f[T: literal str](t: T) -> T: + return t +``` + +```python +_T = TypeVar("_T", bound=LiteralString) +def f(t: _T) -> _T: + return t +``` + the `LiteralString` import is added once per module and [polyfilled](polyfills.md) to `typing_extensions` below 3.11 diff --git a/docs/basedpython/features/typed-dict-literal.md b/docs/basedpython/features/typed-dict-literal.md index 67768a765f..db512c7510 100644 --- a/docs/basedpython/features/typed-dict-literal.md +++ b/docs/basedpython/features/typed-dict-literal.md @@ -62,9 +62,8 @@ a pack that hasn't been specialized yet shows as the pending splice — `{"tag": ## lowering each shape is hoisted to one module-level `TypedDict` class. those classes land ahead of -everything the module defines, and a field can name a class declared later or a type parameter -that only exists inside the generic class the literal was written in, so field types are emitted -as forward references: +everything the module defines, and a field can name a class declared later, so field types are +emitted as forward references: ```python class _TypedDict_(TypedDict, closed=True): @@ -72,6 +71,11 @@ class _TypedDict_(TypedDict, closed=True): age: "int" ``` +a field naming a type parameter of the generic the literal was written in reads that parameter +from module scope, where `typing.get_type_hints` resolves it, and a field whose type is another +typed dict literal, an [anonymous named tuple](anonymous-named-tuple.md) or an +[inline protocol](inline-protocol.md) names that type's class + a `{**Kwargs}` splice has no fields to erase to — they are only known at the specialization site, and python erases type arguments anyway — so it lowers to `extra_items="object"` diff --git a/ty.schema.json b/ty.schema.json index 8687669929..856fd165a6 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -1612,8 +1612,8 @@ ] }, "invalid-repeated-underscore": { - "title": "detects a repeated `_` parameter the lowering has no python spelling for", - "description": "## What it does\n\nChecks for a parameter list that repeats `_` in a shape the lowering has no python spelling for.\n\n## Why is this bad?\n\nPython refuses two parameters of one name, so every `_` after the first is given a name of its own.\nThat name is the lowering's rather than the author's, so a call reaches the parameter by position\nalone: it is positional-only. Where that cannot be written, or would change a parameter the author\nnamed, the lowering refuses the definition. Reported here, the refusal arrives while the file is\nbeing checked rather than when it is transpiled.\n\n## Examples\n\nPython's `/` makes every parameter before it positional-only, so a named parameter ahead of a\nrepeated `_` would stop being reachable by keyword:\n\n```by\n# error: [invalid-repeated-underscore]\ndef pair(a: int, _: int, b: int, _: int) -> int:\n return a + b\n```\n\nWriting the `/` says so, and is accepted:\n\n```by\ndef explicit(a: int, _: int, b: int, _: int, /) -> int:\n return a + b\n```\n\nA repeated `_` after `*` is keyword-only, and only a keyword reaches it:\n\n```by\n# error: [invalid-repeated-underscore]\ndef keyword(_: int, *, _: int) -> None: ...\n```\n\nPython before 3.8 has no `/` at all, so a module targeting it cannot make a repeated `_`\npositional-only, and every definition that needs the `/` is refused there.\n\nA method that overrides one takes the names the overridden method gives those positions. A name the\nbody reads from an enclosing scope would then read the parameter instead. Refused, the override\nkeeps no names a caller of the base could pass by keyword, which is reported too:\n\n```by\nx = 1\n\nclass Base:\n def f(self, x: int, y: int) -> int:\n return x + y\n\nclass Override(Base):\n # error: [invalid-repeated-underscore]\n # error: [invalid-method-override]\n override def f(self, _: int, _: int) -> int:\n return x\n```", + "title": "detects a repeated `_` parameter the lowering has no python spelling for, or a read of one", + "description": "## What it does\n\nChecks for a parameter list that repeats `_` in a shape the lowering has no python spelling for, and\nfor a read of `_` that may be one of those parameters.\n\n## Why is this bad?\n\nPython refuses two parameters of one name, so every `_` after the first is given a name of its own.\nThat name is the lowering's rather than the author's, so a call reaches the parameter by position\nalone: it is positional-only. Where that cannot be written, or would change a parameter the author\nnamed, the lowering refuses the definition. Reported here, the refusal arrives while the file is\nbeing checked rather than when it is transpiled.\n\n## Examples\n\nPython's `/` makes every parameter before it positional-only, so a named parameter ahead of a\nrepeated `_` would stop being reachable by keyword:\n\n```by\n# error: [invalid-repeated-underscore]\ndef pair(a: int, _: int, b: int, _: int) -> int:\n return a + b\n```\n\nWriting the `/` says so, and is accepted:\n\n```by\ndef explicit(a: int, _: int, b: int, _: int, /) -> int:\n return a + b\n```\n\nA repeated `_` after `*` is keyword-only, and only a keyword reaches it:\n\n```by\n# error: [invalid-repeated-underscore]\ndef keyword(_: int, *, _: int) -> None: ...\n```\n\nPython before 3.8 has no `/` at all, so a module targeting it cannot make a repeated `_`\npositional-only, and every definition that needs the `/` is refused there.\n\nA method that overrides one takes the names the overridden method gives those positions. A name the\nbody reads from an enclosing scope would then read the parameter instead. Refused, the override\nkeeps no names a caller of the base could pass by keyword, which is reported too:\n\n```by\nx = 1\n\nclass Base:\n def f(self, x: int, y: int) -> int:\n return x + y\n\nclass Override(Base):\n # error: [invalid-repeated-underscore]\n # error: [invalid-method-override]\n override def f(self, _: int, _: int) -> int:\n return x\n```\n\nPython binds `_` to only one of the parameters, the first, while ty reads a body as binding each `_`\nin turn. A read of `_` would mean a different parameter to each, so it is reported, wherever the\nparameters may be the binding it reads — in the body, in a nested function, lambda or comprehension,\nand in `_ += 1`:\n\n```by\ndef first(_: int, _: str) -> str:\n # error: [invalid-repeated-underscore]\n return _\n```\n\nA read that only a binding the body makes reaches reads that binding, and `del _` reads no value:\n\n```by\ndef rebound(_: int, _: int) -> int:\n _ = 5\n return _\n```", "default": "error", "oneOf": [ {